diff --git a/README.md b/README.md index dd94c8e..438bdff 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,22 @@ For development purposes see the `docs` directory for notes on development, depl An API for `masquer` is in deployment at `https://masquer.fly.dev/masq` — try it out with the interactive [Swagger UI](https://masquer.fly.dev/docs) or [ReDoc](https://masquer.fly.dev/redoc) documentation. -The API returns JSON objects, making it compatible with any language that can make HTTP requests and parse JSON. +The API returns a JSON array, making it compatible with any language that can make HTTP requests and parse JSON. Here is an example using `curl` from the command line to get a random user-agent and referer: ```console -$ curl -X GET 'https://masquer.fly.dev/masq?ua=true&rf=true' -H 'accept: application/json' -{ - "Referer":"https://www.google.com", - "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3" -} +$ curl -X GET 'https://masquer.fly.dev/api/v1/masq?ua=true&rf=true' -H 'accept: application/json' +[ + { + "Referer":"https://www.google.com", + "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3" + } +] ``` +The optional `count` parameter specifies the number of objects to return in the response. The default value is `1`. + Refer to the [API docs](`https://masquer.fly.dev/docs`) for other examples, or see [more details below](#examples) in the package documentation.

diff --git a/assets/useragents.json b/assets/useragents.json index 4f497e2..4b79cbf 100644 --- a/assets/useragents.json +++ b/assets/useragents.json @@ -1 +1 @@ -[{"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3", "pct": 33.33}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", "pct": 24.67}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3", "pct": 15.75}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.", "pct": 7.87}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.", "pct": 5.77}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", "pct": 2.1}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.", "pct": 1.57}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 OPR/112.0.0.", "pct": 1.57}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.", "pct": 1.05}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", "pct": 1.05}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", "pct": 0.52}, {"ua": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", "pct": 0.52}, {"ua": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.", "pct": 0.52}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.14", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.10", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 0.52}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.3", "pct": 0.52}] \ No newline at end of file +[{"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", "pct": 33.73}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", "pct": 30.6}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.", "pct": 7.71}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.3", "pct": 7.23}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.", "pct": 4.82}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.", "pct": 4.34}, {"ua": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 1.93}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", "pct": 1.93}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.3", "pct": 0.96}, {"ua": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.3", "pct": 0.96}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.", "pct": 0.96}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", "pct": 0.96}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/26.0 Chrome/122.0.0.0 Safari/537.3", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Geck", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Config/91.2.1916.1", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 OPR/112.0.0.", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.3", "pct": 0.48}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.", "pct": 0.48}] \ No newline at end of file diff --git a/docs/deployment.md b/docs/deployment.md index f7ce537..ece9540 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -82,7 +82,7 @@ If the app launches successfully, visit the documentation link provided and test If the tests work from the documentation, make a `curl` request from a separate terminal window: ```console -$ curl -X GET 'http://127.0.0.1:8000/masq?ua=true&rf=true' -H 'accept: application/json' +$ curl -X GET 'http://127.0.0.1:8000/api/v1/masq?ua=true&rf=true&count=2' -H 'accept: application/json' ``` If the expected response is received, proceed to the Docker image tests. diff --git a/docs/development.md b/docs/development.md index 4332493..e444cf0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -157,8 +157,8 @@ YYYY-MM-DD HH:MM:SS - {log-level} - {file}:{line-number} - {message} Two examples are displayed below: ``` -2024-09-17 14:52:26 - INFO - routes.py:19 - Request: [ua=True rf=False hd=False] -2024-09-21 13:13:34 - WARNING - app.py:29 - Invalid args: [ua='True' rf=False hd=False] +2024-09-17 14:52:26 - INFO - routes.py:19 - Request: [ua=True rf=False hd=False count=1] +2024-09-21 13:13:34 - WARNING - app.py:29 - Invalid args: [ua='True' rf=False hd=False count=1] ``` ### Output diff --git a/docs/docker.md b/docs/docker.md index 1be2342..0e5a218 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -108,9 +108,16 @@ The first several characters of the container's `sha256` are displayed under the With the container running, test a few API calls with different arguments and verify that the expected output is received, for example: ```console -$ curl -X GET 'http://127.0.0.1:8000/masq' -H 'accept: application/json' -$ curl -X GET 'http://127.0.0.1:8000/masq?ua=true&rf=true&hd=true' -H 'accept: application/json' -$ curl -X GET 'http://127.0.0.1:8000/masq?ua=true&rf=true&hd=false' -H 'accept: application/json' +$ curl -X GET 'http://127.0.0.1:8000/api/v1/masq' -H 'accept: application/json' +$ curl -X GET 'http://127.0.0.1:8000/api/v1/masq?ua=true&rf=true&hd=true&count=3' -H 'accept: application/json' +$ curl -X GET 'http://127.0.0.1:8000/api/v1/masq?ua=true&rf=true&hd=false' -H 'accept: application/json' +``` + +Proceed to the next stage if the tests are successful — if not, stop the container daemon with: + +```console +$ docker stop 392 # change '392' to match the container's ID +392 ```

@@ -119,10 +126,11 @@ $ curl -X GET 'http://127.0.0.1:8000/masq?ua=true&rf=true&hd=false' -H 'accept: ### Check logs on container -If the API is working as expected, stop the container daemon and then run a shell on the container in the foreground to check that expected logs were generated. +Start the container if it is not currently running, then enter the shell on the container to view the logs: ```console -$ docker stop 392 # change '392' to match the container's ID +$ docker start 392 # change '392' to match the container's ID +392 $ docker exec -it 392 sh # 'sh' runs the basic shell /usr/src/app/src $ ls ``` @@ -136,8 +144,8 @@ Change to the `logs` directory and `cat` the `app.log` contents — output s ```console /usr/src/app/src $ cd ../logs /usr/src/app/logs $ cat app.log -2024-09-22 04:34:25 - INFO - main.py:43 - FastAPI app init OK -2024-09-22 04:36:59 - INFO - routes.py:19 - Request: [ua=True rf=False hd=False] +2024-09-28 04:34:25 - INFO - main.py:43 - FastAPI app init OK +2024-09-28 04:36:59 - INFO - routes.py:19 - Request: [ua=True rf=False hd=False count=1] ... ``` diff --git a/fly.toml b/fly.toml index e0eb1a8..805ee58 100644 --- a/fly.toml +++ b/fly.toml @@ -11,12 +11,12 @@ primary_region = 'sin' [http_service] internal_port = 8000 force_https = true - auto_stop_machines = true + auto_stop_machines = 'suspend' auto_start_machines = true - min_machines_running = 0 + min_machines_running = 1 processes = ['app'] [[vm]] memory = '1gb' cpu_kind = 'shared' - cpus = 1 + cpus = 2 diff --git a/src/api/main.py b/src/api/main.py index c17028a..0c54400 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -27,13 +27,13 @@ ### cURL ```console -$ curl -X GET 'https://masquer.fly.dev/masq?ua=true&rf=false&hd=false' -H 'accept: application/json' +$ curl -X GET 'https://masquer.fly.dev/api/v1/masq?ua=true&rf=false&hd=false&count=3' -H 'accept: application/json' ``` ### JavaScript (using `fetch`) ```javascript -fetch('https://masquer.fly.dev/masq?ua=true&rf=false&hd=false') +fetch('https://masquer.fly.dev/api/v1/masq?ua=true&rf=false&hd=false&count=3') .then(response => response.json()) .then(data => console.log(data)) ``` @@ -42,7 +42,7 @@ ```python import requests -response = requests.get("https://masquer.fly.dev/masq", params={"ua": True, "rf": False, "hd": False}) +response = requests.get("https://masquer.fly.dev/api/v1/masq", params={"ua": True, "rf": False, "hd": False, "count": 3}) print(response.json()) ``` """ diff --git a/src/api/routes.py b/src/api/routes.py index ff54df1..f359a4a 100644 --- a/src/api/routes.py +++ b/src/api/routes.py @@ -9,15 +9,37 @@ logger = get_logger(__name__) router = APIRouter() +MAX_COUNT = 250 +MIN_COUNT = 1 -@router.get("/masq") -def get_masq( + +@router.get("/masq") # maintain for backwards compatibility +@router.get("/api/v0/masq") +def get_masq_v0( ua: Union[bool, None] = True, rf: Union[bool, None] = False, hd: Union[bool, None] = False, ): logger.info(f"Request: [{ua=} {rf=} {hd=}]") response = masq(ua, rf, hd) - logger.info(f"Response: [{response}]") + logger.debug(f"Response: {response}") + + return JSONResponse(content=response) + + +@router.get("/api/v1/masq") +def get_masq( + ua: Union[bool, None] = True, + rf: Union[bool, None] = False, + hd: Union[bool, None] = False, + count: Union[int, None] = 1, +): + logger.info(f"Request: [{ua=} {rf=} {hd=} {count=}]") + if count >= MAX_COUNT: + count = MAX_COUNT + if count <= MIN_COUNT: + count = MIN_COUNT + response = [masq(ua, rf, hd) for _ in range(count)] + logger.debug(f"Response: {response}") return JSONResponse(content=response) diff --git a/src/masquer/__about__.py b/src/masquer/__about__.py index bc86c94..67bc602 100644 --- a/src/masquer/__about__.py +++ b/src/masquer/__about__.py @@ -1 +1 @@ -__version__ = "1.2.2" +__version__ = "1.3.0" diff --git a/src/masquer/utils/assets.py b/src/masquer/utils/assets.py index 4ed30d3..42f92fa 100644 --- a/src/masquer/utils/assets.py +++ b/src/masquer/utils/assets.py @@ -20,46 +20,46 @@ ] REFERER_WEIGHTS = [79.65, 11.98, 3.21, 2.47, 0.88, 0.56] USERAGENTS = [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36 Edg/129.0.0.", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/26.0 Chrome/122.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/115.", + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Geck", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Config/91.2.1916.1", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 OPR/112.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", - "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.14", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.10", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.3", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.3", ] USERAGENT_WEIGHTS = [ - 33.33, - 24.67, - 15.75, - 7.87, - 5.77, - 2.1, - 1.57, - 1.57, - 1.05, - 1.05, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, - 0.52, + 33.73, + 30.6, + 7.71, + 7.23, + 4.82, + 4.34, + 1.93, + 1.93, + 0.96, + 0.96, + 0.96, + 0.96, + 0.48, + 0.48, + 0.48, + 0.48, + 0.48, + 0.48, + 0.48, + 0.48, ] diff --git a/src/masquer/utils/response.py b/src/masquer/utils/response.py index da9e3ef..f103d5c 100644 --- a/src/masquer/utils/response.py +++ b/src/masquer/utils/response.py @@ -27,7 +27,7 @@ def get_response( response_data = dict() if header_requested: - response_data = HEADER_DATA + response_data = dict(HEADER_DATA) if referer_requested: referer = select_data(REFERERS, REFERER_WEIGHTS)