diff --git a/README.md b/README.md index 7561b6a..7958588 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- +

@@ -235,22 +235,17 @@ The following details will assist with making and testing changes to a cloned ve ### Updates -The root directory includes `update.py`: if you make changes to the JSON assets stored in the `assets` directory, sync those changes with the `assets.py` file inside the `masquer` package by running `update.py` from the terminal: +The root directory includes a shell script `update.sh` for convenience to update JSON assets for the `masquer` package. ![](https://img.shields.io/badge/Linux-FCC624.svg?style=flat&logo=Linux&logoColor=black) -![](https://img.shields.io/badge/macOS-000000.svg?style=flat&logo=Apple&logoColor=white) ```console -$ python3 update.py -Asset update successful +$ chmod +x update.sh +$ ./update.sh +Update successful ``` -![](https://img.shields.io/badge/Windows-0078D4.svg?style=flat&logo=Windows&logoColor=white) - -```console -$ python update.py -Asset update successful -``` +If using your own JSON data you can still make use of the `update_assets()` function inside `update.py` to sync changes with the `assets.py` file inside the `masquer` package. ### Tests diff --git a/assets/referers.json b/assets/referers.json index 55371f2..74b0129 100644 --- a/assets/referers.json +++ b/assets/referers.json @@ -1,26 +1 @@ -[ - { - "ref": "https://www.google.com/", - "pct": 80.51 - }, - { - "ref": "https://bing.com/", - "pct": 11.37 - }, - { - "ref": "https://search.yahoo.com/", - "pct": 2.97 - }, - { - "ref": "https://yandex.com/", - "pct": 2.32 - }, - { - "ref": "https://duckduckgo.com/", - "pct": 0.72 - }, - { - "ref": "https://www.baidu.com/", - "pct": 0.59 - } -] \ No newline at end of file +[{"ref": "https://www.google.com", "pct": 80.51}, {"ref": "https://bing.com", "pct": 11.37}, {"ref": "https://search.yahoo.com", "pct": 2.97}, {"ref": "https://yandex.com", "pct": 2.32}, {"ref": "https://duckduckgo.com", "pct": 0.72}, {"ref": "https://www.baidu.com", "pct": 0.59}] \ No newline at end of file diff --git a/assets/useragents.json b/assets/useragents.json index 4455687..10ea384 100644 --- a/assets/useragents.json +++ b/assets/useragents.json @@ -1,102 +1 @@ -[ - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.3", - "pct": 27.78 - }, - { - "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.3", - "pct": 26.92 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", - "pct": 9.83 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.", - "pct": 9.4 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", - "pct": 5.56 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", - "pct": 4.27 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.", - "pct": 2.56 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", - "pct": 2.14 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.4", - "pct": 1.71 - }, - { - "ua": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "pct": 1.28 - }, - { - "ua": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Geck", - "pct": 0.85 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", - "pct": 0.85 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.5", - "pct": 0.85 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "pct": 0.85 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.", - "pct": 0.85 - }, - { - "ua": "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/115.", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/25.0 Chrome/121.0.0.0 Safari/537.3", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.3", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.3", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.3", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Geck", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.3", - "pct": 0.43 - }, - { - "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.", - "pct": 0.43 - } -] \ No newline at end of file +[{"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.3", "pct": 27.66}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3", "pct": 9.07}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.1", "pct": 8.05}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.1", "pct": 6.69}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.", "pct": 6.35}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3", "pct": 5.9}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.1", "pct": 5.33}, {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.1", "pct": 4.65}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.", "pct": 4.08}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", "pct": 4.08}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", "pct": 3.17}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.", "pct": 1.81}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", "pct": 1.36}, {"ua": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 1.36}, {"ua": "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 0.91}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 OPR/110.0.0.", "pct": 0.91}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", "pct": 0.91}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", "pct": 0.91}, {"ua": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.3", "pct": 0.91}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/25.0 Chrome/121.0.0.0 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Geck", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.4", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.5", "pct": 0.45}, {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Avast/125.0.0.", "pct": 0.45}] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fa7ccfc..56a3209 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ [project.optional-dependencies] dev = [ + "beautifulsoup4==4.12.3", "fastapi==0.111.0", "hatch==1.11.1", "hatchling==1.24.2", diff --git a/src/masquer/__about__.py b/src/masquer/__about__.py index a82b376..c68196d 100644 --- a/src/masquer/__about__.py +++ b/src/masquer/__about__.py @@ -1 +1 @@ -__version__ = "1.1.1" +__version__ = "1.2.0" diff --git a/src/masquer/utils/assets.py b/src/masquer/utils/assets.py index b7b9b9b..e567e6c 100644 --- a/src/masquer/utils/assets.py +++ b/src/masquer/utils/assets.py @@ -11,65 +11,79 @@ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", } REFERERS = [ - "https://www.google.com/", - "https://bing.com/", - "https://search.yahoo.com/", - "https://yandex.com/", - "https://duckduckgo.com/", - "https://www.baidu.com/", + "https://www.google.com", + "https://bing.com", + "https://search.yahoo.com", + "https://yandex.com", + "https://duckduckgo.com", + "https://www.baidu.com", ] REFERER_WEIGHTS = [80.51, 11.37, 2.97, 2.32, 0.72, 0.59] USERAGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.3", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.4", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Geck", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.5", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.", "Mozilla/5.0 (Windows NT 6.1; rv:109.0) Gecko/20100101 Firefox/115.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 OPR/110.0.0.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.3", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/25.0 Chrome/121.0.0.0 Safari/537.3", - "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Geck", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.3", + "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.", + "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Geck", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.3", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.4", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.5", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Avast/125.0.0.", ] USERAGENT_WEIGHTS = [ - 27.78, - 26.92, - 9.83, - 9.4, - 5.56, - 4.27, - 2.56, - 2.14, - 1.71, - 1.28, - 0.85, - 0.85, - 0.85, - 0.85, - 0.85, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, - 0.43, + 27.66, + 9.07, + 8.05, + 6.69, + 6.35, + 5.9, + 5.33, + 4.65, + 4.08, + 4.08, + 3.17, + 1.81, + 1.36, + 1.36, + 0.91, + 0.91, + 0.91, + 0.91, + 0.91, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, + 0.45, ] diff --git a/update.py b/update.py index cf8e605..6ac2c02 100644 --- a/update.py +++ b/update.py @@ -1,5 +1,89 @@ import json import os +import requests +from bs4 import BeautifulSoup + + +# Get absolute path for root directory +ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) +# Path to assets and utils directories +ASSETS_DIR = os.path.join(ROOT_DIR, "assets") +UTILS_DIR = os.path.join(ROOT_DIR, "src", "masquer", "utils") + + +def update_useragents() -> bool: + """ + Gets latest useragent stats + Saves stats to JSON file + """ + try: + response = requests.get("https://www.useragents.me/") + html = BeautifulSoup(response.content, "html.parser") + # Find div for desktop stats + div = html.find("div", id="most-common-desktop-useragents-json-csv") + textarea = div.find(string="JSON").find_next("textarea").contents[0] + # Convert to JSON and save + json_string = json.loads(textarea) + with open(os.path.join(ASSETS_DIR, "useragents.json"), "w") as f: + json.dump(json_string, f) + + return True + + except Exception as e: + print(f"Error fetching useragent data: {e}") + return False + + +def update_referers() -> bool: + """ + Gets latest referer stats + Saves stats to JSON file + """ + SITES = { + "baidu": "https://www.baidu.com", + "bing": "https://bing.com", + "duckduckgo": "https://duckduckgo.com", + "google": "https://www.google.com", + "yahoo!": "https://search.yahoo.com", + "yandex": "https://yandex.com", + } + try: + response = requests.get( + "https://gs.statcounter.com/search-engine-market-share/desktop/worldwide" + ) + html = BeautifulSoup(response.content, "html.parser") + + table = html.find("table", class_="stats-snapshot") + tbody = table.find("tbody") + + search_engines = [] + market_shares = [] + # Loop through rows + for row in tbody.find_all("tr"): + # Find element (search engine name) + search_engine = row.find("th") + # Find element with class "count" (market share) + market_share = row.find("span", class_="count") + # Confirm both elements found + if search_engine and market_share: + # Extract text content from element + search_engines.append(search_engine.text.strip().lower()) + # Convert market share text to float + market_share_text = market_share.text.strip() + market_shares.append(float(market_share_text)) + + output = [] + for i in range(len(search_engines)): + output.append({"ref": SITES[search_engines[i]], "pct": market_shares[i]}) + # Convert to JSON and save + with open(os.path.join(ASSETS_DIR, "referers.json"), "w") as f: + json.dump(output, f) + + return True + + except Exception as e: + print(f"Error fetching referer data: {e}") + return False def extract_data(json_file_path: str) -> dict | list[dict]: @@ -29,16 +113,10 @@ def update_assets() -> bool: Returns True if successful else False """ try: - # Get absolute path for root directory - root_dir = os.path.abspath(os.path.dirname(__file__)) - # Path to assets and utils directories - assets_dir = os.path.join(root_dir, "assets") - utils_dir = os.path.join(root_dir, "src", "masquer", "utils") - # Get JSON file data - header_data = extract_data(os.path.join(assets_dir, "header.json")) - referer_data = extract_data(os.path.join(assets_dir, "referers.json")) - useragent_data = extract_data(os.path.join(assets_dir, "useragents.json")) + header_data = extract_data(os.path.join(ASSETS_DIR, "header.json")) + referer_data = extract_data(os.path.join(ASSETS_DIR, "referers.json")) + useragent_data = extract_data(os.path.join(ASSETS_DIR, "useragents.json")) referers = [obj["ref"] for obj in referer_data] referer_weights = [obj["pct"] for obj in referer_data] @@ -47,7 +125,7 @@ def update_assets() -> bool: useragent_weights = [obj["pct"] for obj in useragent_data] # Save JSON content to assets.py - with open(os.path.join(utils_dir, "assets.py"), "a") as f: + with open(os.path.join(UTILS_DIR, "assets.py"), "a") as f: f.seek(0) f.truncate() f.write("HEADER_DATA = " + str(header_data)) @@ -72,8 +150,14 @@ def update_assets() -> bool: return False -update = update_assets() -if not update: - print("Failed to update assets.py") -else: - print("Asset update successful") +if __name__ == "__main__": + ua = update_useragents() + rf = update_referers() + if ua and rf: + assets_updated = update_assets() + if not assets_updated: + print("Failed to update assets.py") + else: + print("Update successful") + else: + print("Failed to update JSON assets") diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..932e0c8 --- /dev/null +++ b/update.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +###################################################### +# update.sh +###################################################### + +# This version 23 June 2024 + +# Activates venv and runs ./update.py +# to get and save latest assets + +###################################################### + +# Activate virtual environment +source .venv/bin/activate + +# Check whether virtual environment is activated +if ! source .venv/bin/activate; then + echo "Error: couldn't activate venv." + echo "Create venv or update this script's venv path." + exit 1 # Exit with code 1 to indicate an error +fi + +# Run update.py +python3 update.py + +# Check the exit code of update.py +if [[ $? -eq 1 ]]; then + echo "Update failed — ensure BeautifulSoup4 is installed." +fi + +# Deactivate virtual environment +deactivate