From 32ff5da481b163738f358b6f5b2a0809e6119590 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 5 Dec 2022 17:14:04 +0200 Subject: [PATCH 1/8] - Update git submodule --- gh_ci/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh_ci/yaml_snippets b/gh_ci/yaml_snippets index eb0a9739..2699710f 160000 --- a/gh_ci/yaml_snippets +++ b/gh_ci/yaml_snippets @@ -1 +1 @@ -Subproject commit eb0a9739dc0789ab538d7796c1b514577d9acd93 +Subproject commit 2699710f27151ab1f6b89a0b8c295ff54aa4a352 From a3971aa7fe0806a725d60f3fc8ace0989ff9fb2d Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 5 Dec 2022 17:15:13 +0200 Subject: [PATCH 2/8] - Test mirror's config --- mirrors.d/test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 mirrors.d/test.yml diff --git a/mirrors.d/test.yml b/mirrors.d/test.yml new file mode 100644 index 00000000..ab863a6e --- /dev/null +++ b/mirrors.d/test.yml @@ -0,0 +1,14 @@ +--- +name: mirror.0-1.cloud +address: + http: http://mirror.0-1.cloud/almalinux + https: https://mirror.0-1.cloud/almalinux +geolocation: + country: IR + state_province: Tehran + city: Tehran +update_frequency: 6h +sponsor: Sefroyek Pardaz Engineering Co. +sponsor_url: https://0-1.ir/ +email: info@0-1.ir +... From 601fe3d1c6e9e211a1b6bd51751e541329b470fe Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 5 Dec 2022 17:27:15 +0200 Subject: [PATCH 3/8] - Pass the service config for checking a mirror's config --- gh_ci/config_checker.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gh_ci/config_checker.py b/gh_ci/config_checker.py index 8b1106d6..2b41f93f 100644 --- a/gh_ci/config_checker.py +++ b/gh_ci/config_checker.py @@ -169,6 +169,13 @@ def main(args): err, ) exit(1) + main_config, err_msg = process_main_config(yaml_data=service_config_data) + if err_msg: + logger.error( + 'Main config of the mirror service is invalid because "%s"', + err_msg, + ) + exit(1) exit_code = 0 for mirror_config in args.mirror_configs: mirror_config_data = mirror_config['config_data'] @@ -200,6 +207,7 @@ def main(args): process_mirror_config( yaml_data=mirror_config['config_data'], logger=logger, + main_config=main_config, ) for mirror_config in args.mirror_configs ] From 3fbdfef7848b09982792c399493b99c2053bf5bb Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 5 Dec 2022 17:38:39 +0200 Subject: [PATCH 4/8] - Update git submodule --- gh_ci/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh_ci/yaml_snippets b/gh_ci/yaml_snippets index 2699710f..8d21de87 160000 --- a/gh_ci/yaml_snippets +++ b/gh_ci/yaml_snippets @@ -1 +1 @@ -Subproject commit 2699710f27151ab1f6b89a0b8c295ff54aa4a352 +Subproject commit 8d21de8745c8a85853b4721587774d027d1478d8 From 3aa17644fea41d689835f47b0671ee59b361b919 Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Thu, 27 Jul 2023 16:55:59 +0300 Subject: [PATCH 5/8] - Python libraries in req.txt are updated to the latest versions - New version of config checker - Lib `aiohttp-retry` is added to req.txt - Ret code is fixed - Check extension of a mirror's config - Test config is removed - All *.yaml files are renamed to *.yml - Removed empty fields from the mirror's config --- .github/workflows/ci.yml | 10 +++++++--- gh_ci/config_checker.py | 11 ++++++----- gh_ci/req.txt | 11 ++++++----- ....virtono.com.yaml => almalinux.virtono.com.yml} | 0 ...r.dhakacom.com.yaml => mirror.dhakacom.com.yml} | 2 -- mirrors.d/mirror.hoster.kz.yaml | 13 ------------- ...alcompute.io.yaml => mirror.realcompute.io.yml} | 0 mirrors.d/test.yml | 14 -------------- 8 files changed, 19 insertions(+), 42 deletions(-) rename mirrors.d/{almalinux.virtono.com.yaml => almalinux.virtono.com.yml} (100%) rename mirrors.d/{mirror.dhakacom.com.yaml => mirror.dhakacom.com.yml} (93%) delete mode 100644 mirrors.d/mirror.hoster.kz.yaml rename mirrors.d/{mirror.realcompute.io.yaml => mirror.realcompute.io.yml} (100%) delete mode 100644 mirrors.d/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72a180d1..3d5bb9fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,15 +39,19 @@ jobs: - name: Check YAML configs run: | - mirror_regexp="mirrors.d/.*.(yml|yaml)$" - if ! echo "${{ steps.files.outputs.added_modified }}" | grep -E "${mirror_regexp}" &> /dev/null; then + mirror_regexp="mirrors.d/.*.(yml)$" + modified_files="${{ steps.files.outputs.added_modified }} ${{steps.files.outputs.renamed}}" + if ! echo "${modified_files}" | grep -E "${mirror_regexp}" &> /dev/null; then # we check only the main config if PR doesn't contain any new or modified mirror config python3.9 gh_ci/config_checker.py -sc config.yml else changed_files="" - for changed_file in ${{ steps.files.outputs.added_modified }}; do + for changed_file in ${modified_files}; do if echo "${changed_file}" | grep -E "${mirror_regexp}" &> /dev/null; then changed_files="${changed_file} ${changed_files}" + elif echo "${changed_file}" | grep "mirrors.d" &> /dev/null; then + echo "You add a config with wrong extension. Should be *.yml" + exit 1 fi done python3.9 gh_ci/config_checker.py -sc config.yml -mc ${changed_files} diff --git a/gh_ci/config_checker.py b/gh_ci/config_checker.py index 2b41f93f..47e33566 100644 --- a/gh_ci/config_checker.py +++ b/gh_ci/config_checker.py @@ -85,14 +85,15 @@ async def are_mirrors_available( headers={"Connection": "close"} ) as http_session: for mirror in mirrors: - mirror_name, is_available = await mirror_available( + is_available = await mirror_available( mirror_info=mirror, http_session=http_session, logger=logger, main_config=main_config, ) - if not is_available: - ret_code = 1 + # True is 1, False is 0, so + # we get 1 if a mirror is not available + ret_code += int(not is_available) return ret_code @@ -107,12 +108,12 @@ def do_mirrors_have_valid_geo_data( ui_url = 'https://nominatim.openstreetmap.org/ui/details.html' for mirror in mirrors: if any(getattr(mirror.geolocation, geo_attr) is None for geo_attr in ( - 'city', 'state', 'country' + 'city', 'state_province', 'country' )): continue params = { 'city': mirror.geolocation.city, - 'state': mirror.geolocation.state, + 'state': mirror.geolocation.state_province, 'country': mirror.geolocation.country, 'format': 'json', } diff --git a/gh_ci/req.txt b/gh_ci/req.txt index 6e0384e1..6e550137 100644 --- a/gh_ci/req.txt +++ b/gh_ci/req.txt @@ -1,6 +1,7 @@ -pyyaml==6.0 -jsonschema==4.2.1 -aiohttp==3.8.1 -requests==2.26.0 +pyyaml==6.0.1 +jsonschema==4.18.4 +aiohttp==3.8.5 +aiohttp-retry==2.8.3 +requests==2.31.0 dataclasses==0.6 -syncer==1.3.0 +syncer==2.0.3 diff --git a/mirrors.d/almalinux.virtono.com.yaml b/mirrors.d/almalinux.virtono.com.yml similarity index 100% rename from mirrors.d/almalinux.virtono.com.yaml rename to mirrors.d/almalinux.virtono.com.yml diff --git a/mirrors.d/mirror.dhakacom.com.yaml b/mirrors.d/mirror.dhakacom.com.yml similarity index 93% rename from mirrors.d/mirror.dhakacom.com.yaml rename to mirrors.d/mirror.dhakacom.com.yml index f3b8633f..d6e70ac0 100644 --- a/mirrors.d/mirror.dhakacom.com.yaml +++ b/mirrors.d/mirror.dhakacom.com.yml @@ -1,8 +1,6 @@ name: mirror.amberit.com.bd address: http: http://mirror.dhakacom.com/almalinux/ - https: - rsync: geolocation: country: BD state_province: Dhaka diff --git a/mirrors.d/mirror.hoster.kz.yaml b/mirrors.d/mirror.hoster.kz.yaml deleted file mode 100644 index 35c639e6..00000000 --- a/mirrors.d/mirror.hoster.kz.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: mirror.hoster.kz -address: - http: http://mirror.hoster.kz/almalinux/ - https: https://mirror.hoster.kz/almalinux/ - rsync: rsync://mirror.hoster.kz/almalinux/ -geolocation: - country: KZ - state_province: - city: Almaty -update_frequency: 3h -sponsor: Hoster.KZ -sponsor_url: https://hoster.kz/ -email: support@hoster.kz diff --git a/mirrors.d/mirror.realcompute.io.yaml b/mirrors.d/mirror.realcompute.io.yml similarity index 100% rename from mirrors.d/mirror.realcompute.io.yaml rename to mirrors.d/mirror.realcompute.io.yml diff --git a/mirrors.d/test.yml b/mirrors.d/test.yml deleted file mode 100644 index ab863a6e..00000000 --- a/mirrors.d/test.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: mirror.0-1.cloud -address: - http: http://mirror.0-1.cloud/almalinux - https: https://mirror.0-1.cloud/almalinux -geolocation: - country: IR - state_province: Tehran - city: Tehran -update_frequency: 6h -sponsor: Sefroyek Pardaz Engineering Co. -sponsor_url: https://0-1.ir/ -email: info@0-1.ir -... From 55729e50448b6cac81c5f6eed6a376ace5322b71 Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Thu, 27 Jul 2023 16:57:11 +0300 Subject: [PATCH 6/8] - The Git submodule is updated --- gh_ci/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh_ci/yaml_snippets b/gh_ci/yaml_snippets index 8d21de87..2002eab3 160000 --- a/gh_ci/yaml_snippets +++ b/gh_ci/yaml_snippets @@ -1 +1 @@ -Subproject commit 8d21de8745c8a85853b4721587774d027d1478d8 +Subproject commit 2002eab31e3cc54cba2e2a9affe734e4a347803e From 78f5ae96b36b23fbcb21f3133d545a2cfe164ba3 Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Thu, 27 Jul 2023 17:08:20 +0300 Subject: [PATCH 7/8] - The Git submodule is updated --- gh_ci/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh_ci/yaml_snippets b/gh_ci/yaml_snippets index 2002eab3..32952430 160000 --- a/gh_ci/yaml_snippets +++ b/gh_ci/yaml_snippets @@ -1 +1 @@ -Subproject commit 2002eab31e3cc54cba2e2a9affe734e4a347803e +Subproject commit 32952430a71ba2cdc7c3bc7dd16b38dd9b0df021 From d3063a9982b343e1f9b3bd6fe5bbc514d0bbf3d7 Mon Sep 17 00:00:00 2001 From: Stepan Oksanichenko Date: Thu, 27 Jul 2023 17:11:33 +0300 Subject: [PATCH 8/8] - The Git submodule is updated --- gh_ci/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh_ci/yaml_snippets b/gh_ci/yaml_snippets index 32952430..8d81a726 160000 --- a/gh_ci/yaml_snippets +++ b/gh_ci/yaml_snippets @@ -1 +1 @@ -Subproject commit 32952430a71ba2cdc7c3bc7dd16b38dd9b0df021 +Subproject commit 8d81a726e6969ce117e3fe0b330a578c46af5885