Merge pull request #845 from AlmaLinux/new_gh_ci

- GH CI is improved and fixed

.github/workflows/ci.yml

@@ -39,15 +39,19 @@ jobs:
 
       - name: Check YAML configs
         run: |
-          mirror_regexp="mirrors.d/.*.(yml|yaml)$"
-          if ! echo "${{ steps.files.outputs.added_modified }}" | grep -E "${mirror_regexp}" &> /dev/null; then
+          mirror_regexp="mirrors.d/.*.(yml)$"
+          modified_files="${{ steps.files.outputs.added_modified }} ${{steps.files.outputs.renamed}}"
+          if ! echo "${modified_files}" | grep -E "${mirror_regexp}" &> /dev/null; then
             # we check only the main config if PR doesn't contain any new or modified mirror config
             python3.9 gh_ci/config_checker.py -sc config.yml
           else
             changed_files=""
-            for changed_file in ${{ steps.files.outputs.added_modified }}; do
+            for changed_file in ${modified_files}; do
               if echo "${changed_file}" | grep -E "${mirror_regexp}" &> /dev/null; then
                 changed_files="${changed_file} ${changed_files}"
+              elif echo "${changed_file}" | grep "mirrors.d" &> /dev/null; then
+                echo "You add a config with wrong extension. Should be *.yml"
+                exit 1
               fi
             done
             python3.9 gh_ci/config_checker.py -sc config.yml -mc ${changed_files}

gh_ci/config_checker.py

@@ -85,14 +85,15 @@ async def are_mirrors_available(
             headers={"Connection": "close"}
     ) as http_session:
         for mirror in mirrors:
-            mirror_name, is_available = await mirror_available(
+            is_available = await mirror_available(
                 mirror_info=mirror,
                 http_session=http_session,
                 logger=logger,
                 main_config=main_config,
             )
-            if not is_available:
-                ret_code = 1
+            # True is 1, False is 0, so
+            # we get 1 if a mirror is not available
+            ret_code += int(not is_available)
     return ret_code
 
 
@@ -107,12 +108,12 @@ def do_mirrors_have_valid_geo_data(
     ui_url = 'https://nominatim.openstreetmap.org/ui/details.html'
     for mirror in mirrors:
         if any(getattr(mirror.geolocation, geo_attr) is None for geo_attr in (
-            'city', 'state', 'country'
+            'city', 'state_province', 'country'
         )):
             continue
         params = {
             'city': mirror.geolocation.city,
-            'state': mirror.geolocation.state,
+            'state': mirror.geolocation.state_province,
             'country': mirror.geolocation.country,
             'format': 'json',
         }
@@ -169,6 +170,13 @@ def main(args):
             err,
         )
         exit(1)
+    main_config, err_msg = process_main_config(yaml_data=service_config_data)
+    if err_msg:
+        logger.error(
+            'Main config of the mirror service is invalid because "%s"',
+            err_msg,
+        )
+        exit(1)
     exit_code = 0
     for mirror_config in args.mirror_configs:
         mirror_config_data = mirror_config['config_data']
@@ -200,6 +208,7 @@ def main(args):
         process_mirror_config(
             yaml_data=mirror_config['config_data'],
             logger=logger,
+            main_config=main_config,
         )
         for mirror_config in args.mirror_configs
     ]

gh_ci/req.txt

@@ -1,6 +1,7 @@
-pyyaml==6.0
-jsonschema==4.2.1
-aiohttp==3.8.1
-requests==2.26.0
+pyyaml==6.0.1
+jsonschema==4.18.4
+aiohttp==3.8.5
+aiohttp-retry==2.8.3
+requests==2.31.0
 dataclasses==0.6
-syncer==1.3.0
+syncer==2.0.3

mirrors.d/mirror.dhakacom.com.yaml → mirrors.d/mirror.dhakacom.com.yml

@@ -1,8 +1,6 @@
 name: mirror.amberit.com.bd
 address:
   http: http://mirror.dhakacom.com/almalinux/
-  https:
-  rsync:
 geolocation:
   country: BD
   state_province: Dhaka