Next Release: Minor 🎉 (#364)

.github/buildEverything.py → .github/buildFirmware.py

@@ -1,4 +1,4 @@
-#!/usr/lib/python3
+#!/usr/bin/env python3
 import re
 import os
 import shutil
@@ -9,38 +9,29 @@
 
 pioConfig = 'platformio.ini'
 
-# Configure editions
-editions = []
-fh = open(pioConfig, 'r')
-for line in fh.readlines():
-    if line.startswith('[env:'):
-        editions.append(line.strip()[5:-1])
-fh.close()
-
-# Find all languages
-languages = []
-langIncludePath = os.path.join('include', 'locales')
-for fName in os.listdir(langIncludePath):
-    if os.path.join(os.path.join(langIncludePath, fName)):
-        languages.append(fName[:-2])  # This drops off the .h
-
 # And compile!
 def compile_model(lang, edition):
     # Configs
     includeConfig = os.path.join('include', 'config.h')
 
     # Modify configs language
-    logging.info('Setting language to ' + lang + '...')
-    configIn = open(includeConfig, 'r')
-    configStr = configIn.read()
-    configIn.close()
-    configStr, hitCount = re.subn('("locales/).+(\.h")', r'\1' + lang + r'\2', configStr)
-    if hitCount != 1:
-        logging.error('Error on setting language!')
-        exit(1)
-    configOut = open(includeConfig, 'w')
-    configOut.write(configStr)
-    configOut.close()
+    if lang is not None:
+        logging.info('Setting language to ' + lang + '...')
+        configIn = open(includeConfig, 'r')
+        configStr = configIn.read()
+        configIn.close()
+        configStr, hitCount = re.subn('("locales/).+(\.h")', r'\1' + lang + r'\2', configStr)
+        if hitCount != 1:
+            logging.error('Error on setting language (failed to replace key in config)!')
+            exit(1)
+        configOut = open(includeConfig, 'w')
+        configOut.write(configStr)
+        configOut.close()
+    else:
+        import getWorkflowMatrix
+        lang = getWorkflowMatrix.get(getWorkflowMatrix.GettableInfo.LANG) # get the default language
+        assert len(lang) == 1, 'More than one default language found!'
+        lang = list(lang)[0] # get the first element of the set
 
     # Always clean after changing the language (just-in-case)
     logging.info('Cleanup...')
@@ -73,55 +64,46 @@ def doBuild(makeDebug):
         # Compile firmware
         logging.info('Compiling ' + filename + '...')
         try:
-            res = subprocess.run(['pio', 'run', '-e', edition], capture_output=True)
+            res = subprocess.run(['pio', 'run', '-e', edition])
         except KeyboardInterrupt:
             exit(3)
         if res.returncode != 0:
             logging.error('COMPILATION FAILED')
-            logging.error(res.stdout.decode())
-            logging.error(res.stderr.decode())
             exit(2)
         # "Export" firmware.bin
         shutil.copy(os.path.join('.pio', 'build', edition, 'firmware.bin'), os.path.join('.', filename))
-    if str(args["support_build"]) == "" or str(args["support_build"]) == "debug":
+    if 'debug' in args.support_build.lower():
         doBuild(True)
-    if str(args["support_build"]) == "" or str(args["support_build"]) == "release":
+    if 'release' in args.support_build.lower():
         doBuild(False)
 
 if __name__ == "__main__":
 
     ap = argparse.ArgumentParser()
-
-    ap.add_argument("-l", "--support-language", type=str, required=True, help="# model language to compile. (Enter 'all' to compile all language packs.)")
-    ap.add_argument("-m", "--support-model", type=str, required=True, help="# model type to compile. (Enter 'all' to compile all model packs.)")
-    ap.add_argument("-f", "--support-feature", type=str, required=False, default="", help="# feature to compile. (Enter a feature to compile.)")
-    ap.add_argument("-b", "--support-build", type=str, required=False, default="", help="# Build configuration to compile. (Enter a Debug/Release to compile.)")
-    args = vars(ap.parse_args())
+    ap.add_argument("-m", "--support-model", type=str, required=True, help="model to compile")
+    ap.add_argument("-l", "--support-language", type=str, required=False, help="language to compile")
+    ap.add_argument("-f", "--support-feature", type=str, required=False, default="", help="override feature(s) to compile - provide multiple separated by '|'")
+    ap.add_argument("-b", "--support-build", type=str, required=False, default="debug,release", help="compile release or debug")
+    args = ap.parse_args()
 
-    if str(args["support_feature"]) != "":
-        logging.info('Setting flag ' + str(args["support_feature"]) + '...')
+    if len(args.support_feature) > 0:
+        overwriteFeatures = set(args.support_feature.split('|'))
+        logging.info(f'Setting flags to {overwriteFeatures}...')
         # Setup build flag (using the config file via replacing, as platformio does not allow setting the property using Python)
         configIn = open(pioConfig, 'r')
         configStr = configIn.read()
         configIn.close()
-        configStr, hitCount = re.subn('build_flags =','build_flags =\n    -D '+str(args["support_feature"]),configStr)
+        # Build the feature-string
+        featureStr = ''
+        for feature in overwriteFeatures:
+            featureStr += f'    -D {feature}\n'
+        # This will replace all defines from the build_flags line - except the first one (which is the platform header)
+        configStr, hitCount = re.subn('build_flags =(\s+-.+)(\s+-.+)*','build_flags =' + r'\1' + '\n' + featureStr, configStr)
         if hitCount == 0:
             logging.error('Error on setting build flag!')
             exit(5)
         configOut = open(pioConfig, 'w')
         configOut.write(configStr)
         configOut.close()
 
-    #if you want all-language packs
-    if args["support_language"] == "all" and args["support_model"] == "all":    
-        for lang in languages:
-            for edition in editions:
-                compile_model(lang,edition)
-    elif args["support_language"] == "all" and args["support_model"] != "all":
-        for lang in languages:
-            compile_model(lang, args["support_model"])
-    elif args["support_language"] != "all" and args["support_model"] == "all":
-        for edition in editions:
-            compile_model(args["support_language"], edition)
-    else :
-        compile_model(args["support_language"], args["support_model"])
+    compile_model(args.support_language, args.support_model)

.github/getWorkflowMatrix.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import os
+import re
+import json
+import argparse
+
+from enum import Enum
+
+class GettableInfo(str, Enum):
+    LANG = 'default_language'
+    MODEL = 'default_model'
+    FLAGS = 'all_flags'
+    MODELS = 'all_models'
+    LANGS = 'all_languages'
+
+def get(info: GettableInfo):
+    returnArray = set()
+    if info == GettableInfo.LANG:
+        langBasePath = 'include/locales/'
+        defaultFile = langBasePath + 'en-US.h'
+        if not os.path.exists(defaultFile):
+            # Well, instead just take the first header-file in the folder
+            for file in os.listdir(langBasePath):
+                if file.endswith('.h'):
+                    defaultFile = langBasePath + file
+                    break
+        # Strip the path and the extension
+        returnArray.add(os.path.splitext(os.path.basename(defaultFile))[0])
+    elif  info == GettableInfo.MODEL:
+        with open('platformio.ini') as f:
+            for line in f:
+                if line.startswith('default_envs'):
+                    returnArray.add(line.split('=')[1].strip())
+                    break
+    elif info == GettableInfo.FLAGS:
+        with open('docs/firmware/osw_os.md') as f:
+            for line in f:
+                match = re.match(r'^`(.+)` \| .+ \| (.+)$', line)
+                if match is None:
+                    continue
+                feature = match.group(1).strip()
+                featureRequirements = match.group(2).strip()
+                required = [feature] + [req.strip('`,').strip() for req in featureRequirements.split(' ')]
+                try:
+                    required.remove('-') # remove the "no flags" flag
+                except ValueError:
+                    pass # no "no flags" flag
+                for flag in required:
+                    if re.match(r'^[\-|A-Z0-9_]+$', flag) is None:
+                        # We drop any requirement that is not a proper flag
+                        break
+                else:
+                    returnArray.add(str('|'.join(required)))
+    elif info == GettableInfo.MODELS:
+        with open('platformio.ini') as f:
+            for line in f:
+                match = re.match(r'^\[env:(.+)\]$', line)
+                if match is None: continue
+                env = match.group(1)
+                if env.startswith('GPS'):
+                    # GPS is NOT build anymore by default
+                    continue
+                returnArray.add(env)
+    elif info == GettableInfo.LANGS:
+        langBasePath = 'include/locales'
+        for file in os.listdir(langBasePath):
+            if file.endswith('.h'):
+                returnArray.add(os.path.splitext(file)[0])
+    else:
+        raise NotImplementedError('Unknown info: ' + info)
+    return returnArray
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument('info', type=GettableInfo)
+    args = ap.parse_args()
+
+    print(json.dumps(list(get(args.info))))

.github/workflows/test-FEATURE.yml

@@ -30,13 +30,13 @@ jobs:
           submodules: recursive
       - id: get-flag
         run: |
-          echo "feature=$(./.github/setMatrix.sh --get-flag)" >> $GITHUB_OUTPUT
+          echo "feature=$(python3 .github/getWorkflowMatrix.py all_flags)" >> $GITHUB_OUTPUT
       - id: default_mod
         run: |
-          echo "default_model=$(./.github/setMatrix.sh --default-mod)" >> $GITHUB_OUTPUT
+          echo "default_model=$(python3 .github/getWorkflowMatrix.py default_model)" >> $GITHUB_OUTPUT
       - id: default_lang
         run: |
-          echo "default_language=$(./.github/setMatrix.sh --default-lang)" >> $GITHUB_OUTPUT
+          echo "default_language=$(python3 .github/getWorkflowMatrix.py default_language)" >> $GITHUB_OUTPUT
     outputs:
       feature: ${{ steps.get-flag.outputs.feature }}
       default_model: ${{ steps.default_mod.outputs.default_model }}
@@ -77,4 +77,4 @@ jobs:
     - name: Rename config
       run: mv include/config.h.example include/config.h
     - name: Compile language ${{ matrix.language }} model ${{ matrix.model }} feature ${{ matrix.feature }}
-      run: python3 .github/buildEverything.py -l ${{ matrix.language }} -m ${{ matrix.model }} -f ${{ matrix.feature }} -b debug
+      run: python3 .github/buildFirmware.py -l "${{ matrix.language }}" -m "${{ matrix.model }}" -f "${{ matrix.feature }}" -b debug

.github/workflows/test-OS.yml

@@ -24,13 +24,13 @@ jobs:
       - id: get_lang
         run: |
           if [ ${{ steps.filter.outputs.i8n }} == "true" ]; then
-            echo "get-languages=$(./.github/setMatrix.sh --get-languages)" >> $GITHUB_OUTPUT
+            echo "get-languages=$(python3 .github/getWorkflowMatrix.py all_languages)" >> $GITHUB_OUTPUT
           else
-            echo "get-languages=$(./.github/setMatrix.sh --default-lang)" >> $GITHUB_OUTPUT
+            echo "get-languages=$(python3 .github/getWorkflowMatrix.py default_language)" >> $GITHUB_OUTPUT
           fi
       - id: get_mod
         run: |
-          echo "get-models=$(./.github/setMatrix.sh --get-models)" >> $GITHUB_OUTPUT
+          echo "get-models=$(python3 .github/getWorkflowMatrix.py all_models)" >> $GITHUB_OUTPUT
     outputs:
       get-models: ${{ steps.get_mod.outputs.get-models }}
       get-languages: ${{ steps.get_lang.outputs.get-languages }}
@@ -73,4 +73,4 @@ jobs:
         mv include/config.h.example include/config.h
 
     - name: Compile language ${{ matrix.language }} model ${{ matrix.model }}
-      run: python3 .github/buildEverything.py -l ${{ matrix.language }} -m ${{ matrix.model }} -b debug
+      run: python3 .github/buildFirmware.py -l "${{ matrix.language }}" -m "${{ matrix.model }}" -b debug

.github/workflows/test-OSW.yml

@@ -38,12 +38,12 @@ jobs:
       - id: get-languages
         run: |
           if [ "${{ steps.filter.outputs.i8n }}" == "true" -o "${{ github.head_ref == 'master' }}" == "true" -o "${{ github.base_ref == 'master' }}" == "true" ]; then
-            echo "languages_matrix=$(./.github/setMatrix.sh --get-languages)" >> $GITHUB_OUTPUT
+            echo "languages_matrix=$(python3 .github/getWorkflowMatrix.py all_languages)" >> $GITHUB_OUTPUT
           else
-            echo "languages_matrix=$(./.github/setMatrix.sh --default-lang)" >> $GITHUB_OUTPUT
+            echo "languages_matrix=$(python3 .github/getWorkflowMatrix.py default_language)" >> $GITHUB_OUTPUT
           fi
       - id: get-models
-        run: echo "models_matrix=$(./.github/setMatrix.sh --get-models)" >> $GITHUB_OUTPUT
+        run: echo "models_matrix=$(python3 .github/getWorkflowMatrix.py all_models)" >> $GITHUB_OUTPUT
     outputs:
       languages_matrix: ${{ steps.get-languages.outputs.languages_matrix }}
       models_matrix: ${{ steps.get-models.outputs.models_matrix }}
@@ -83,7 +83,7 @@ jobs:
     - name: Rename config
       run: mv include/config.h.example include/config.h
     - name: Compile language ${{ matrix.language }} model ${{ matrix.model }}
-      run: python3 .github/buildEverything.py -l ${{ matrix.language }} -m ${{ matrix.model }} -b ${{ matrix.build-configuration }}
+      run: python3 .github/buildFirmware.py -l "${{ matrix.language }}" -m "${{ matrix.model }}" -b "${{ matrix.build-configuration }}"
     - name: Upload firmware artifacts
       uses: actions/upload-artifact@v3
       with: