Merge branch 'master' into Add_newHw_pixFlamingof767

.github/problem-matchers/autotestfail.json

@@ -0,0 +1,17 @@
+{
+    "__comment": "by buzz try to match common autotest warnings and errors that arent caught by gcc.json or python.json",
+    "problemMatcher": [
+        {
+            "owner": "autotest-fail-matcher",
+            "severity": "error",
+            "pattern": [
+                {
+                    "regexp": "^(.*)(TIMEOUT|Build failed|FAILED STEP):(.*)$",
+                    "column": 1,
+                    "code": 2,
+                    "message": 3
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/autotestwarn.json

@@ -0,0 +1,17 @@
+{
+    "__comment": "by buzz try to match common autotest warnings and errors that arent caught by gcc.json or python.json",
+    "problemMatcher": [
+        {
+            "owner": "autotest-warn-matcher",
+            "severity": "warning",
+            "pattern": [
+                {
+                    "regexp": "^(.*)(WARN|WARNING):(.*)$",
+                    "column": 1,
+                    "code": 2,
+                    "message": 3
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/gcc.json

@@ -0,0 +1,18 @@
+{
+    "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule",
+    "problemMatcher": [
+        {
+            "owner": "gcc-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5
+                }
+            ]
+        }
+    ]
+}

.github/problem-matchers/python.json

@@ -0,0 +1,22 @@
+{
+    "__comment": "inspired by https://github.com/microsoft/vscode-python/issues/3828#issuecomment-575439587",
+    "problemMatcher": [
+        {
+            "owner": "python-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^.*File \\\"([^\\\"]|.*)\\\", line (\\d+).*",
+                    "file": 1,
+                    "line": 2
+                },
+                {
+                    "regexp": "^.*raise.*$"
+                },
+                {
+                    "regexp": "^\\s*(.*)\\s*$",
+                    "message": 1
+                }
+            ]
+        }
+    ]
+}

.github/workflows/esp32_build.yml

@@ -151,29 +151,59 @@ jobs:
       matrix:
         config: [
             esp32buzz,
+            esp32s3empty,
         ]
         gcc: [10]
 
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
+
+      - name: Register gcc problem matcher
+        run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+
+      - name: Register python problem matcher
+        run: echo "::add-matcher::.github/problem-matchers/python.json"
+
+      - name: Register autotest warn matcher
+        run: echo "::add-matcher::.github/problem-matchers/autotestwarn.json"
+
+      - name: Register autotest fail matcher
+        run: echo "::add-matcher::.github/problem-matchers/autotestfail.json"
+
       - name: Install Prerequisites
         shell: bash
         run: |
-          sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
+          sudo apt-get update
+          sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-venv python3-setuptools python3-serial python3-gevent python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
           sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+          update-alternatives --query python
+          python --version
+          pip3 install gevent
 
-          rm -rf /usr/local/bin/cmake
+          # we actualy want 3.11 .. but the above only gave us 3.10, not ok with esp32 builds.
+          sudo add-apt-repository ppa:deadsnakes/ppa
+          sudo apt-get update
+          sudo apt-get install python3.11 python3.11-venv python3.11-distutils  -y
+          sudo apt-get install python3 python3-pip python3-venv python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools
+          update-alternatives --query python
+          pip3 install gevent
+          python --version
+          python3.11 --version
+          which python3.11
+          sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.11 11
+          update-alternatives --query python
 
-          sudo apt remove --purge --auto-remove cmake
-          sudo apt update && \
-          sudo apt install -y software-properties-common lsb-release && \
-          sudo apt clean all
+          rm -rf /usr/local/bin/cmake
+          sudo apt-get remove --purge --auto-remove cmake
+          sudo apt-get update && \
+          sudo apt-get install -y software-properties-common lsb-release && \
+          sudo apt-get clean all
           wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
           sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main"
-          sudo apt update
-          sudo apt install cmake
+          sudo apt-get update
+          sudo apt-get install cmake
 
 
           git submodule update --init --recursive --depth=1
@@ -183,7 +213,7 @@ jobs:
 
           cd modules/esp_idf
           echo "Installing ESP-IDF tools...."
-          ./install.sh 2>&1 > /dev/null
+          ./install.sh esp32,esp32s3 
           cd ../..
           
           
@@ -195,32 +225,48 @@ jobs:
           source ~/.bash_profile
           PATH="/github/home/.local/bin:$PATH"
           echo $PATH
-          
+          echo "### Configure ${{matrix.config}} :rocket:" >> $GITHUB_STEP_SUMMARY
           cd modules/esp_idf
           ./install.sh
           source ./export.sh
           cd ../..
           python -m pip install --progress-bar off future lxml pymavlink MAVProxy pexpect flake8 geocoder empy==3.3.4 dronecan
           which cmake
-          ./waf configure --board ${{matrix.config}}
-          ./waf plane
-          cp build/esp32buzz/esp-idf_build/ardupilot.bin ./ArduPlane.bin
-          cp build/esp32buzz/esp-idf_build/ardupilot.elf ./ArduPlane.elf
-          ./waf copter
-          cp build/esp32buzz/esp-idf_build/ardupilot.bin ./ArduCopter.bin
-          cp build/esp32buzz/esp-idf_build/ardupilot.elf ./ArduCopter.elf
+          ./waf configure --board ${{matrix.config}} 
+          echo './waf configure --board ${{matrix.config}}' >> $GITHUB_STEP_SUMMARY  
+          echo "### Build Plane ${{matrix.config}} :rocket:" >> $GITHUB_STEP_SUMMARY
+          echo './waf plane' >> $GITHUB_STEP_SUMMARY  
+          ./waf plane | tee >( grep -vE 'Compiling|Generat|includes.list|Parsing|Validation|Building|Linking|Detecting|linker|\*\*\*\*\*\*\*|Partition|SubType|Checking|esp-idf' --color=never --line-buffered > summary.log)  
+          cat summary.log >> $GITHUB_STEP_SUMMARY 
+
+          cp build/${{matrix.config}}/esp-idf_build/ardupilot.bin ./ArduPlane.${{matrix.config}}.bin
+          cp build/${{matrix.config}}/esp-idf_build/ardupilot.elf ./ArduPlane.${{matrix.config}}.elf
+          echo "### Build Copter ${{matrix.config}} :rocket:" >> $GITHUB_STEP_SUMMARY
+          echo './waf copter' >> $GITHUB_STEP_SUMMARY  
+          ./waf copter | tee >( grep -vE 'Compiling|Generat|includes.list|Parsing|Validation|Building|Linking|Detecting|linker|\*\*\*\*\*\*\*|Partition|SubType|Checking|esp-idf' --color=never --line-buffered > summary2.log)  
+
+          cat summary2.log >> $GITHUB_STEP_SUMMARY  
+
+          cp build/${{matrix.config}}/esp-idf_build/ardupilot.bin ./ArduCopter.${{matrix.config}}.bin
+          cp build/${{matrix.config}}/esp-idf_build/ardupilot.elf ./ArduCopter.${{matrix.config}}.elf
+
+          cp build/${{matrix.config}}/esp-idf_build/bootloader/bootloader.bin ./bootloader.${{matrix.config}}.bin
+          cp build/${{matrix.config}}/esp-idf_build/partition_table/partition-table.bin ./partition-table.${{matrix.config}}.bin
+
+          echo "### Assets avail in artifact:" >> $GITHUB_STEP_SUMMARY
+          ls bootloader* partition* Ardu*.elf Ardu*.bin >> $GITHUB_STEP_SUMMARY
 
       - name: Archive artifacts
         uses: actions/upload-artifact@v3
         with:
            name: esp32-binaries -${{matrix.config}}
            path: |
-              /home/runner/work/ardupilot/ardupilot/ArduPlane.bin
-              /home/runner/work/ardupilot/ardupilot/ArduPlane.elf
-              /home/runner/work/ardupilot/ardupilot/ArduCopter.bin
-              /home/runner/work/ardupilot/ardupilot/ArduCopter.elf
-              /home/runner/work/ardupilot/ardupilot/build/esp32buzz/esp-idf_build/bootloader/bootloader.bin
-              /home/runner/work/ardupilot/ardupilot/build/esp32buzz/esp-idf_build/partition_table/partition-table.bin
+              /home/runner/work/ardupilot/ardupilot/ArduPlane.${{matrix.config}}.bin
+              /home/runner/work/ardupilot/ardupilot/ArduPlane.${{matrix.config}}.elf
+              /home/runner/work/ardupilot/ardupilot/ArduCopter.${{matrix.config}}.bin
+              /home/runner/work/ardupilot/ardupilot/ArduCopter.${{matrix.config}}.elf
+              /home/runner/work/ardupilot/ardupilot/bootloader.${{matrix.config}}.bin
+              /home/runner/work/ardupilot/ardupilot/partition-table.${{matrix.config}}.bin
 
 
            retention-days: 14

.github/workflows/test_chibios.yml

@@ -155,7 +155,10 @@ jobs:
             CubeRedPrimary-bootloader,
             configure-all,
             build-options-defaults-test,
-            signing
+            signing,
+            CubeOrange-PPP,
+            CubeOrange-SOHW,
+            Pixhawk6X-PPPGW,
         ]
         toolchain: [
             chibios,

.github/workflows/test_coverage.yml

@@ -72,7 +72,7 @@ jobs:
             Tools/scripts/run_coverage.py -f
           else
             Tools/scripts/run_coverage.py -i
-            ./waf configure --board sitl_periph_gps --debug --coverage
+            ./waf configure --board sitl_periph_universal --debug --coverage
             ./waf build --target bin/AP_Periph
             Tools/scripts/run_coverage.py -i
             Tools/autotest/autotest.py test.CAN --debug --coverage

.github/workflows/test_sitl_blimp.yml

@@ -200,7 +200,7 @@ jobs:
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug
           ./waf build --target bin/blimp
           ccache -s
           ccache -z

.github/workflows/test_sitl_copter.yml

@@ -198,7 +198,7 @@ jobs:
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug
           ./waf build --target bin/arducopter
           ccache -s
           ccache -z

.github/workflows/test_sitl_periph.yml

@@ -185,14 +185,15 @@ jobs:
       - name: run dronecan dsdlc generator test
         run: |
           PATH="/github/home/.local/bin:$PATH"
+          python -m pip install --upgrade dronecan
           python modules/DroneCAN/dronecan_dsdlc/dronecan_dsdlc.py -O dsdlc_generated modules/DroneCAN/DSDL/uavcan modules/DroneCAN/DSDL/dronecan modules/DroneCAN/DSDL/com --run-test
 
-      - name: build sitl_periph_gps
+      - name: build sitl_periph_universal
         shell: bash
         run: |
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl_periph_gps
+          ./waf configure --board sitl_periph_universal --debug
           ./waf build --target bin/AP_Periph
           ccache -s
           ccache -z

.github/workflows/test_sitl_plane.yml

@@ -199,7 +199,7 @@ jobs:
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug
           ./waf build --target bin/arduplane
           ccache -s
           ccache -z

.github/workflows/test_sitl_rover.yml

@@ -192,13 +192,15 @@ jobs:
       - name: build rover ${{ matrix.toolchain }}
         shell: bash
         run: |
+          sudo apt-get update || true
+          sudo apt-get -y install ppp || true
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
           if [[ ${{ matrix.toolchain }} == "clang" ]]; then
             export CC=clang
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug --enable-math-check-indexes
           ./waf build --target bin/ardurover
           ccache -s
           ccache -z

.github/workflows/test_sitl_sub.yml

@@ -201,7 +201,7 @@ jobs:
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug
           ./waf build --target bin/ardusub
           ccache -s
           ccache -z

.github/workflows/test_sitl_tracker.yml

@@ -201,7 +201,7 @@ jobs:
             export CXX=clang++
           fi
           PATH="/github/home/.local/bin:$PATH"
-          ./waf configure --board sitl
+          ./waf configure --board sitl --debug
           ./waf build --target bin/antennatracker
           ccache -s
           ccache -z

.github/workflows/test_unit_tests.yml

@@ -148,6 +148,22 @@ jobs:
           PATH="/github/home/.local/bin:$PATH"
           Tools/scripts/build_ci.sh
 
+      - name: Unittest extract_param_defaults
+        env:
+          PYTHONPATH: "${{ github.workspace }}/Tools/scripts:${{ github.workspace }}/modules/mavlink/pymavlink"
+        if: matrix.config == 'unit-tests'
+        run: |
+          Tools/autotest/unittest/extract_param_defaults_unittest.py
+
+      - name: Unittest annotate_params
+        env:
+          PYTHONPATH: "${{ github.workspace }}/Tools/scripts"
+        if: matrix.config == 'unit-tests'
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests mock
+          Tools/autotest/unittest/annotate_params_unittest.py
+
       - name: Archive buildlog artifacts
         uses: actions/upload-artifact@v3
         if: failure()

.gitmodules

@@ -39,3 +39,6 @@
 	path = modules/Micro-CDR
 	url = https://github.com/ardupilot/Micro-CDR.git
 	branch = master
+[submodule "modules/lwip"]
+	path = modules/lwip
+	url = https://github.com/ArduPilot/lwip.git

.vscode/settings.default.json

@@ -1,4 +1,5 @@
 {
     "cSpell.language": "en-GB",
-    "astyle.astylerc": "${workspaceFolder}/Tools/CodeStyle/astylerc"
+    "astyle.astylerc": "${workspaceFolder}/Tools/CodeStyle/astylerc",
+    "Lua.runtime.version": "Lua 5.3"
 }