From 224502ec211f9772eaad80886cc5a259bcea93c4 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 19:04:19 -0700 Subject: [PATCH 01/15] add openpilot --- .gitmodules | 3 +++ 3rdparty/openpilot | 1 + 2 files changed, 4 insertions(+) create mode 160000 3rdparty/openpilot diff --git a/.gitmodules b/.gitmodules index 5a50669ae..99b4485bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "3rdparty/opendbc"] path = 3rdparty/opendbc_repo url = ../../commaai/opendbc.git +[submodule "3rdparty/openpilot"] + path = 3rdparty/openpilot + url = https://github.com/commaai/openpilot diff --git a/3rdparty/openpilot b/3rdparty/openpilot new file mode 160000 index 000000000..4e2624783 --- /dev/null +++ b/3rdparty/openpilot @@ -0,0 +1 @@ +Subproject commit 4e2624783fd5b102301cea3a812609cda8f6f552 From 960da1b2edb77fe546fb9ffe6d0db7bf6ec6dabb Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 19:15:21 -0700 Subject: [PATCH 02/15] remove opendbc and cereal --- .gitmodules | 6 ------ 3rdparty/cereal | 1 - 3rdparty/opendbc | 1 - 3rdparty/opendbc_repo | 1 - 4 files changed, 9 deletions(-) delete mode 160000 3rdparty/cereal delete mode 120000 3rdparty/opendbc delete mode 160000 3rdparty/opendbc_repo diff --git a/.gitmodules b/.gitmodules index 99b4485bc..0e1918218 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ -[submodule "3rdparty/cereal"] - path = 3rdparty/cereal - url = ../../commaai/cereal.git -[submodule "3rdparty/opendbc"] - path = 3rdparty/opendbc_repo - url = ../../commaai/opendbc.git [submodule "3rdparty/openpilot"] path = 3rdparty/openpilot url = https://github.com/commaai/openpilot diff --git a/3rdparty/cereal b/3rdparty/cereal deleted file mode 160000 index 8ae5c51a6..000000000 --- a/3rdparty/cereal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8ae5c51a6515c171f3a82c9511a18d79a9e55c07 diff --git a/3rdparty/opendbc b/3rdparty/opendbc deleted file mode 120000 index 7cd9a5bd1..000000000 --- a/3rdparty/opendbc +++ /dev/null @@ -1 +0,0 @@ -opendbc_repo/opendbc \ No newline at end of file diff --git a/3rdparty/opendbc_repo b/3rdparty/opendbc_repo deleted file mode 160000 index a40652d34..000000000 --- a/3rdparty/opendbc_repo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a40652d341ea3161a6a10001fa74284d9d650f8c From f23127320481baa4bb90180075e0094a93478c1f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 19:16:56 -0700 Subject: [PATCH 03/15] symlinks --- 3rdparty/cereal | 1 + 3rdparty/msgq | 1 + 3rdparty/opendbc | 1 + 3 files changed, 3 insertions(+) create mode 120000 3rdparty/cereal create mode 120000 3rdparty/msgq create mode 120000 3rdparty/opendbc diff --git a/3rdparty/cereal b/3rdparty/cereal new file mode 120000 index 000000000..19c42a96d --- /dev/null +++ b/3rdparty/cereal @@ -0,0 +1 @@ +openpilot/cereal \ No newline at end of file diff --git a/3rdparty/msgq b/3rdparty/msgq new file mode 120000 index 000000000..65a23eec0 --- /dev/null +++ b/3rdparty/msgq @@ -0,0 +1 @@ +openpilot/msgq_repo/msgq \ No newline at end of file diff --git a/3rdparty/opendbc b/3rdparty/opendbc new file mode 120000 index 000000000..e09a6047b --- /dev/null +++ b/3rdparty/opendbc @@ -0,0 +1 @@ +openpilot/opendbc_repo/opendbc \ No newline at end of file From 655397da6e7d43c0e84832cd35b32ee71601e5d1 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:06:02 -0700 Subject: [PATCH 04/15] use uv! --- Dockerfile | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index d75107027..0d7fe098d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,17 +35,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsqlite3-dev \ clang \ ocl-icd-opencl-dev \ - opencl-headers + opencl-headers \ + portaudio19-dev -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" -RUN pyenv install 3.11.4 && \ - pyenv global 3.11.4 && \ - pyenv rehash +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.cargo/bin:${PATH}" +RUN uv venv --python 3.11.4 +RUN uv pip install pkgconfig jinja2 Cython -RUN pip3 install pkgconfig jinja2 - -# installs scons, pycapnp, cython, etc. ENV PYTHONPATH /tmp/plotjuggler/3rdparty -COPY 3rdparty/opendbc_repo/requirements.txt /tmp/ -RUN pip3 install Cython && pip3 install --no-cache-dir -r /tmp/requirements.txt +COPY 3rdparty/openpilot/opendbc_repo/pyproject.toml /tmp/opendbc_repo/pyproject.toml +COPY 3rdparty/openpilot/pyproject.toml /tmp/openpilot/pyproject.toml +RUN uv pip install --no-cache-dir /tmp/opendbc_repo +RUN uv pip install --no-cache-dir /tmp/openpilot From 72d5329c30f05299ce672a2e12a8ca9704571209 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:07:18 -0700 Subject: [PATCH 05/15] source venv --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 81dc6e6b0..ddb4a4e0b 100755 --- a/build.sh +++ b/build.sh @@ -6,4 +6,4 @@ docker run \ --rm \ --volume $PWD:/tmp/plotjuggler \ --workdir /tmp/plotjuggler plotjuggler:latest \ - /bin/bash -c "mkdir -p build && cd build && cmake .. && make -j$(nproc)" + /bin/bash -c "source /.venv/bin/activate && mkdir -p build && cd build && cmake .. && make -j$(nproc)" From 9dd4d31748d15d01b563ce6d1daa5fe36d498b1d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:08:52 -0700 Subject: [PATCH 06/15] fix FindCereal.cmake --- cmake/FindCereal.cmake | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cmake/FindCereal.cmake b/cmake/FindCereal.cmake index 3d9dd1579..c092e545a 100644 --- a/cmake/FindCereal.cmake +++ b/cmake/FindCereal.cmake @@ -5,8 +5,13 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) # build cereal before we search for required Cereal plugin libs below execute_process( - COMMAND scons -j8 --minimal - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/cereal + COMMAND scons -j8 --minimal -u + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal + ) + + execute_process( + COMMAND scons -j8 --minimal -u + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq ) find_path(Cereal_INCLUDE_DIRS @@ -29,14 +34,21 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) NAMES cereal PATHS - ${CMAKE_SOURCE_DIR}/3rdparty/cereal + ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal + ) + + find_library(Cereal_LIBRARY_COMMON + NAMES + common + PATHS + ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/common ) - find_library(Cereal_LIBRARY_MESSAGING + find_library(Cereal_LIBRARY_MSGQ NAMES - messaging + msgq PATHS - ${CMAKE_SOURCE_DIR}/3rdparty/cereal + ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/msgq_repo ) find_library(Cereal_LIBRARY_ZMQ @@ -47,7 +59,8 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) set(Cereal_LIBRARIES ${Cereal_LIBRARIES} ${Cereal_LIBRARY_CEREAL} - ${Cereal_LIBRARY_MESSAGING} + ${Cereal_LIBRARY_COMMON} + ${Cereal_LIBRARY_MSGQ} ${Cereal_LIBRARY_ZMQ} ) From 4b32ada7abfecf768bea2c02953c0d469225715a Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:13:42 -0700 Subject: [PATCH 07/15] bump openpilot --- 3rdparty/openpilot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/openpilot b/3rdparty/openpilot index 4e2624783..756d64e2c 160000 --- a/3rdparty/openpilot +++ b/3rdparty/openpilot @@ -1 +1 @@ -Subproject commit 4e2624783fd5b102301cea3a812609cda8f6f552 +Subproject commit 756d64e2c3a38983caa56cefd55e58fb663332c6 From 595a1054e3b525949d376016a42af32f2fd127ca Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:15:28 -0700 Subject: [PATCH 08/15] add common too --- 3rdparty/common | 1 + 1 file changed, 1 insertion(+) create mode 120000 3rdparty/common diff --git a/3rdparty/common b/3rdparty/common new file mode 120000 index 000000000..47b9db83f --- /dev/null +++ b/3rdparty/common @@ -0,0 +1 @@ +openpilot/common \ No newline at end of file From 6dff05ca463abc1c7ef6abf11e3836046501ddb5 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:18:28 -0700 Subject: [PATCH 09/15] not needed --- cmake/FindCereal.cmake | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmake/FindCereal.cmake b/cmake/FindCereal.cmake index c092e545a..4f2f8af3f 100644 --- a/cmake/FindCereal.cmake +++ b/cmake/FindCereal.cmake @@ -37,13 +37,6 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/cereal ) - find_library(Cereal_LIBRARY_COMMON - NAMES - common - PATHS - ${CMAKE_SOURCE_DIR}/3rdparty/openpilot/common - ) - find_library(Cereal_LIBRARY_MSGQ NAMES msgq @@ -59,7 +52,6 @@ else (Cereal_LIBRARIES AND Cereal_INCLUDE_DIRS) set(Cereal_LIBRARIES ${Cereal_LIBRARIES} ${Cereal_LIBRARY_CEREAL} - ${Cereal_LIBRARY_COMMON} ${Cereal_LIBRARY_MSGQ} ${Cereal_LIBRARY_ZMQ} ) From 3195ab132bcd956c95f52baca65a3c8881c0edb6 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:20:51 -0700 Subject: [PATCH 10/15] fix build --- .github/workflows/openpilot.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index e247d65b9..ad2c89371 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -16,12 +16,12 @@ jobs: arch: ${{ fromJson( ((github.repository == 'commaai/plotjuggler') && ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} + (github.event.pull_request.head.repo.full_name == 'commaai/plotjuggler'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-4vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - name: build run: ./build.sh - name: zip plugins @@ -42,7 +42,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - uses: actions/setup-python@v2 with: python-version: '3.11.x' From b500f6be5ac0c64ccf50dec15f4c949c11cf741e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:37:32 -0700 Subject: [PATCH 11/15] fix macos --- .github/workflows/openpilot.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index ad2c89371..c889b0c6d 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -49,11 +49,14 @@ jobs: - name: setup dependencies run: | brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar - sudo pip3 install --no-cache-dir -r 3rdparty/opendbc_repo/requirements.txt + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv + uv pip install 3rdparty/opendbc_repo - name: build run: | export PYTHONPATH=$PWD/3rdparty # force usage of protobuf@3 instead of protobuf + source .venv/bin/activate mkdir -p build && cd build && cmake -DCMAKE_PREFIX_PATH="$(brew --prefix)/opt/protobuf@3" .. && make -j8 - name: zip plugins run: eval "$ZIP" From a871fc58ac7b451d1e84138c0dcdacea9a8dded2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:40:35 -0700 Subject: [PATCH 12/15] test --- .github/workflows/openpilot.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index c889b0c6d..711da8ccd 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -48,10 +48,10 @@ jobs: python-version: '3.11.x' - name: setup dependencies run: | + uv venv + uv pip install 3rdparty/openpilot/opendbc_repo brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar curl -LsSf https://astral.sh/uv/install.sh | sh - uv venv - uv pip install 3rdparty/opendbc_repo - name: build run: | export PYTHONPATH=$PWD/3rdparty From cac2fc4b95936a2485e93844c5b6e2404a3e1274 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:41:23 -0700 Subject: [PATCH 13/15] test --- .github/workflows/openpilot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index 711da8ccd..40529cc56 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -48,10 +48,10 @@ jobs: python-version: '3.11.x' - name: setup dependencies run: | + curl -LsSf https://astral.sh/uv/install.sh | sh uv venv uv pip install 3rdparty/openpilot/opendbc_repo brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar - curl -LsSf https://astral.sh/uv/install.sh | sh - name: build run: | export PYTHONPATH=$PWD/3rdparty From f2d1ae211f17b363802304d751f81f212afed03b Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:42:43 -0700 Subject: [PATCH 14/15] need op too --- .github/workflows/openpilot.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index 40529cc56..2d855d2df 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -48,10 +48,10 @@ jobs: python-version: '3.11.x' - name: setup dependencies run: | + brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar curl -LsSf https://astral.sh/uv/install.sh | sh uv venv - uv pip install 3rdparty/openpilot/opendbc_repo - brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar + uv pip install 3rdparty/openpilot 3rdparty/openpilot/opendbc_repo - name: build run: | export PYTHONPATH=$PWD/3rdparty From 80e68b23d20f30a98e05566d3cabd53afa6090ff Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 27 Aug 2024 20:45:26 -0700 Subject: [PATCH 15/15] portaudio --- .github/workflows/openpilot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openpilot.yaml b/.github/workflows/openpilot.yaml index 2d855d2df..4ec0d7f35 100644 --- a/.github/workflows/openpilot.yaml +++ b/.github/workflows/openpilot.yaml @@ -48,7 +48,7 @@ jobs: python-version: '3.11.x' - name: setup dependencies run: | - brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar + brew install cmake bzip2 capnp qt@5 zeromq protobuf@3 gnu-tar portaudio curl -LsSf https://astral.sh/uv/install.sh | sh uv venv uv pip install 3rdparty/openpilot 3rdparty/openpilot/opendbc_repo