ci: faster ui preview (#33890)

* fast * fast * cleanup
commaai · Oct 30, 2024 · fe53185 · fe53185
1 parent 3f09d19
commit fe53185
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 38 deletions.
diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
@@ -352,19 +352,29 @@ jobs:
   create_ui_report:
     # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
     name: Create UI Report
-    runs-on: ubuntu-latest
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
       - uses: ./.github/workflows/setup-with-retry
+      - name: caching frames
+        id: frames-cache
+        uses: actions/cache@v4
+        with:
+          path: .ci_cache/comma_download_cache
+          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
       - name: Build openpilot
         run: ${{ env.RUN }} "scons -j$(nproc)"
       - name: Create Test Report
+        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }}
         run: >
             ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                             source selfdrive/test/setup_xvfb.sh &&
-                            python3 selfdrive/ui/tests/test_ui/run.py"
+                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
+                            chmod -R 777 /tmp/comma_download_cache"
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:

diff --git a/selfdrive/ui/tests/test_ui/run.py b/selfdrive/ui/tests/test_ui/run.py
@@ -3,11 +3,10 @@
 import pathlib
 import shutil
 import sys
-import jinja2
-import matplotlib.pyplot as plt
-import numpy as np
 import os
 import pywinctl
+import pyautogui
+import pickle
 import time
 
 from cereal import log
@@ -23,8 +22,9 @@
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.framereader import FrameReader
 from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.cache import DEFAULT_CACHE_DIR
 
-UI_DELAY = 0.5 # may be slower on CI?
+UI_DELAY = 0.1 # may be slower on CI?
 TEST_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"
 
 STREAMS: list[tuple[VisionStreamType, CameraConfig, bytes]] = []
@@ -222,41 +222,20 @@ def setup(self):
       print(f"failed to find ui window, assuming that it's in the top left (for Xvfb) {e}")
       self.ui = namedtuple("bb", ["left", "top", "width", "height"])(0,0,2160,1080)
 
-  def screenshot(self):
-    import pyautogui
-    im = pyautogui.screenshot(region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height))
+  def screenshot(self, name):
+    im = pyautogui.screenshot(SCREENSHOTS_DIR / f"{name}.png", region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height))
     assert im.width == 2160
     assert im.height == 1080
-    img = np.array(im)
-    im.close()
-    return img
 
   def click(self, x, y, *args, **kwargs):
-    import pyautogui
     pyautogui.click(self.ui.left + x, self.ui.top + y, *args, **kwargs)
     time.sleep(UI_DELAY) # give enough time for the UI to react
 
   @with_processes(["ui"])
   def test_ui(self, name, setup_case):
     self.setup()
-
     setup_case(self.click, self.pm)
-
-    im = self.screenshot()
-    plt.imsave(SCREENSHOTS_DIR / f"{name}.png", im)
-
-
-def create_html_report():
-  OUTPUT_FILE = TEST_OUTPUT_DIR / "index.html"
-
-  with open(TEST_DIR / "template.html") as f:
-    template = jinja2.Template(f.read())
-
-  cases = {f.stem: (str(f.relative_to(TEST_OUTPUT_DIR)), "reference.png") for f in SCREENSHOTS_DIR.glob("*.png")}
-  cases = dict(sorted(cases.items()))
-
-  with open(OUTPUT_FILE, "w") as f:
-    f.write(template.render(cases=cases))
+    self.screenshot(name)
 
 def create_screenshots():
   if TEST_OUTPUT_DIR.exists():
@@ -277,13 +256,23 @@ def create_screenshots():
       break
 
   cam = DEVICE_CAMERAS[("tici", "ar0231")]
-  road_img = FrameReader(route.camera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
-  STREAMS.append((VisionStreamType.VISION_STREAM_ROAD, cam.fcam, road_img.flatten().tobytes()))
 
-  wide_road_img = FrameReader(route.ecamera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
-  STREAMS.append((VisionStreamType.VISION_STREAM_WIDE_ROAD, cam.ecam, wide_road_img.flatten().tobytes()))
+  frames_cache = f'{DEFAULT_CACHE_DIR}/ui_frames'
+  if os.path.isfile(frames_cache):
+    with open(frames_cache, 'rb') as f:
+      frames = pickle.load(f)
+      road_img = frames[0]
+      wide_road_img = frames[1]
+      driver_img = frames[2]
+  else:
+    with open(frames_cache, 'wb') as f:
+      road_img = FrameReader(route.camera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
+      wide_road_img = FrameReader(route.ecamera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
+      driver_img = FrameReader(route.dcamera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
+      pickle.dump([road_img, wide_road_img, driver_img], f)
 
-  driver_img = FrameReader(route.dcamera_paths()[segnum]).get(0, pix_fmt="nv12")[0]
+  STREAMS.append((VisionStreamType.VISION_STREAM_ROAD, cam.fcam, road_img.flatten().tobytes()))
+  STREAMS.append((VisionStreamType.VISION_STREAM_WIDE_ROAD, cam.ecam, wide_road_img.flatten().tobytes()))
   STREAMS.append((VisionStreamType.VISION_STREAM_DRIVER, cam.dcam, driver_img.flatten().tobytes()))
 
   t = TestUI()
@@ -302,6 +291,3 @@ def create_screenshots():
 if __name__ == "__main__":
   print("creating test screenshots")
   create_screenshots()
-
-  print("creating html report")
-  create_html_report()