Config improvements #327
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: API & frontend | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_USER: skyportal | |
| POSTGRES_PASSWORD: anything | |
| ports: | |
| - 5432:5432 | |
| # needed because the postgres container does not provide a | |
| # healthcheck | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_subset: | |
| [ | |
| "api_and_utils_pt1", | |
| "api_and_utils_pt2", | |
| "frontend_pt1", | |
| "frontend_pt2", | |
| "flaky", | |
| "external", | |
| ] | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 20 | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: main | |
| submodules: true | |
| - name: Checkout branch being tested | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| clean: false | |
| - name: Fetch submodules | |
| run: | | |
| git submodule update --init --recursive | |
| - uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.npm | |
| key: ${{ runner.os }}-npm-${{ hashFiles('package.json') }} | |
| - uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| key: ${{ runner.os }}-${{ hashFiles('**/requirements*.txt') }} | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt update -y | |
| ### firefox installation | |
| sudo snap remove firefox | |
| sudo add-apt-repository ppa:mozillateam/ppa | |
| printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox | |
| sudo apt install -y wget unzip firefox libcurl4-gnutls-dev libgnutls28-dev | |
| # if nginx is already installed, remove it | |
| sudo apt remove -y nginx nginx-common nginx-core nginx-full | |
| sudo apt purge -y nginx nginx-common nginx-core nginx-full | |
| # add the PPA repository with brotli support for nginx | |
| sudo add-apt-repository ppa:ondrej/nginx -y | |
| sudo apt update -y | |
| sudo apt install nginx libnginx-mod-http-brotli-static libnginx-mod-http-brotli-filter -y | |
| pip install pip==24.0 | |
| pip install wheel numpy | |
| export NPM_PACKAGES="${HOME}/.npm-packages" | |
| export PATH=${NPM_PACKAGES}/bin:$PATH | |
| export NODE_PATH="$NPM_PACKAGES/lib/node_modules:$NODE_PATH" | |
| sudo npm -g install npm@latest | |
| which python; python --version | |
| echo npm $(npm --version) | |
| echo node $(node --version) | |
| nginx -v | |
| firefox --version | |
| - name: Install Python dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| pip install -r skyportal/requirements.txt | |
| pip install -r skyportal/baselayer/requirements.txt | |
| - name: Apply Icare patches | |
| run: | | |
| ./icare.sh build --skip_services_check | |
| - name: Write SkyPortal configs | |
| run: | | |
| cd skyportal | |
| cat << EOF > config.yaml | |
| database: | |
| database: skyportal | |
| host: localhost | |
| port: 5432 | |
| user: skyportal | |
| password: anything | |
| EOF | |
| cat << EOF > config.default.yaml | |
| database: | |
| database: skyportal | |
| host: localhost | |
| port: 5432 | |
| user: skyportal | |
| password: anything | |
| EOF | |
| cat << EOF > test_config.yaml | |
| database: | |
| database: skyportal_test | |
| user: skyportal | |
| host: localhost | |
| port: 5432 | |
| password: anything | |
| server: | |
| auth: | |
| debug_login: True | |
| EOF | |
| - name: Initialize SkyPortal | |
| run: | | |
| # Usually, we create databases on the local machine, so | |
| # `createdb` just works out of the box. However, when doing | |
| # this on a server, as in our case, we need to specify extra | |
| # options. | |
| # | |
| # db_init should not complain if the databases exist already | |
| # | |
| echo "localhost:5432:*:skyportal:anything" > ~/.pgpass | |
| chmod 600 ~/.pgpass | |
| createdb -h localhost -U skyportal skyportal_test | |
| psql -U skyportal -h localhost -c "GRANT ALL PRIVILEGES ON DATABASE skyportal_test TO skyportal;" skyportal_test | |
| export NPM_CONFIG_LEGACY_PEER_DEPS="true" | |
| ./icare.sh build --init --skip_services_check | |
| pip list --format=columns | |
| npm ls --depth 0 || true | |
| - name: Install Geckodriver / Selenium | |
| run: | | |
| GECKO_VER=0.34.0 | |
| CACHED_DOWNLOAD_DIR=~/.local/downloads | |
| FILENAME=geckodriver-v${GECKO_VER}-linux64.tar.gz | |
| if [[ ! -f ${CACHED_DOWNLOAD_DIR=}/${FILENAME} ]]; then | |
| wget https://github.com/mozilla/geckodriver/releases/download/v${GECKO_VER}/${FILENAME} --directory-prefix=${CACHED_DOWNLOAD_DIR} --no-clobber | |
| fi | |
| sudo tar -xzf ${CACHED_DOWNLOAD_DIR}/geckodriver-v${GECKO_VER}-linux64.tar.gz -C /usr/local/bin | |
| geckodriver --version | |
| python -c "import selenium; print(f'Selenium {selenium.__version__}')" | |
| - name: Patch test_frontend.py for a longer timeout | |
| run: | | |
| cd patched_skyportal | |
| sed -i 's/timeout=60/timeout=180/g' baselayer/tools/test_frontend.py | |
| - name: Patch Notifications.jsx for a longer display | |
| run: | | |
| cd patched_skyportal | |
| sed -i 's/duration = 3000/duration = 15000/g' baselayer/static/js/components/Notifications.jsx | |
| - name: Refresh SkyPortal dependencies for tested version | |
| run: | | |
| cd patched_skyportal | |
| export PYTHONPATH=$PYTHONPATH:$(pwd) | |
| export NPM_CONFIG_LEGACY_PEER_DEPS="true" | |
| make dependencies | |
| - name: Run front-end tests part 1 | |
| if: ${{ matrix.test_subset == 'frontend_pt1' }} | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/frontend/sources_and_observingruns_etc' | |
| - name: Run front-end tests part 2 | |
| if: ${{ matrix.test_subset == 'frontend_pt2' }} | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless '--ignore=skyportal/tests/frontend/sources_and_observingruns_etc skyportal/tests/frontend' | |
| - name: Run API & utils tests part 1 | |
| if: ${{ matrix.test_subset == 'api_and_utils_pt1' }} | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/api/candidates_sources_events' | |
| - name: Run API & utils tests part 2 | |
| if: ${{ matrix.test_subset == 'api_and_utils_pt2' }} | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless '--ignore=skyportal/tests/api/candidates_sources_events skyportal/tests/api skyportal/tests/tools skyportal/tests/utils skyportal/tests/rate_limiting' | |
| - name: Run flaky tests | |
| if: ${{ matrix.test_subset == 'flaky' }} | |
| continue-on-error: true | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/flaky' | |
| - name: Run external tests | |
| if: ${{ matrix.test_subset == 'external' }} | |
| continue-on-error: true | |
| run: | | |
| cd patched_skyportal | |
| PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/external' | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v2 | |
| if: ${{ always() }} | |
| with: | |
| name: logs | |
| path: patched_skyportal/log | |
| - name: Upload test post-mortem reports | |
| uses: actions/upload-artifact@v2 | |
| if: ${{ always() }} | |
| with: | |
| name: test-results | |
| path: patched_skyportal/test-results | |
| - name: Upload generated plots | |
| uses: actions/upload-artifact@v2 | |
| if: ${{ always() }} | |
| with: | |
| name: generated-plots | |
| path: patched_skyportal/skyportal/tests/data/*_expected.png |