This repository has been archived by the owner on Jun 9, 2024. It is now read-only.
CI #766
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_dispatch: | |
| branches: [master] | |
| schedule: | |
| - cron: "0 8 * * *" | |
| push: | |
| branches: [master, ci-test*] | |
| paths-ignore: | |
| - "reports/**" | |
| pull_request: | |
| branches: [stable, master, release-*] | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| env: | |
| min-python-version: "3.10" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| submodules: true | |
| - name: Set up Python ${{ env.min-python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.min-python-version }} | |
| - id: get_date | |
| name: Get date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python - | |
| - name: Install dependencies | |
| run: | | |
| export POETRY_VIRTUALENVS_IN_PROJECT=true | |
| poetry install -vvv | |
| - name: Lint with flake8 | |
| run: poetry run flake8 | |
| - name: Check black formatting | |
| run: poetry run black . --exclude test.py --check | |
| if: success() || failure() | |
| - name: Check isort formatting | |
| run: poetry run isort . --check | |
| if: success() || failure() | |
| - name: Check mypy formatting | |
| run: poetry run mypy --ignore-missing-imports . | |
| if: success() || failure() | |
| - name: Check for unused imports and pass statements | |
| run: | | |
| cmd="poetry run autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports --ignore-pass-after-docstring agbenchmark" | |
| $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1) | |
| if: success() || failure() | |
| tests: | |
| env: | |
| GH_TOKEN: ${{ github.event_name == 'pull_request' && github.token || secrets.PAT }} | |
| min-python-version: "3.10" | |
| name: "${{ matrix.agent-name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| agent-name: | |
| - "gpt-engineer" | |
| - "smol-developer" | |
| - "Auto-GPT" | |
| - "mini-agi" | |
| - "beebot" | |
| - "BabyAGI" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| submodules: true | |
| token: ${{ env.GH_TOKEN }} | |
| - name: Set up Python ${{ env.min-python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.min-python-version }} | |
| - id: get_date | |
| name: Get date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python - | |
| - name: Install dependencies | |
| run: | | |
| poetry install -vvv | |
| poetry build | |
| - name: Run regression tests | |
| run: | | |
| cd agent/$AGENT_NAME | |
| prefix="" | |
| if [ "$AGENT_NAME" == "gpt-engineer" ]; then | |
| make install | |
| source venv/bin/activate | |
| elif [ "$AGENT_NAME" == "Auto-GPT" ]; then | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -r requirements.txt | |
| pip uninstall agbenchmark -y | |
| elif [ "$AGENT_NAME" == "mini-agi" ]; then | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -r requirements.txt | |
| cp .env_example .env | |
| elif [ "$AGENT_NAME" == "smol-developer" ]; then | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -r requirements.txt | |
| elif [ "$AGENT_NAME" == "BabyAGI" ]; then | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -r requirements.txt | |
| elif [ "$AGENT_NAME" == "SuperAGI" ]; then | |
| cp config_template.yaml config.yaml | |
| sed -i 's/OPENAI_API_KEY:.*/OPENAI_API_KEY: "'"${{ secrets.OPENAI_API_KEY }}"'"/' config.yaml | |
| docker-compose up -d --build | |
| elif [ "$AGENT_NAME" == "beebot" ]; then | |
| poetry install | |
| poetry run playwright install | |
| prefix="poetry run " | |
| else | |
| echo "Unknown agent name: $AGENT_NAME" | |
| exit 1 | |
| fi | |
| pip install ../../dist/*.whl | |
| if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then | |
| set +e # Ignore non-zero exit codes and continue execution | |
| ${prefix}agbenchmark start --maintain --mock | |
| EXIT_CODE=$? | |
| set -e # Stop ignoring non-zero exit codes | |
| # Check if the exit code was 5, and if so, exit with 0 instead | |
| if [ $EXIT_CODE -eq 5 ] | |
| then | |
| echo "regression_tests.json is empty." | |
| exit 0 | |
| else | |
| exit $EXIT_CODE | |
| fi | |
| set +e # Ignore non-zero exit codes and continue execution | |
| improve_cmd = ${prefix}agbenchmark start --improve --mock | |
| EXIT_CODE=$? | |
| set -e # Stop ignoring non-zero exit codes | |
| # Check if the exit code was 5, and if so, exit with 0 instead | |
| if [ $EXIT_CODE -eq 5 ] | |
| then | |
| echo "regression_tests.json is empty." | |
| exit 0 | |
| else | |
| exit $EXIT_CODE | |
| fi | |
| ${prefix}agbenchmark start --mock | |
| ${prefix}agbenchmark start --mock --category=retrieval | |
| ${prefix}agbenchmark start --mock --category=interface | |
| ${prefix}agbenchmark start --mock --category=code | |
| ${prefix}agbenchmark start --mock --category=memory | |
| ${prefix}agbenchmark start --mock --category=iterate | |
| ${prefix}agbenchmark start --mock --suite TestReturnCode | |
| ${prefix}agbenchmark start --mock --suite TestRevenueRetrieval | |
| else | |
| bash -c "$(curl -fsSL https://raw.githubusercontent.com/merwanehamadi/helicone/b7ab4bc53e51d8ab29fff19ce5986ab7720970c6/mitmproxy.sh)" -s start | |
| ${prefix}agbenchmark start || echo "This command will always return a non zero exit code unless all the challenges are solved." | |
| fi | |
| cd ../.. | |
| env: | |
| GITHUB_EVENT_NAME: ${{ github.event_name }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AGENT_NAME: ${{ matrix.agent-name }} | |
| PROMPT_USER: false # For mini-agi. TODO: Remove this once mini-agi follows the standards. | |
| HELICONE_API_KEY: ${{ secrets.HELICONE_API_KEY }} | |
| REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt | |
| HELICONE_CACHE_ENABLED: false | |
| HELICONE_PROPERTY_AGENT: ${{ matrix.agent-name }} | |
| REPORT_LOCATION: ${{ format('../../reports/{0}', matrix.agent-name) }} | |
| WOLFRAM_ALPHA_APPID: ${{ secrets.WOLFRAM_ALPHA_APPID }} | |
| SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }} | |
| BING_SUBSCRIPTION_KEY: ${{ secrets.BING_SUBSCRIPTION_KEY }} | |
| - name: Upload reports | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.agent-name }} | |
| path: reports/${{ matrix.agent-name }} | |
| - name: Authenticate and Push to Branch | |
| if: (success() || failure()) && (github.event_name != 'pull_request') | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Auto-GPT-Bot" | |
| git add reports/* || echo "nothing to commit" | |
| commit_message="${{ matrix.agent-name }}-$(date +'%Y%m%d%H%M%S')" | |
| git commit -m "${commit_message}" | |
| git stash | |
| current_branch=${{ github.ref_name }} | |
| attempts=0 | |
| max_attempts=3 | |
| while [ $attempts -lt $max_attempts ]; do | |
| git fetch origin $current_branch | |
| git rebase origin/$current_branch | |
| if git push origin HEAD; then | |
| echo "Success!" | |
| poetry run python send_to_googledrive.py || echo "Failed to upload to Google Drive" | |
| exit 0 | |
| else | |
| echo "Attempt $(($attempts + 1)) failed. Retrying..." | |
| attempts=$(($attempts + 1)) | |
| fi | |
| done | |
| echo "Failed after $max_attempts attempts." | |
| env: | |
| GDRIVE_BASE64: ${{ secrets.GDRIVE_BASE64 }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} |