fix: dramatically reduce checkpoint memory consumption (#2956) #7602
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: python_build | |
| on: | |
| merge_group: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| defaults: | |
| run: | |
| working-directory: ./python | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Check Python | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions | |
| make check-python | |
| - name: Check Rust | |
| run: make check-rust | |
| test-minimal: | |
| name: Python Build (Python 3.8 PyArrow 16.0.0) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: 3.8 | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| # Install minimum PyArrow version | |
| pip install -e .[pandas,devel] pyarrow==16.0.0 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make unit-test | |
| test: | |
| name: Python Build (Python 3.10 PyArrow latest) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop | |
| - name: Download Data Acceptance Tests (DAT) files | |
| run: make setup-dat | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| python -m pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules | |
| - name: Test without pandas | |
| run: | | |
| source venv/bin/activate | |
| pip uninstall --yes pandas | |
| python -m pytest -m "not pandas and not integration and not benchmark" | |
| pip install pandas | |
| test-pyspark: | |
| name: PySpark Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop-pyspark | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make test-pyspark | |
| multi-python-running: | |
| name: Running with Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| maturin develop | |
| - name: Run deltalake | |
| run: | | |
| source venv/bin/activate | |
| python -c 'import deltalake' |