Try to free up disk space before running the actual tox tasks

.github/workflows/tox.yaml

@@ -44,6 +44,8 @@ jobs:
             ~/.cache/pip
             .tox
           key: ${{runner.os}}-${{github.ref}}-${{hashFiles('requirements.txt', 'setup.py', 'tox.ini')}}
+      - name: Free disk space
+        run: sh build_scripts/free_disk_space.sh
       - name: Install Tox and Python Packages
         run: pip install tox
       - name: Run Tox

build_scripts/free_disk_space.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# The Azure provided machines typically have the following disk allocation:
+# Total space: 85GB
+# Allocated: 67 GB
+# Free: 17 GB
+# This script frees up 28 GB of disk space by deleting unneeded packages and 
+# large directories.
+# The Flink end to end tests download and generate more than 17 GB of files,
+# causing unpredictable behavior and build failures.
+#
+echo "=============================================================================="
+echo "Freeing up disk space on CI system"
+echo "=============================================================================="
+
+echo "Listing 100 largest packages (largest at bottom)"
+dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+df -h
+echo "Removing large packages"
+sudo apt-get remove -y '^ghc-8.*'
+sudo apt-get remove -y '^dotnet-.*'
+sudo apt-get remove -y '^llvm-.*'
+sudo apt-get remove -y '^temurin-.*'
+sudo apt-get remove -y 'php.*'
+sudo apt-get remove -y azure-cli google-cloud-sdk google-cloud-cli hhvm google-chrome-stable firefox powershell mono-devel microsoft-edge-stable google-chrome-stable powershell linux-modules-6.5.0-1025-azure mysql-server-core-8.0 libllvm15 libllvm14
+sudo apt-get autoremove -y
+sudo apt-get clean
+df -h
+echo "Removing large directories"
+# deleting 15GB
+rm -rf /usr/share/dotnet/
+df -h

tox.ini

@@ -13,6 +13,8 @@ isolated_build = True
 # this issue is resolved by running pytest-cov within tox development mode, thus not creating an sdist
 usedevelop = true
 commands =
+; Clean up coverage data and run the tests as well as the notebook tests.
+; The latter are run only in this environment.
     coverage erase
     pytest -n 4 --cov --cov-append --cov-report=term-missing tests
     pytest -n 4 notebooks
@@ -32,13 +34,15 @@ deps =
     -rrequirements_lightgbm.txt
     -rrequirements_geoanalytics.txt
     -rrequirements_tensorflow.txt
+whitelist_externals =
+    sh
+allowlist_externals =
+    sh
 
 # This environment tests the latest dependencies
 [testenv:py_latest_dependencies]
 commands =
     pytest
-; NOTE: notebooks are not executed here again (the build would become very slow) so they are not tested with
-;   latest dependencies. Hopefully, this is not a problem
 deps =
     pytest
 ;   NOTE: For some reason including this into setup.py makes things slow locally, fail silently on gitlab