From 4a640c12a656911db49d8da93b76c5b76dc228a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20K=C3=A4hm?= Date: Thu, 26 Oct 2023 19:11:32 +0200 Subject: [PATCH] [TASK] sync the CI stuff from main branch into 11.6.x Relates: #3781 --- .github/ISSUE_TEMPLATE/bug_report.md | 8 +- .github/workflows/ci.yml | 71 ++++++----------- ...> GET_LOCAL_PACKAGE_VERSION_CONSTRAINT.sh} | 0 Build/Test/IntegrationTests.xml | 1 + Build/Test/IntegrationTestsBootstrap.php | 6 +- Build/Test/UnitTests.xml | 1 + Build/Test/bootstrap.sh | 77 +++++-------------- Build/Test/cibuild.sh | 72 ++++++++--------- Build/Test/phpstan-constants.php | 5 ++ Build/Test/phpstan.neon | 20 +++++ Docker/Ci/environment.yml | 1 - composer.json | 61 ++++++++++++--- 12 files changed, 160 insertions(+), 163 deletions(-) rename Build/Helpers/{GET_LACAL_PACKAGE_VERSION_CONSTRAINT.sh => GET_LOCAL_PACKAGE_VERSION_CONSTRAINT.sh} (100%) create mode 100644 Build/Test/phpstan-constants.php create mode 100644 Build/Test/phpstan.neon diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6a88cad5b4..820824f63a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,11 +24,11 @@ A clear and concise description of what you expected to happen. If applicable, add screenshots to help explain your problem. **Used versions (please complete the following information):** - - TYPO3 Version: [e.g. 11.5.26] + - TYPO3 Version: [e.g. 12.4.0] - Browser: [e.g. chrome, safari] - - EXT:solr Version: [e.g. 11.6.0] - - Used Apache Solr Version: [e.g. 9.3.1] - - PHP Version: [e.g. 8.1.0] + - EXT:solr Version: [e.g. 12.0.0] + - Used Apache Solr Version: [e.g. 9.3.0] + - PHP Version: [e.g. 8.2.0] - MySQL Version: [e.g. 8.0.0] **Additional context** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e016b3478d..4f093e183f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,18 +23,21 @@ jobs: outputs: matrix: ${{ steps.collect_build_matrix.outputs.matrix }} steps: - # Workaround for issue with actions/checkout@v2 wrong PR commit checkout: See https://github.com/actions/checkout/issues/299#issuecomment-677674415 + # Workaround for issue with actions/checkout "wrong PR commit checkout": + # See: + # ** https://github.com/actions/checkout/issues/299#issuecomment-677674415 + # ** https://github.com/actions/checkout/issues/1359#issuecomment-1631503791 - name: Checkout current state of Pull Request if: github.event_name == 'pull_request' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Checkout current state of Branch if: github.event_name == 'push' - uses: actions/checkout@v2 - # End: Workaround for issue with actions/checkout@v2 wrong PR commit checkout + uses: actions/checkout@v3 + # End: Workaround for issue with actions/checkout... - name: "Resolve target branch of pull request." if: github.event_name == 'pull_request' @@ -66,13 +69,15 @@ jobs: echo "BRANCH_NAME="$BRANCH_NAME echo -e "matrix : " echo $matrix - echo ::set-output name=matrix::$(echo $matrix) + echo "matrix=$(echo $matrix)" >> $GITHUB_OUTPUT + >&2 echo -e "Example Annotation on error. Is visible in Actions workflow view." + >&2 echo -e "Non-stable releases can not be published to TER. The tags containing beta-1 is invalid for TER." - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Build Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . file: ./Docker/SolrServer/Dockerfile @@ -88,7 +93,7 @@ jobs: ./Build/Test/cibuild_docker.sh - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: solrci-image path: /tmp/solrci-image.tar @@ -104,40 +109,39 @@ jobs: TYPO3_DATABASE_HOST: '127.0.0.1' TYPO3_DATABASE_USERNAME: 'root' TYPO3_DATABASE_PASSWORD: 'root' - PHP_CS_FIXER_VERSION: '^3.2.1' TYPO3_VERSION: ${{ matrix.TYPO3 }} name: TYPO3 ${{ matrix.TYPO3 }} on PHP ${{ matrix.PHP }} steps: - # Workaround for issue with actions/checkout@v2 wrong PR commit checkout: See https://github.com/actions/checkout/issues/299#issuecomment-677674415 + # Workaround for issue with actions/checkout "wrong PR commit checkout". See: ci_bootstrapping job - name: Checkout current state of Pull Request if: github.event_name == 'pull_request' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 2 ref: ${{ github.event.pull_request.head.sha }} - name: Checkout current state of Branch if: github.event_name == 'push' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 2 - # End: Workaround for issue with actions/checkout@v2 wrong PR commit checkout + # End: Workaround for issue with actions/checkout... - name: Mount RAMFS run: | mkdir -p ${{ env.CI_BUILD_DIRECTORY }} - sudo mount -t tmpfs -o size=3G none ${{ env.CI_BUILD_DIRECTORY }} + sudo mount -t tmpfs -o size=1G none ${{ env.CI_BUILD_DIRECTORY }} sudo mkdir -p ${{ env.CI_BUILD_DIRECTORY }}/data-{solr,mysql} \ && sudo chown $USER ${{ env.CI_BUILD_DIRECTORY }}/data-mysql \ && sudo chown 8983:8983 ${{ env.CI_BUILD_DIRECTORY }}/data-solr - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Download solrci-image from "ci_bootstrapping" job - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: solrci-image path: /tmp @@ -160,36 +164,9 @@ jobs: with: php-version: ${{ matrix.PHP }} coverage: pcov - tools: composer:2.1.14 - - - name: Resolve CI build cache key - # CI_CACHE_VERSION is used and can be increased to be able to invalidate caches. - # For example if some composer dependencies added or removed in composer.json or - # in Build/Test/bootstrap.sh - run: | - export CURRENT_TYPO3_VERSION_REFERNCE=$(./Build/Helpers/TYPO3_SOURCE_REFERENCE.sh "$TYPO3_VERSION" --short) - export CURRENT_SOLARIUM_VERSION=$(cat composer.json | jq --raw-output '.require."solarium/solarium"') - export CI_CACHE_VERSION="2023.01.20@00:00" - export CI_BUILD_CACHE_KEY=${{ runner.os }}-PHP:${{ matrix.PHP }}-TYPO3:$TYPO3_VERSION@$CURRENT_TYPO3_VERSION_REFERNCE-SOLARIUM:$CURRENT_SOLARIUM_VERSION-"CI_CACHE_VERSION:"$CI_CACHE_VERSION - echo "COMPOSER_GLOBAL_REQUEREMENTS=$(composer config home)" >> $GITHUB_ENV - echo "CI_BUILD_CACHE_KEY=$CI_BUILD_CACHE_KEY" >> $GITHUB_ENV - echo "The key for actions/cache@v2 is \"$CI_BUILD_CACHE_KEY\"" - - - name: Restore ci build caches - id: restore_ci_build_caches - uses: actions/cache@v2 - with: - path: | - ${{ env.CI_BUILD_DIRECTORY }}/Web - ${{ env.CI_BUILD_DIRECTORY }}/bin - ${{ env.CI_BUILD_DIRECTORY }}/vendor - ${{ env.COMPOSER_GLOBAL_REQUEREMENTS }} - composer.json - composer.lock - key: ${{ env.CI_BUILD_CACHE_KEY }} + tools: composer:2.5.5 - name: CI-Bootstrap - if: steps.restore_ci_build_caches.outputs.cache-hit != 'true' run: | ./Build/Test/bootstrap.sh --skip-solr-install echo "Current Size of EXT:Solr build Artefacts before run: " \ @@ -204,8 +181,8 @@ jobs: - name: Upload code coverage to Scrutinizer run: | - .Build/bin/ocular code-coverage:upload --format=php-clover coverage.unit.clover - .Build/bin/ocular code-coverage:upload --format=php-clover coverage.integration.clover + ocular code-coverage:upload --format=php-clover coverage.unit.clover + ocular code-coverage:upload --format=php-clover coverage.integration.clover - name: Clean up run: | @@ -225,7 +202,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - diff --git a/Build/Helpers/GET_LACAL_PACKAGE_VERSION_CONSTRAINT.sh b/Build/Helpers/GET_LOCAL_PACKAGE_VERSION_CONSTRAINT.sh similarity index 100% rename from Build/Helpers/GET_LACAL_PACKAGE_VERSION_CONSTRAINT.sh rename to Build/Helpers/GET_LOCAL_PACKAGE_VERSION_CONSTRAINT.sh diff --git a/Build/Test/IntegrationTests.xml b/Build/Test/IntegrationTests.xml index 55d51013f0..4f81a8e17d 100644 --- a/Build/Test/IntegrationTests.xml +++ b/Build/Test/IntegrationTests.xml @@ -1,6 +1,7 @@ defineOriginalRootPath(); $testbase->createDirectory(ORIGINAL_ROOT . 'typo3temp/var/tests'); $testbase->createDirectory(ORIGINAL_ROOT . 'typo3temp/var/transient'); diff --git a/Build/Test/UnitTests.xml b/Build/Test/UnitTests.xml index d5e24cf3bc..79f343afcf 100644 --- a/Build/Test/UnitTests.xml +++ b/Build/Test/UnitTests.xml @@ -1,6 +1,7 @@ /dev/null 2>&1 then echo "Couldn't find wget." exit 1 fi -COMPOSER_NO_INTERACTION=1 - -# Setup TYPO3 environment variables -export TYPO3_PATH_PACKAGES="${EXTENSION_ROOTPATH}.Build/vendor/" -export TYPO3_PATH_WEB="${EXTENSION_ROOTPATH}.Build/Web/" +# Use latest TYPO3 LTS stable version, if version number is compatible with get.typo3.org API +if [[ $TYPO3_VERSION =~ ^[0-9]+$ ]] ; then + TYPO3_VERSION=$("${BASH_SOURCE%/*}/../Helpers/TYPO3_GET_LATEST_VERSION.sh" "$TYPO3_VERSION") +fi -echo "Installing test environment" -echo "Using extension path $EXTENSION_ROOTPATH" -echo "Using package path $TYPO3_PATH_PACKAGES" -echo "Using web path $TYPO3_PATH_WEB" +echo "Using TYPO3 Version: $TYPO3_VERSION" +echo "Using database host: $TYPO3_DATABASE_HOST" +echo "Using database dbname: $TYPO3_DATABASE_NAME" +echo "Using database user: $TYPO3_DATABASE_USERNAME" +echo "Using database password: $TYPO3_DATABASE_PASSWORD" -# Install TYPO3 sources -if [[ $TYPO3_VERSION = *"master"* ]]; then - composer config minimum-stability dev -fi +COMPOSER_NO_INTERACTION=1 -if ! composer require --dev --update-with-dependencies --prefer-source \ - typo3/cms-core:"$TYPO3_VERSION" \ - typo3/cms-backend:"$TYPO3_VERSION" \ - typo3/cms-recordlist:"$TYPO3_VERSION" \ - typo3/cms-fluid:"$TYPO3_VERSION" \ - typo3/cms-fluid-styled-content:"$TYPO3_VERSION" \ - typo3/cms-frontend:"$TYPO3_VERSION" \ - typo3/cms-extbase:"$TYPO3_VERSION" \ - typo3/cms-reports:"$TYPO3_VERSION" \ - typo3/cms-scheduler:"$TYPO3_VERSION" \ - typo3/cms-tstemplate:"$TYPO3_VERSION" \ - typo3/cms-install:"$TYPO3_VERSION" +echo "Installing test environment" +if ! composer tests:setup then echo "The test environment could not be installed by composer as expected. Please fix this issue." exit 1 fi -mkdir -p $TYPO3_PATH_WEB/uploads $TYPO3_PATH_WEB/typo3temp +echo "Install third party tools globally:" +export PATH=$PATH:$(composer config --global home)/vendor/bin +if ! composer tests:setup:global-require +then + "The test environment could not be installed by composer as expected. Please fix this issue." + exit 1 +fi diff --git a/Build/Test/cibuild.sh b/Build/Test/cibuild.sh index 2f12bb4a73..17a40e61ee 100755 --- a/Build/Test/cibuild.sh +++ b/Build/Test/cibuild.sh @@ -1,13 +1,6 @@ #!/usr/bin/env bash -TYPO3_PATH_WEB="$(pwd)/.Build/Web/" -export TYPO3_PATH_WEB -TYPO3_PATH_PACKAGES="$(pwd)/.Build/vendor/" -export TYPO3_PATH_PACKAGES - -TYPO3_BIN_DIR="$(pwd)/.Build/bin/" -export TYPO3_BIN_DIR -export PATH="$TYPO3_BIN_DIR:$PATH" +EXIT_CODE=0 COMPOSERS_BIN_DIR="$(composer config home)/vendor/bin" # Add COMPOSERS_BIN_DIR to $PATH, if not present @@ -25,54 +18,48 @@ echo "Run PHP Lint" if ! find . -name \*.php ! -path "./.Build/*" 2>/dev/null | parallel --gnu php -d display_errors=stderr -l {} > /dev/null then echo "There are syntax errors, please check and fix them." - exit 1 + EXIT_CODE=1 else echo "No syntax errors! Great job!" fi -echo "Check compliance against TYPO3 Coding Standards" -if ! .Build/bin/php-cs-fixer --version > /dev/null 2>&1 + +echo "TYPO3 Coding Standards compliance: See https://github.com/TYPO3/coding-standards" +if ! composer t3:standards:fix -- --diff --verbose --dry-run && rm .php-cs-fixer.cache then - echo "TYPO3 https://github.com/TYPO3/coding-standards is not set properly." - echo "Please fix that asap to avoid unwanted changes in the future." - exit 1 + echo "Some files are not compliant to TYPO3 Coding Standards" + echo "Please fix the files listed above." + echo "Tip for auto fix: " + echo " TYPO3_VERSION=\"${TYPO3_VERSION}\" composer tests:setup && composer t3:standards:fix" + EXIT_CODE=3 else - echo "TYPO3 Coding Standards compliance: See https://github.com/TYPO3/coding-standards" - if ! composer t3:standards:fix -- --diff --verbose --dry-run && rm .php-cs-fixer.cache - then - echo "Some files are not compliant to TYPO3 Coding Standards" - echo "Please fix the files listed above." - echo "Tip for auto fix: " - echo " composer tests:setup && composer t3:standards:fix" - exit 1 - else - echo "The code is TYPO3 Coding Standards compliant! Great job!" - fi + echo "The code is TYPO3 Coding Standards compliant! Great job!" fi echo -e "\n\n" echo "Run XML Lint" -if ! xmllint --version > /dev/null 2>&1; then - echo "XML Lint not found, skipping XML linting." -else - echo -e "\n\n" - echo "Check syntax of XML files" - if ! composer lint:xlf - then - echo "Some XML files are not valid" - echo "Please fix the files listed above" - exit 1 - fi +if ! composer tests:lint-xml +then + EXIT_CODE=4 fi +echo -e "\n\n" +echo "Run PHPStan analysis" +if ! composer tests:phpstan +then + # disable fail temporary + #EXIT_CODE=7 + echo "Error during running the PHPStan analysis, please check and fix them." + echo "Tip for working on them: " + echo " TYPO3_VERSION=\"${TYPO3_VERSION}\" composer tests:setup && composer tests:phpstan" +fi echo -e "\n\n" echo "Run unit tests" -UNIT_BOOTSTRAP="Build/Test/UnitTestsBootstrap.php" -if ! .Build/bin/phpunit --colors -c Build/Test/UnitTests.xml --bootstrap=$UNIT_BOOTSTRAP --coverage-clover=coverage.unit.clover +if ! composer tests:unit -- --coverage-clover=coverage.unit.clover then echo "Error during running the unit tests please check and fix them" - exit 1 + EXIT_CODE=5 fi # @@ -109,9 +96,10 @@ fi echo -e "\n\n" echo "Run integration tests" -INTEGRATION_BOOTSTRAP="Build/Test/IntegrationTestsBootstrap.php" -if ! .Build/bin/phpunit --colors -c Build/Test/IntegrationTests.xml --bootstrap=$INTEGRATION_BOOTSTRAP --coverage-clover=coverage.integration.clover +if ! composer tests:integration -- --coverage-clover=coverage.integration.clover then echo "Error during running the integration tests please check and fix them" - exit 1 + EXIT_CODE=6 fi + +exit $EXIT_CODE diff --git a/Build/Test/phpstan-constants.php b/Build/Test/phpstan-constants.php new file mode 100644 index 0000000000..1df075f0fc --- /dev/null +++ b/Build/Test/phpstan-constants.php @@ -0,0 +1,5 @@ +&2 echo \"Can not proceed, because env var TYPO3_VERSION is not set\"; exit 1; else echo \"Setup test environment for TYPO3 ${TYPO3_VERSION}\"; fi", "if echo $TYPO3_VERSION | grep -q \"dev\"; then $COMPOSER_BINARY config minimum-stability dev; fi" ], + "tests:setup:global-require": [ + "echo \"Install global composer dependencies.\"", + "@composer global require sclable/xml-lint", + "@composer global require scrutinizer/ocular" + ], "tests:setup": [ + "# @todo: Abort if composer.json has not staged/commited changes on composer.json, to avoid losing necessery changes.", "@tests:env", - "@composer req --prefer-source --update-with-all-dependencies typo3/cms-core:${TYPO3_VERSION}", + "@composer req --update-with-all-dependencies typo3/cms-core:${TYPO3_VERSION}", "@tests:restore-git" ], "tests:unit": [ - "phpunit --colors --config=Build/Test/UnitTests.xml --bootstrap=Build/Test/UnitTestsBootstrap.php" + "phpunit --config=Build/Test/UnitTests.xml" ], "tests:integration": [ - "phpunit --colors --config=Build/Test/IntegrationTests.xml --bootstrap=.Build/Web/typo3conf/ext/solr/Build/Test/IntegrationTestsBootstrap.php" + "Composer\\Config::disableProcessTimeout", + "phpunit --config=Build/Test/IntegrationTests.xml" + ], + "tests:phpstan": [ + "phpstan analyze -c Build/Test/phpstan.neon" ], "t3:standards:fix": [ "php-cs-fixer fix" ], - "lint:xlf": [ - "xmllint Resources/Private/Language/ -p '*.xlf'" + "tests:lint-xml": [ + "echo \"Run XML Lint\"", + "if ! $($COMPOSER_BINARY config home)/vendor/bin/xmllint Resources/Private/Language/ --pattern='*.xlf' --exclude=.Build/*; then echo \"Some XML files are not valid\" && echo \"Please fix the files listed above\" && export EXIT_CODE=4; else echo \"No syntax errors! Great job!\"; fi" ] }, "extra": { @@ -107,7 +147,6 @@ }, "typo3/cms": { "extension-key": "solr", - "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/Web" }, "TYPO3-Solr": {