From 41b32876cc4aebe0bfd5484f1176c30db04fcd85 Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Fri, 26 Apr 2024 02:32:15 -0700 Subject: [PATCH] [Main] Make pipeline ymls work with the SDK repo (#40451) Co-authored-by: Viktor Hofer --- .config/dotnet-tools.json | 12 +++ .devcontainer/vmr/README.md | 55 ++++++++++++ .devcontainer/vmr/devcontainer.json | 25 ++++++ .devcontainer/vmr/init.sh | 30 +++++++ .devcontainer/vmr/synchronize-vmr.sh | 4 + eng/Build.props | 9 ++ eng/install-scancode.sh | 25 ++++++ eng/pipelines/templates/jobs/vmr-build.yml | 6 +- .../templates/jobs/vmr-synchronization.yml | 6 +- .../templates/steps/vmr-pull-updates.yml | 32 +++---- eng/pipelines/vmr-build-pr.yml | 14 +-- src/SourceBuild/README.md | 2 +- .../content/.devcontainer/synchronize-vmr.sh | 2 +- src/SourceBuild/content/README.md | 4 +- src/SourceBuild/content/eng/pipelines/ci.yml | 12 +-- src/SourceBuild/content/eng/pipelines/pr.yml | 6 +- .../pipelines/source-build-sdk-diff-tests.yml | 1 + .../templates/jobs/sdk-diff-tests.yml | 16 ++-- .../pipelines/templates/stages/vmr-scan.yml | 8 +- .../steps/download-pipeline-artifact.yml | 4 +- .../templates/variables/pipelines.yml | 6 +- .../eng/pipelines/vmr-license-scan.yml | 5 +- .../repo-projects/Directory.Build.props | 2 +- .../repo-projects/Directory.Build.targets | 2 +- .../content/repo-projects/dotnet.proj | 2 +- .../content/repo-projects/installer.proj | 88 ------------------- .../content/repo-projects/sdk.proj | 60 ++++++++++++- .../assets/LicenseExclusions.txt | 8 +- src/VirtualMonoRepo/InitializeVMR.proj | 10 +-- .../Tasks/VirtualMonoRepo_Initialize.cs | 6 +- src/VirtualMonoRepo/source-mappings.json | 18 ++-- 31 files changed, 310 insertions(+), 170 deletions(-) create mode 100644 .config/dotnet-tools.json create mode 100644 .devcontainer/vmr/README.md create mode 100644 .devcontainer/vmr/devcontainer.json create mode 100644 .devcontainer/vmr/init.sh create mode 100644 .devcontainer/vmr/synchronize-vmr.sh create mode 100644 eng/Build.props create mode 100755 eng/install-scancode.sh delete mode 100644 src/SourceBuild/content/repo-projects/installer.proj diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 000000000000..034bb9016f12 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.darc": { + "version": "1.1.0-beta.24210.2", + "commands": [ + "darc" + ] + } + } +} diff --git a/.devcontainer/vmr/README.md b/.devcontainer/vmr/README.md new file mode 100644 index 000000000000..92d7858e8fc5 --- /dev/null +++ b/.devcontainer/vmr/README.md @@ -0,0 +1,55 @@ + + +This Codespace can help you debug the source build of .NET. In case you have run this from a +`dotnet/sdk` PR branch, it will contain the VMR (`dotnet/dotnet`) checked out into +`/workspaces/dotnet` with the PR changes pulled into it. You can then attempt to source-build +the VMR which is what the VMR leg in the sdk PR build doing. This build takes about 45 +minutes and, after completion, produces an archived .NET SDK located in +`/workspaces/dotnet/artifacts/assets/Release`. + +## Build the SDK + +To build the VMR, run following: +```bash +cd /workspaces/dotnet +./build.sh --online +``` + +> Please note that, at this time, the build modifies some of the checked-in sources so it might +be preferential to rebuild the Codespace between attempts (or reset the working tree changes). + +For more details, see the instructions at https://github.com/dotnet/dotnet. + +## Synchronize your changes in locally + +When debugging the build, you have two options how to test your changes in this environment. + +### Making changes to the VMR directly + +You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You +can then try to build the VMR and see if the change works for you. + +### Pull changes into the Codespace from your fork + +You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the +fix into a branch; can be in your fork too. Once you have the commit pushed, you can pull this +version of the repository into the Codespace by running: + +``` +/workspaces/synchronize-vmr.sh \ + --repository : \ + --remote : +``` + +You can now proceed building the VMR in the Codespace using instructions above. You can repeat +this process and sync a new commit from your fork. Only note that, at this time, Source-Build +modifies some of the checked-in sources so you'll need to revert the working tree changes +between attempts. diff --git a/.devcontainer/vmr/devcontainer.json b/.devcontainer/vmr/devcontainer.json new file mode 100644 index 000000000000..950e3aaa7afe --- /dev/null +++ b/.devcontainer/vmr/devcontainer.json @@ -0,0 +1,25 @@ +// Container suitable for investigating issues with source build +// Contains the VMR (dotnet/dotnet) with applied changes from the current PR +// The container supports source-building the SDK +{ + "name": "VMR with PR changes", + "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39", + "hostRequirements": { + // A completely source built .NET is >64 GB with all the repos/artifacts + "storage": "128gb" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-dotnettools.csharp" + ] + }, + "codespaces": { + "openFiles": [ + "sdk/.devcontainer/vmr/README.md" + ] + } + }, + "onCreateCommand": "${containerWorkspaceFolder}/sdk/.devcontainer/vmr/init.sh", + "workspaceFolder": "/workspaces" +} diff --git a/.devcontainer/vmr/init.sh b/.devcontainer/vmr/init.sh new file mode 100644 index 000000000000..bab633b97ada --- /dev/null +++ b/.devcontainer/vmr/init.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -ex + +source="${BASH_SOURCE[0]}" +script_root="$( cd -P "$( dirname "$source" )" && pwd )" + +sdk_dir=$(realpath "$script_root/../..") +workspace_dir=$(realpath "$sdk_dir/../") +tmp_dir=$(realpath "$workspace_dir/tmp") +vmr_dir=$(realpath "$workspace_dir/dotnet") + +cp "$sdk_dir/.devcontainer/vmr/synchronize-vmr.sh" "$workspace_dir" + +mkdir -p "$tmp_dir" + +# Codespaces performs a shallow fetch only +git -C "$sdk_dir" fetch --all --unshallow + +# We will try to figure out, which branch is the current (PR) branch based off of +# We need this to figure out, which VMR branch to use +vmr_branch=$(git -C "$sdk_dir" log --pretty=format:'%D' HEAD^ \ + | grep 'origin/' \ + | head -n1 \ + | sed 's@origin/@@' \ + | sed 's@,.*@@') + +"$workspace_dir/synchronize-vmr.sh" --branch "$vmr_branch" --debug + +(cd "$vmr_dir" && ./prep-source-build.sh) diff --git a/.devcontainer/vmr/synchronize-vmr.sh b/.devcontainer/vmr/synchronize-vmr.sh new file mode 100644 index 000000000000..5e9a9c307ff6 --- /dev/null +++ b/.devcontainer/vmr/synchronize-vmr.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +(cd /workspaces/sdk \ + && ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*) diff --git a/eng/Build.props b/eng/Build.props new file mode 100644 index 000000000000..fd1f82f9c459 --- /dev/null +++ b/eng/Build.props @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/eng/install-scancode.sh b/eng/install-scancode.sh new file mode 100755 index 000000000000..049b749c2810 --- /dev/null +++ b/eng/install-scancode.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +# Install instructions: https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html#installation-as-a-library-via-pip + +# See latest release at https://github.com/nexB/scancode-toolkit/releases +SCANCODE_VERSION="32.1.0" + +pyEnvPath="/tmp/scancode-env" +python3 -m venv $pyEnvPath +source $pyEnvPath/bin/activate +pip install scancode-toolkit==$SCANCODE_VERSION +deactivate + +# Setup a script which executes scancode in the virtual environment +cat > /usr/local/bin/scancode << EOF +#!/bin/bash +set -euo pipefail +source $pyEnvPath/bin/activate +scancode "\$@" +deactivate +EOF + +chmod +x /usr/local/bin/scancode diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 68101af79673..daa7b0fd9b5e 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -1,5 +1,5 @@ ### This job builds https://github.com/dotnet/dotnet with given parameters -### If run in an installer PR, new changes are applied to a local copy of the VMR, then it is built and tested +### If run in an sdk PR, new changes are applied to a local copy of the VMR, then it is built and tested parameters: - name: architecture @@ -86,7 +86,7 @@ parameters: type: boolean default: false -#### INSTALLER parameters #### +#### sdk parameters #### - name: isBuiltFromVmr displayName: True when build is running from dotnet/dotnet directly @@ -175,7 +175,7 @@ jobs: parameters: vmrPath: $(vmrPath) vmrBranch: ${{ parameters.vmrBranch }} - targetRef: $(Build.SourceVersion) # Synchronize the current installer commit + targetRef: $(Build.SourceVersion) # Synchronize the current sdk commit - ${{ if parameters.buildFromArchive }}: - script: | diff --git a/eng/pipelines/templates/jobs/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml index ae950fd34a12..feda4e6752cc 100644 --- a/eng/pipelines/templates/jobs/vmr-synchronization.yml +++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml @@ -3,7 +3,7 @@ parameters: - name: targetRef - displayName: Target revision of dotnet/installer to synchronize + displayName: Target revision of dotnet/sdk to synchronize type: string default: $(Build.SourceVersion) @@ -73,7 +73,7 @@ jobs: --github-pat '$(BotAccount-dotnet-bot-repo-PAT)' --verbose displayName: Push changes to dotnet/dotnet (public) - workingDirectory: $(Agent.BuildDirectory)/installer + workingDirectory: $(Agent.BuildDirectory)/sdk # Push internal/release branches to the internal VMR - ${{ if startsWith(parameters.vmrBranch, 'internal/release/') }}: @@ -86,4 +86,4 @@ jobs: --azdev-pat '$(dn-bot-dnceng-build-rw-code-rw)' --verbose displayName: Push changes to dotnet-dotnet (internal) - workingDirectory: $(Agent.BuildDirectory)/installer + workingDirectory: $(Agent.BuildDirectory)/sdk diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml index 4bf96835916b..21fab30961cf 100644 --- a/eng/pipelines/templates/steps/vmr-pull-updates.yml +++ b/eng/pipelines/templates/steps/vmr-pull-updates.yml @@ -8,7 +8,7 @@ parameters: type: string - name: targetRef - displayName: Target revision in dotnet/installer to synchronize + displayName: Target revision in dotnet/sdk to synchronize type: string default: $(Build.SourceVersion) @@ -19,16 +19,16 @@ parameters: steps: - checkout: self - displayName: Clone dotnet/installer - path: installer + displayName: Clone dotnet/sdk + path: sdk # This step is needed so that when we get a detached HEAD / shallow clone, -# we still pull the commit into the temporary installer clone to use it during the sync. +# we still pull the commit into the temporary sdk clone to use it during the sync. - script: | - git branch installer-head + git branch sdk-head git rev-parse HEAD displayName: Label PR commit - workingDirectory: $(Agent.BuildDirectory)/installer + workingDirectory: $(Agent.BuildDirectory)/sdk - script: | git checkout -B ${{ parameters.vmrBranch }} @@ -54,17 +54,17 @@ steps: --tmp $(Agent.TempDirectory) --azdev-pat '$(dn-bot-all-orgs-code-r)' --branch ${{ parameters.vmrBranch }} - --repository "installer:${{ parameters.targetRef }}" + --repository "sdk:${{ parameters.targetRef }}" --recursive - --remote "installer:$(Agent.BuildDirectory)/installer" - --component-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md - --tpn-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt + --remote "sdk:$(Agent.BuildDirectory)/sdk" + --component-template $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/Component.template.md + --tpn-template $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt --debug || (echo "##vso[task.logissue type=error]Failed to synchronize the VMR" && exit 1) displayName: Synchronize dotnet/dotnet (Unix) condition: ne(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/installer + workingDirectory: $(Agent.BuildDirectory)/sdk - script: git config --global diff.astextplain.textconv echo displayName: Disable astextplain in git diff (Windows) @@ -76,12 +76,12 @@ steps: -tmp $(Agent.TempDirectory) ` -azdevPat '$(dn-bot-all-orgs-code-r)' ` -branch ${{ parameters.vmrBranch }} ` - -repository "installer:${{ parameters.targetRef }}" ` + -repository "sdk:${{ parameters.targetRef }}" ` -recursive ` # passing remote fails for some reason, but it is the default anyway - # -remote "installer:$(Agent.BuildDirectory)/installer" - -componentTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md ` - -tpnTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt ` + # -remote "sdk:$(Agent.BuildDirectory)/sdk" + -componentTemplate $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/Component.template.md ` + -tpnTemplate $(Agent.BuildDirectory)/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt ` -debugOutput if ($LASTEXITCODE -ne 0) { @@ -90,7 +90,7 @@ steps: } displayName: Synchronize dotnet/dotnet (Windows) condition: eq(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/installer + workingDirectory: $(Agent.BuildDirectory)/sdk - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - publish: $(Agent.TempDirectory) diff --git a/eng/pipelines/vmr-build-pr.yml b/eng/pipelines/vmr-build-pr.yml index 09c0f949f340..6a1ef42f0563 100644 --- a/eng/pipelines/vmr-build-pr.yml +++ b/eng/pipelines/vmr-build-pr.yml @@ -1,11 +1,13 @@ # This YAML is used by these PR pipelines: # -# - installer-source-build -# https://dev.azure.com/dnceng-public/public/_build?definitionId=233 -# - installer-unified-build -# https://dev.azure.com/dnceng-public/public/_build?definitionId=277 -# - installer-unified-build-full -# https://dev.azure.com/dnceng-public/public/_build?definitionId=279 +# - sdk-source-build +# https://dev.azure.com/dnceng-public/public/_build?definitionId=286 +# - sdk-unified-build +# https://dev.azure.com/dnceng-public/public/_build?definitionId=287 +# - sdk-unified-build-full +# https://dev.azure.com/dnceng-public/public/_build?definitionId=288 +# - dotnet-sdk-source-build-internal +# https://dev.azure.com/dnceng/internal/_build?definitionId=1378 trigger: none pr: diff --git a/src/SourceBuild/README.md b/src/SourceBuild/README.md index 1b946e99077c..d4da3d004b5d 100644 --- a/src/SourceBuild/README.md +++ b/src/SourceBuild/README.md @@ -15,4 +15,4 @@ For more information, see [dotnet/source-build](https://github.com/dotnet/source When making changes to the source build infrastructure, devs would typically make and test the changes in a local clone of [dotnet/dotnet](https://github.com/dotnet/dotnet). Once complete you would copy the changed files here and make a PR. To validate the end to end experience, you -can synchronize the VMR with any changes made here by running [eng/vmr-sync.sh](https://github.com/dotnet/installer/blob/main/eng/vmr-sync.sh). +can synchronize the VMR with any changes made here by running [eng/vmr-sync.sh](https://github.com/dotnet/sdk/blob/main/eng/vmr-sync.sh). diff --git a/src/SourceBuild/content/.devcontainer/synchronize-vmr.sh b/src/SourceBuild/content/.devcontainer/synchronize-vmr.sh index 091ae0e26d8d..ce3a2a74291c 100755 --- a/src/SourceBuild/content/.devcontainer/synchronize-vmr.sh +++ b/src/SourceBuild/content/.devcontainer/synchronize-vmr.sh @@ -1,4 +1,4 @@ #!/bin/bash -(cd /workspaces/dotnet/src/installer \ +(cd /workspaces/dotnet/src/sdk \ && ./eng/vmr-sync.sh --vmr /workspaces/dotnet --tmp /workspaces/tmp $*) diff --git a/src/SourceBuild/content/README.md b/src/SourceBuild/content/README.md index a1082deb026d..ec1c43fd9c56 100644 --- a/src/SourceBuild/content/README.md +++ b/src/SourceBuild/content/README.md @@ -9,7 +9,7 @@ What this means: In the VMR, you can find: - source files of [each product repository](#list-of-components) which are mirrored inside of their respective directories under [`src/`](https://github.com/dotnet/dotnet/tree/main/src), - tooling that enables [building the whole .NET product from source](https://github.com/dotnet/source-build) on Linux platforms, -- small customizations, in the form of [patches](https://github.com/dotnet/dotnet/tree/main/src/installer/src/SourceBuild/patches), applied on top of the original code to make the build possible, +- small customizations, in the form of [patches](https://github.com/dotnet/dotnet/tree/main/src/sdk/src/SourceBuild/patches), applied on top of the original code to make the build possible, - *[in future]* E2E tests for the whole .NET product. Just like the development repositories, the VMR will have a release branch for every feature band (e.g. `release/8.0.1xx`). @@ -173,7 +173,7 @@ Alternatively, you can also provide a manifest file where this information can b Sometimes you want to make a change in a repository and test that change in the VMR. You could of course make the change in the VMR directly (locally, as the VMR is read-only for now) but in case it's already available in your repository, you can synchronize it into the VMR (again locally). -To do this, you can either start a [dotnet/dotnet](https://github.com/dotnet/dotnet) Codespace - you will see instructions right after it starts. Alternatively, you can clone the repository locally and use the [vmr-sync.sh](src/installer/eng/vmr-sync.sh) or [vmr-sync.ps1](src/installer/eng/vmr-sync.ps1) script to pull your changes in. Please refer to the documentation in the script for more details. +To do this, you can either start a [dotnet/dotnet](https://github.com/dotnet/dotnet) Codespace - you will see instructions right after it starts. Alternatively, you can clone the repository locally and use the [vmr-sync.sh](src/sdk/eng/vmr-sync.sh) or [vmr-sync.ps1](src/sdk/eng/vmr-sync.ps1) script to pull your changes in. Please refer to the documentation in the script for more details. ## List of components diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 59ce02278571..95557e9ce552 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -4,30 +4,30 @@ # - dotnet-source-build (public) # https://dev.azure.com/dnceng-public/public/_build?definitionId=240 # - PR: ultralite build -# - CI: release/* only, every batched commit, full build +# - CI: release/* and main-sdk only, every batched commit, full build # - Schedule: main only, full build # # - dotnet-unified-build (public) # https://dev.azure.com/dnceng-public/public/_build?definitionId=278 # - PR: lite build -# - CI: release/* only, every batched commit, full build +# - CI: release/* and main-sdk only, every batched commit, full build # - Schedule: main only, full build # # - dotnet-source-build (internal) # https://dev.azure.com/dnceng/internal/_build?definitionId=1219 # - PR: ultralite build -# - CI: release/* and internal/release/* only, every batched commit, full build +# - CI: release/*, internal/release/*, and main-sdk only, every batched commit, full build # - Schedule: main only, full build # # - dotnet-source-build-lite (internal) # https://dev.azure.com/dnceng/internal/_build?definitionId=1299 -# - PR: release/* and main, lite build, on-demand trigger +# - PR: release/*, main, and main-sdk only, lite build, on-demand trigger # - CI: main only, every batched commit, lite build # # - dotnet-unified-build (internal) # https://dev.azure.com/dnceng/internal/_build?definitionId=1330 # - PR: lite build -# - CI: release/*, internal/release/* and main, every batched commit, full build +# - CI: release/*, internal/release/*, main, and main-sdk, every batched commit, full build variables: # enable source-only build for pipelines that contain the -source-build string @@ -75,7 +75,7 @@ extends: - ${{ if and(ne(variables.isPRTrigger, 'true'), eq(variables['System.TeamProject'], 'internal')) }}: - template: /eng/pipelines/templates/stages/vmr-scan.yml@self - - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml@self + - template: /src/sdk/eng/pipelines/templates/stages/vmr-build.yml@self parameters: isBuiltFromVmr: true isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }} diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml index 6553c2a90a89..967f8e0cdcc5 100644 --- a/src/SourceBuild/content/eng/pipelines/pr.yml +++ b/src/SourceBuild/content/eng/pipelines/pr.yml @@ -4,13 +4,13 @@ # - dotnet-source-build (public) # https://dev.azure.com/dnceng-public/public/_build?definitionId=240 # - PR: ultralite build -# - CI: release/* only, every batched commit, full build +# - CI: release/* and main-sdk only, every batched commit, full build # - Schedule: main only, full build # # - dotnet-unified-build (public) # https://dev.azure.com/dnceng-public/public/_build?definitionId=278 # - PR: lite build -# - CI: release/* only, every batched commit, full build +# - CI: release/* and main-sdk only, every batched commit, full build # - Schedule: main only, full build # @@ -34,7 +34,7 @@ variables: - template: /eng/common/templates/variables/pool-providers.yml@self stages: -- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml +- template: /src/sdk/eng/pipelines/templates/stages/vmr-build.yml parameters: isBuiltFromVmr: true isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }} diff --git a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml index 0303a17c0f2b..69f945074b86 100644 --- a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml @@ -4,6 +4,7 @@ schedules: branches: include: - main + - main-sdk # Relies on dotnet-source-build being in the same repo as this pipeline # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/pipeline-triggers?view=azure-devops#branch-considerations diff --git a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml index 8e680cdfbe6c..7fe815613a56 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml @@ -38,17 +38,17 @@ jobs: echo "Dotnet-dotnet build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$dotnet_dotnet_build&view=results" - installer_sha=$(az pipelines build tag list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --build-id $dotnet_dotnet_build --query "[?contains(@, 'installer')]" --output tsv | sed "s,installer ,,g") - installer_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(INSTALLER_OFFICIAL_CI_PIPELINE_ID)' --query "[?sourceVersion == '$installer_sha'].id" --output tsv) - if [[ -z "$installer_build" ]]; then - echo "Could not find a build of installer for commit '$installer_sha'" + sdk_sha=$(az pipelines build tag list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --build-id $dotnet_dotnet_build --query "[?contains(@, 'sdk')]" --output tsv | sed "s,sdk ,,g") + sdk_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(SDK_OFFICIAL_CI_PIPELINE_ID)' --query "[?sourceVersion == '$sdk_sha'].id" --output tsv) + if [[ -z "$sdk_build" ]]; then + echo "Could not find a build of sdk for commit '$sdk_sha'" exit 1 fi - echo "Installer build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$installer_build&view=results" + echo "Sdk build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$sdk_build&view=results" - echo "##vso[build.addbuildtag]installer-$installer_sha" - echo "##vso[task.setvariable variable=InstallerBuildId]$installer_build" + echo "##vso[build.addbuildtag]sdk-$sdk_sha" + echo "##vso[task.setvariable variable=SdkBuildId]$sdk_build" echo "##vso[task.setvariable variable=DotnetDotnetBuildId]$dotnet_dotnet_build" displayName: Find associated builds name: Get_Build_Ids @@ -86,7 +86,7 @@ jobs: msft_sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-$platform-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) if [[ -z "$msft_sdk_tarball_name" ]]; then - echo "Microsoft SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(InstallerBuildId) might have failed." + echo "Microsoft SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build https://dev.azure.com/dnceng/internal/_build/results?buildId=$(SdkBuildId) might have failed." exit 1 fi diff --git a/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml b/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml index ca377cca8f3f..9fc38359b036 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml @@ -18,14 +18,14 @@ stages: InitializeDotNetCli true ./.dotnet/dotnet tool restore displayName: Initialize tooling - workingDirectory: $(Build.SourcesDirectory)/src/installer + workingDirectory: $(Build.SourcesDirectory)/src/sdk - script: | set -e - sha=`./.dotnet/dotnet darc vmr get-version --vmr "$(Build.SourcesDirectory)" installer` + sha=`./.dotnet/dotnet darc vmr get-version --vmr "$(Build.SourcesDirectory)" sdk` echo "##vso[build.addbuildtag]$sha" displayName: Tag the build - workingDirectory: $(Build.SourcesDirectory)/src/installer + workingDirectory: $(Build.SourcesDirectory)/src/sdk - script: | ./eng/detect-binaries.sh @@ -39,5 +39,5 @@ stages: --tmp "$(Agent.TempDirectory)" || (echo '##[error]Found cloaked files in the VMR' && exit 1) displayName: Scan for cloaked files - workingDirectory: $(Build.SourcesDirectory)/src/installer + workingDirectory: $(Build.SourcesDirectory)/src/sdk continueOnError: true \ No newline at end of file diff --git a/src/SourceBuild/content/eng/pipelines/templates/steps/download-pipeline-artifact.yml b/src/SourceBuild/content/eng/pipelines/templates/steps/download-pipeline-artifact.yml index 21df363d7875..e23b61412782 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/steps/download-pipeline-artifact.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/steps/download-pipeline-artifact.yml @@ -1,11 +1,11 @@ parameters: - name: pipeline type: string - default: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID) + default: $(SDK_OFFICIAL_CI_PIPELINE_ID) - name: buildId type: string - default: $(InstallerBuildId) + default: $(SdkBuildId) - name: artifact type: string diff --git a/src/SourceBuild/content/eng/pipelines/templates/variables/pipelines.yml b/src/SourceBuild/content/eng/pipelines/templates/variables/pipelines.yml index 2db47a1beff7..c4e1b06601fb 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/variables/pipelines.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/variables/pipelines.yml @@ -3,9 +3,7 @@ variables: value: internal - name: AZDO_ORG value: https://dev.azure.com/dnceng/ -- name: INSTALLER_OFFICIAL_CI_PIPELINE_ID - value: 286 -- name: INSTALLER_TARBALL_BUILD_CI_PIPELINE_ID - value: 1011 +- name: SDK_OFFICIAL_CI_PIPELINE_ID + value: 140 - name: DOTNET_DOTNET_CI_PIPELINE_ID value: 1219 \ No newline at end of file diff --git a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml index d73cceecff3e..8a477aaa4e40 100644 --- a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml @@ -6,6 +6,7 @@ schedules: branches: include: - main + - main-sdk - release/*.0.1xx* - internal/release/*.0.1xx* @@ -20,7 +21,7 @@ parameters: default: " " # Set it to an empty string to allow it be an optional parameter variables: - installerRoot: '$(Build.SourcesDirectory)/src/installer' + sdkRoot: '$(Build.SourcesDirectory)/src/sdk' jobs: - job: Setup @@ -80,7 +81,7 @@ jobs: artifactFeeds: public/dotnet-public-pypi onlyAddExtraIndex: false - - script: $(installerRoot)/eng/install-scancode.sh + - script: $(sdkRoot)/eng/install-scancode.sh displayName: Install Scancode - script: > diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index 06df3623a2b3..559a9f7e9ee9 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -230,7 +230,7 @@ - + diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 8e4e9e2cd488..07637c787008 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -505,7 +505,7 @@ + See https://github.com/dotnet/source-build/issues/4234 --> - + diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj deleted file mode 100644 index 39098c28df1e..000000000000 --- a/src/SourceBuild/content/repo-projects/installer.proj +++ /dev/null @@ -1,88 +0,0 @@ - - - - - $(ProjectDirectory)build$(ShellExtension) - - - $(FlagParameterPrefix)pack $(FlagParameterPrefix)publish - - - $(BuildArgs) /p:NETCoreAppMaximumVersion=99.9 - - $(BuildArgs) /p:OSName=$(TargetRid.Substring(0, $(TargetRid.IndexOf("-")))) - $(BuildArgs) /p:PortableOSName=$(__PortableTargetOS) - $(BuildArgs) /p:Rid=$(TargetRid) - $(BuildArgs) /p:Architecture=$(TargetArchitecture) - $(BuildArgs) /p:DOTNET_INSTALL_DIR=$(DotNetRoot) - - $(BuildArgs) /p:AspNetCoreInstallerRid=$(TargetRid) - - $(BuildArgs) /p:PortableBuild=true - $(BuildArgs) /p:NetRuntimeRid=$(TargetRid) - - $(BuildArgs) /p:SkipBuildingInstallers=true - - $(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(ArtifactsAssetsDir) - - $(BuildArgs) /p:FallbackPublicBaseURL=https://dotnetbuilds.blob.core.windows.net/public/ - $(BuildArgs) /p:UsePortableLinuxSharedFramework=false - - $(BuildArgs) /p:PgoInstrument=true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/SourceBuild/content/repo-projects/sdk.proj b/src/SourceBuild/content/repo-projects/sdk.proj index f6579a28b578..aaced2442703 100644 --- a/src/SourceBuild/content/repo-projects/sdk.proj +++ b/src/SourceBuild/content/repo-projects/sdk.proj @@ -10,12 +10,41 @@ $(BuildArgs) /p:NativeAotSupported=false $(BuildArgs) $(FlagParameterPrefix)v $(LogVerbosity) + + + $(BuildArgs) /p:NETCoreAppMaximumVersion=99.9 + + + $(BuildArgs) /p:OSName=$(TargetRid.Substring(0, $(TargetRid.IndexOf("-")))) + $(BuildArgs) /p:PortableOSName=$(__PortableTargetOS) + $(BuildArgs) /p:Rid=$(TargetRid) + $(BuildArgs) /p:Architecture=$(TargetArchitecture) + $(BuildArgs) /p:DOTNET_INSTALL_DIR=$(DotNetRoot) + + $(BuildArgs) /p:AspNetCoreInstallerRid=$(TargetRid) + + $(BuildArgs) /p:PortableBuild=true + $(BuildArgs) /p:NetRuntimeRid=$(TargetRid) + + $(BuildArgs) /p:SkipBuildingInstallers=true + + $(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(ArtifactsAssetsDir) + + $(BuildArgs) /p:FallbackPublicBaseURL=https://dotnetbuilds.blob.core.windows.net/public/ + $(BuildArgs) /p:UsePortableLinuxSharedFramework=false + + $(BuildArgs) /p:PgoInstrument=true + - @@ -23,13 +52,15 @@ - + + + @@ -38,4 +69,29 @@ + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt index 574e9754e0bd..7da23e6d685b 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt @@ -3,7 +3,7 @@ # This list is processed using FileSystemName.MatchesSimpleExpression # # Format: -# Exclude the file entirely from license scanning: +# Exclude the file entirely from license scanning: # # Exclude a specific detected license expression from the scan results for the file: # |[,...] @@ -207,6 +207,12 @@ src/runtime/src/mono/sample/wasm/browser-webpack/package-lock.json # False positive src/sdk/THIRD-PARTY-NOTICES.TXT|unknown-license-reference +src/sdk/src/Installer/core-sdk-tasks/BuildFPMToolPreReqs.cs|json +src/sdk/src/Installer/redist-installer/packaging/osx/resources/en.lproj/welcome.html|cecill-c + +# Configuration, doesn't apply to source directly +src/sdk/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt +src/sdk/src/Installer/redist-installer/packaging/**/*.json # # source-build-externals diff --git a/src/VirtualMonoRepo/InitializeVMR.proj b/src/VirtualMonoRepo/InitializeVMR.proj index e3d98fc3ca96..fa5eb249197e 100644 --- a/src/VirtualMonoRepo/InitializeVMR.proj +++ b/src/VirtualMonoRepo/InitializeVMR.proj @@ -35,7 +35,7 @@ - + @@ -45,19 +45,19 @@ @(RootRepoCommitSha) - + - diff --git a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs index 5ab29423193b..4846e6d53355 100644 --- a/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs +++ b/src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs @@ -40,7 +40,7 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask public string PackageVersion { get; set; } - public string InstallerPath { get; set; } + public string SdkPath { get; set; } public string ComponentTemplatePath { get; set; } @@ -60,9 +60,9 @@ private async Task ExecuteAsync() VmrPath = Path.GetFullPath(VmrPath); TmpPath = Path.GetFullPath(TmpPath); - var additionalRemotes = InstallerPath == null + var additionalRemotes = SdkPath == null ? Array.Empty() - : new[] { new AdditionalRemote("installer", InstallerPath) }; + : new[] { new AdditionalRemote("sdk", SdkPath) }; var vmrInitializer = _serviceProvider.Value.GetRequiredService(); await vmrInitializer.InitializeRepository( diff --git a/src/VirtualMonoRepo/source-mappings.json b/src/VirtualMonoRepo/source-mappings.json index 32bf4bc285d4..bb12a88c7f97 100644 --- a/src/VirtualMonoRepo/source-mappings.json +++ b/src/VirtualMonoRepo/source-mappings.json @@ -10,23 +10,23 @@ { // Location within the VMR where the source-build patches are stored // These patches are applied on top of the code synchronized into the VMR - "patchesPath": "src/installer/src/SourceBuild/patches", + "patchesPath": "src/sdk/src/SourceBuild/patches", - "sourceMappingsPath": "src/installer/src/VirtualMonoRepo/source-mappings.json", + "sourceMappingsPath": "src/sdk/src/VirtualMonoRepo/source-mappings.json", // Some files are copied outside of the src/ directory into other locations // When files in the source paths are changed, they are automatically synchronized too "additionalMappings": [ { - "source": "src/installer/src/SourceBuild/content", + "source": "src/sdk/src/SourceBuild/content", "destination": "" }, { - "source": "src/installer/eng/common", + "source": "src/sdk/eng/common", "destination": "eng/common" }, { - "source": "src/installer/src/VirtualMonoRepo/source-mappings.json", + "source": "src/sdk/src/VirtualMonoRepo/source-mappings.json", "destination": "src" } ], @@ -128,7 +128,11 @@ }, { "name": "sdk", - "defaultRemote": "https://github.com/dotnet/sdk" + "defaultRemote": "https://github.com/dotnet/sdk", + "exclude": [ + // Exclude the VMR content as this is copied to the root of VMR via the synchronization process. + "src/SourceBuild/content/**/*" + ] }, { "name": "source-build-externals", @@ -184,7 +188,7 @@ // TODO: Remove the xliff-tasks mapping once the synchronization flags it as unused // We no longer synchronize it but we can't remove it yet until // it disappears from all of the Version.Details.xml files. - // https://github.com/dotnet/installer/issues/17661 + // https://github.com/dotnet/source-build/issues/4359 "name": "xliff-tasks", "defaultRemote": "https://github.com/dotnet/xliff-tasks", "ignoreDefaults": true,