From c800698eb5391b61d8b2435a056bfbf8ede7b3f8 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 5 Aug 2022 20:26:09 -0400 Subject: [PATCH] CI: Detect dependency changes in `eng/Version.Details.xml`, and allow .. conditioning on that. --- .../common/evaluate-changed-darc-deps.yml | 15 +++ eng/pipelines/common/evaluate-paths-job.yml | 7 ++ eng/pipelines/evaluate-changed-darc-deps.sh | 98 +++++++++++++++++++ eng/pipelines/get-changed-darc-deps.py | 62 ++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 eng/pipelines/common/evaluate-changed-darc-deps.yml create mode 100755 eng/pipelines/evaluate-changed-darc-deps.sh create mode 100644 eng/pipelines/get-changed-darc-deps.py diff --git a/eng/pipelines/common/evaluate-changed-darc-deps.yml b/eng/pipelines/common/evaluate-changed-darc-deps.yml new file mode 100644 index 0000000000000..1a816f5876b1a --- /dev/null +++ b/eng/pipelines/common/evaluate-changed-darc-deps.yml @@ -0,0 +1,15 @@ +# This step template evaluates changes in dependencies defined in `eng/Version.Details.xml`. +# For more information on how this works works look at evaluate-changed-darc-deps.sh docs +# at the beginning of that file. + +parameters: + subsetName: '' + # Array containing the arguments that are to be passed down to evaluate-changed-paths.sh + # Note that --azurevariable is always set to the dependency name, no need to pass it down. + arguments: [] + +steps: + - script: eng/pipelines/evaluate-changed-darc-deps.sh + ${{ join(' ', parameters.arguments) }} + displayName: Evaluate eng/Version.Details.xml for dependency changes + name: DarcDependenciesChanged diff --git a/eng/pipelines/common/evaluate-paths-job.yml b/eng/pipelines/common/evaluate-paths-job.yml index 42f8549cbb7f0..71f35bd4f9f4c 100644 --- a/eng/pipelines/common/evaluate-paths-job.yml +++ b/eng/pipelines/common/evaluate-paths-job.yml @@ -50,3 +50,10 @@ jobs: - --includepaths '${{ join('+', path.include) }}' - ${{ if ne(path.exclude[0], '') }}: - --excludepaths '${{ join('+', path.exclude) }}' + + - template: evaluate-changed-darc-deps.yml + parameters: + arguments: + # The commit that we're building is always a merge commit that is merging into the target branch. + # So the first parent of the commit is on the target branch and the second parent is on the source branch. + - --difftarget HEAD^1 diff --git a/eng/pipelines/evaluate-changed-darc-deps.sh b/eng/pipelines/evaluate-changed-darc-deps.sh new file mode 100755 index 0000000000000..b4181a1e50347 --- /dev/null +++ b/eng/pipelines/evaluate-changed-darc-deps.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +: ' + Compares contents of `env/Version.Details.xml` between HEAD and difftarget, and emits variables named for + dependencies that satisfy either of: + 1. version, or sha changed + 2. it is missing from one of the xmls + + The dependency names have `.` replaced with `_`. + + In order to consume these variables in a yaml pipeline, reference them via: $[ dependencies..outputs["."] ] + + Example: + -difftarget ''HEAD^1'' +' + +# Disable globbing in this bash script since we iterate over path patterns +set -f + +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Stop script if command returns non-zero exit code. +# Prevents hidden errors caused by missing error code propagation. +set -e + +usage() +{ + echo "Script that emits an azure devops variable with all the dependencies that changed in 'eng/Version.Details.xml' contained in the current HEAD against the difftarget" + echo " --difftarget SHA or branch to diff against. (i.e: HEAD^1, origin/main, 0f4hd36, etc.)" + echo " --azurevariableprefix Name of azure devops variable to create if change meets filter criteria" + echo "" + + echo "Arguments can also be passed in with a single hyphen." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +eng_root=`cd -P "$scriptroot/.." && pwd` + +azure_variable_prefix='' +diff_target='' + +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -difftarget) + diff_target=$2 + shift + ;; + -azurevariableprefix) + azure_variable_prefix=$2 + shift + ;; + esac + + shift +done + +if [[ -z "$diff_target" ]]; then + echo "Argument -difftarget is required" + usage + exit 1 +fi + +oldXmlPath=`mktemp` + +ci=true # Needed in order to use pipeline-logging-functions.sh +. "$eng_root/common/pipeline-logging-functions.sh" + +git show $diff_target:eng/Version.Details.xml > $oldXmlPath +# FIXME: errors? +changed_deps=$(python3 "$eng_root/pipelines/get-changed-darc-deps.py" $oldXmlPath eng/Version.Details.xml) +rm -f $oldXmlPath + +if [[ -n "$azure_variable_prefix" ]]; then + azure_variable_prefix="${azure_variable_prefix}_" +fi + +for dep in $changed_deps; do + dep=`echo $dep | tr \. _` + var_name=${azure_variable_prefix}${dep} + echo "Setting pipeline variable $var_name=true" + Write-PipelineSetVariable -name $var_name -value true +done diff --git a/eng/pipelines/get-changed-darc-deps.py b/eng/pipelines/get-changed-darc-deps.py new file mode 100644 index 0000000000000..cae52445eaa0a --- /dev/null +++ b/eng/pipelines/get-changed-darc-deps.py @@ -0,0 +1,62 @@ +# +# Emits a comma separated list of dependencies from `eng/Version.Details.xml` +# that changed as compared to another versions file +# +# - we don't really care which is old, and which is new +# - A dependency name is emitted as changed if: +# 1. version, or sha changed +# 2. it is missing from one of the xmls + +import xml.etree.ElementTree as ET +import sys +from os.path import exists + +def getDependencies(xmlfile): + tree = ET.parse(xmlfile) + root = tree.getroot() + deps = {} + for depElement in root.findall('.//Dependency'): + dep = {} + dep['Version'] = depElement.attrib['Version'] + dep['Sha'] = depElement.find('Sha').text + + deps[depElement.attrib['Name']] = dep + + return deps + +def compare(dict1, dict2): + if dict1 is None or dict2 is None: + print('Nones') + return False + + if (not isinstance(dict1, dict)) or (not isinstance(dict2, dict)): + print('Not dict') + return False + + changed_names = [] + all_keys = set(dict1.keys()) | set(dict2.keys()) + for key in all_keys: + if key not in dict1 or key not in dict2: + print(key) + # changed_names.append(key) + elif dict1[key] != dict2[key]: + print(key) + # changed_names.append(key) + + print(','.join(changed_names)) + +if len(sys.argv) != 3: + print(f'Usage: {sys.argv[0]} ') + exit(1) + +if not exists(sys.argv[1]): + print(f'Cannot find {sys.argv[1]}') + exit(1) +if not exists(sys.argv[2]): + print(f'Cannot find {sys.argv[2]}') + exit(1) + +newDeps = getDependencies(sys.argv[1]) +oldDeps = getDependencies(sys.argv[2]) + +compare(oldDeps, newDeps)