Merge pull request #1 from clarin-eric/master

Update to latest changes
clarin-eric · Jan 5, 2023 · e44cace · e44cace
2 parents 76a7451 + 9771124
commit e44cace
Show file tree

Hide file tree

Showing 82 changed files with 11,199 additions and 10,179 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 .idea/
+gh-pages/
diff --git a/.travis.yml b/.travis.yml
@@ -1,18 +1,22 @@
+os: 
+  - linux
+language: shell
+addons:
+  apt:
+    packages:
+      - libxml2-utils
+      - ant
 branches:
   only:
   - master
-  - production
-sudo: false
-script:
-- sh -x check-saml-metadata/check_saml_metadata.sh clarin-sp-metadata.xml
 env:
   global:
   - ENCRYPTION_LABEL: f641f006de37
-  - COMMIT_AUTHOR_EMAIL: "[email protected]"
+script:
+- bash check-saml-metadata/check_saml_metadata.sh metadata/
 after_success:
-- sudo apt-get install -y ant
 - bash CI-assets/deploy.sh
 notifications:
   email:
     on_success: change
-on_failure: always
+    on_failure: always
diff --git a/CI-assets/compile.sh b/CI-assets/compile.sh
@@ -1,15 +1,21 @@
 #!/bin/bash
 set -e # Exit with nonzero exit code if anything fails
-QA_VALIDATOR_VERSION=1.0.5
+
+QA_VALIDATOR_VERSION=1.0.9
 SAXON_VERSION=SaxonHE9-9-1-5J
-SAXON_URL=https://netcologne.dl.sourceforge.net/project/saxon/Saxon-HE/9.9/$SAXON_VERSION.zip
+SAXON_URL=https://downloads.sourceforge.net/project/saxon/Saxon-HE/9.9/$SAXON_VERSION.zip
 SCHEMATRON_VERSION=1.0.1-e16ecc4-CLARIN
 INSTALLS_PATH=qa-tmp
+CHANGED_SPS=()
 
+sed_cmd="sed"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    sed_cmd="gsed"
+fi
 
 mkdir -p $INSTALLS_PATH/saxon
 cd $INSTALLS_PATH/saxon
-wget $SAXON_URL
+wget --no-check-certificate $SAXON_URL
 unzip -o $SAXON_VERSION.zip
 rm $SAXON_VERSION.zip
 cd ..
@@ -21,12 +27,47 @@ rm -rf schematron-$SCHEMATRON_VERSION schematron.tar.gz
 wget -O SAML_metadata_QA_validator.tar.gz https://codeload.github.com/clarin-eric/SAML_metadata_QA_validator/tar.gz/$QA_VALIDATOR_VERSION
 tar xvf SAML_metadata_QA_validator.tar.gz
 cd SAML_metadata_QA_validator-$QA_VALIDATOR_VERSION
-ant -v
-rm out/SAML_metadata_QA_validator.concrete.sch out/SAML_metadata_QA_validator.xsl out/.gitignore
-if [ ! -d "../../out/" ]; then
-	mkdir ../../out/
+md_files=../../metadata/*.xml
+for file in ${md_files}
+do
+    ant -v -DinputFile="file:$(realpath ${file})"
+done
+number_of_files=$(ls ${md_files} 2> /dev/null | wc -l)
+if [ ${number_of_files} -gt 1 ]; then
+    echo "Generating and testing aggregated output..."
+    ((xmllint -xpath "/*[local-name()='EntitiesDescriptor' and namespace-uri()='urn:oasis:names:tc:SAML:2.0:metadata']" ../../CI-assets/feed_wrapper.xml  | \
+        head -1; xmllint -xpath "/*[local-name()='EntityDescriptor' and namespace-uri()='urn:oasis:names:tc:SAML:2.0:metadata']" ../../metadata/*;tail -1 ../../CI-assets/feed_wrapper.xml) | \
+        xmllint --nsclean --format -) > ../../$TARGET_BRANCH/aggregated_feed_${SOURCE_BRANCH}.xml
+    ant -v -DinputFile="file:$(realpath ../../$TARGET_BRANCH/aggregated_feed_${SOURCE_BRANCH}.xml)"
 fi
-mv out/* ../../out/
-cd ../.. && rm -rf $INSTALLS_PATH
-
+rm -rf out/*.sch out/*.xsl out/.gitignore
+if [ ! -d "../../$TARGET_BRANCH/reports/" ]; then
+    mkdir ../../$TARGET_BRANCH/reports/
+fi
+for report in out/*results.xml
+do
+    set +e
+    xmllint --format --xpath "//results" $(realpath ../../$TARGET_BRANCH/reports/$(basename ${report})) > previous.xml
+    xmllint --format --xpath "//results" ${report} > current.xml
+    if ! diff -q current.xml previous.xml; then
+        set -e
+        REPORT_NAME=$(basename ${report})
+        echo "Report ${REPORT_NAME} has changed"
 
+        ${sed_cmd} -i "2i <report>" ${report}
+        ${sed_cmd} -i "3i <ReportDate>$(date)</ReportDate>" ${report}
+        ${sed_cmd} -i "4i <FromCommit>${SHA}</FromCommit>" ${report}
+        ${sed_cmd} -i "\$a</report>" ${report}
+        xmllint --output tmp.xml --format ${report}
+        mv tmp.xml ${report}
+
+        filename_wo_ext="${report%_results.xml}"
+        mv ${report} ../../$TARGET_BRANCH/reports/
+        CHANGED_SPS+=( ${REPORT_NAME} )
+    else
+        set -e
+        echo "Report $(basename ${report}) is unhanged. Leaving previous version in place."
+    fi
+    rm previous.xml current.xml
+done
+cd ../.. && rm -rf $INSTALLS_PATH
diff --git a/CI-assets/deploy.sh b/CI-assets/deploy.sh
@@ -3,65 +3,91 @@ set -e # Exit with nonzero exit code if anything fails
 
 SOURCE_BRANCH="master"
 TARGET_BRANCH="gh-pages"
+COMMIT_AUTHOR_EMAIL="[email protected]"
 
 function doCompile {
-  CI-assets/compile.sh
+  . CI-assets/compile.sh
 }
 
 # Pull requests and commits to other branches shouldn't try to deploy, just build to verify
 if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
-    echo "Skipping QA report build."
+    echo "Not ${SOURCE_BRANCH}. Skipping QA report build."
     exit 0
 fi
 
 # Save some useful information
-REPO=`git config remote.origin.url`
+REPO=$(git config remote.origin.url)
 SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:}
-SHA=`git rev-parse --verify HEAD`
+SHA=$(git rev-parse --verify HEAD)
+COMMIT_API_INFO=$(curl --max-time 900 --connect-timeout 240 "https://api.github.com/search/issues?q=${SHA}" 2> /dev/null)
+RELEVANT_PR=$(echo ${COMMIT_API_INFO} | jq .items[].number | head -1)
+PR_TARGET_BRANCH=$(echo ${COMMIT_API_INFO} | jq .items[].pull_request.html_url | head -1 | grep ${TRAVIS_REPO_SLUG})
 
 # Clone the existing qa-output for this repo into out/
-# Create a new empty branch if qa-output doesn't exist yet (should only happen on first deply)
-git clone $REPO out
-cd out
+# Create a new empty branch if qa-output doesn't exist yet (should only happen on first deploy)
+rm -rf $TARGET_BRANCH
+git clone $REPO $TARGET_BRANCH
+cd $TARGET_BRANCH
 git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH
-cd ..
-
-# Clean out existing output
-rm -rf out/*.xml out/*.svrlt || exit 0
 
 # Run our compile script
+cd ..
 doCompile
 
 # Now let's go have some fun with the cloned repo
-cd out
-git config user.name "Travis CI"
+cd $TARGET_BRANCH
+git config user.name "CLARIN SPF QA bot"
 git config user.email "$COMMIT_AUTHOR_EMAIL"
 
 # If there are no changes to the compiled out (e.g. this is a README update) then just bail.
-if git diff --quiet; then
-    echo "No changes to the output on this push; exiting."
-    exit 0
-fi
-
-# Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc
-ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
-ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv"
-ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
-ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
-
-openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in ../CI-assets/deploy_key.enc -out ../CI-assets/deploy_key -d
-chmod 600 ../CI-assets/deploy_key
-eval `ssh-agent -s`
-ssh-add ../CI-assets/deploy_key
+git add -A .
+if git diff $TARGET_BRANCH --quiet; then
+    echo "No changes to the output on this push. Leaving upstream \"$TARGET_BRANCH\" branch untouched."
+else
+    # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc
+    ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
+    ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv"
+    ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
+    ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
 
-# Clean everything else but the build output
-mv .git ../git_tmp && mv master_sps_qa_report* .travis.yml README.md page ../ && cd .. && rm -rf out && mkdir out && mv master_sps_qa_report* .travis.yml README.md page out && mv git_tmp out/.git && cd out
+    openssl aes-256-cbc -K $ENCRYPTED_KEY -iv $ENCRYPTED_IV -in ../CI-assets/deploy_key.enc -out ../CI-assets/deploy_key -d
+    chmod 600 ../CI-assets/deploy_key
+    eval `ssh-agent -s`
+    ssh-add ../CI-assets/deploy_key
 
+    # Commit the "changes", i.e. the new version.
+    git commit -m "Deploy SAML QA report for: ${SHA}"
 
-# Commit the "changes", i.e. the new version.
-# The delta will show diffs between new and old versions.
-git add -A .
-git commit -m "Deploy SAML QA report for: ${SHA}"
+    # Now that we're all set up, we can push.
+    git push $SSH_REPO $TARGET_BRANCH
+fi
 
-# Now that we're all set up, we can push.
-git push $SSH_REPO $TARGET_BRANCH
+# Comment pull request
+if [ ! -z "${RELEVANT_PR}" -a ! -z "${PR_TARGET_BRANCH}" -a "${TRAVIS_EVENT_TYPE}" != "cron" ]; then
+    echo "Commenting pull request..."
+    if [ ${#CHANGED_SPS[@]} -gt 0 ]; then
+        CHANGED_SPS_HTML="<p>The following SPs changed their QA assessment with this pull request:</p><ul>Standalone QA reports:"
+        for report in ${CHANGED_SPS[@]}
+        do
+        	# do not generate entry for aggregated report (it is always present in curl message body. See bellow)
+            if [ "${report}" != "aggregated_feed_master_sps_qa_report_results.xml" ]; then
+                CHANGED_SPS_HTML+="<li><a href=https://clarin-eric.github.io/SPF-SPs-metadata/web/sp_qa_report.html?${report}>${report%_sps_qa_report_results.xml}</a></li>"
+            fi
+        done
+        CHANGED_SPS_HTML+="</ul>"
+    fi
+
+    curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST \
+        -d "{\"body\": \"\
+<img src=https://img.shields.io/github/status/contexts/pulls/${TRAVIS_REPO_SLUG}/${RELEVANT_PR}></img> \
+<img src=https://img.shields.io/github/commit-status/${TRAVIS_REPO_SLUG}/${SOURCE_BRANCH}/${SHA}></img> \
+<p>Automated QA assessment complete.</p>\
+<p>Please check your SP in the <a href=https://clarin-eric.github.io/SPF-SPs-metadata/web/master_qa_report.html>master QA report</a> (or in its standalone QA report) \
+and <strong>fix all entries marked in red</strong>. Any entries marked in yellow should also be fixed, though for those we apply some tolerance on a case by case basis.</p>\
+${CHANGED_SPS_HTML} \
+<p>Your SP has successfully passed our automated QA assessment when the master QA report does not include any entries for it.</p> \
+<p>To submit your SAML fixes, either commit them to this pull request or open a new one.</p> \
+\"}" \
+        "https://api.github.com/repos/${TRAVIS_REPO_SLUG}/issues/${RELEVANT_PR}/comments"
+fi
+exit 0
diff --git a/CI-assets/feed_wrapper.xml b/CI-assets/feed_wrapper.xml
@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='utf-8'?>
+<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:mdui="urn:oasis:names:tc:SAML:metadata:ui"
+    xmlns:mdattr="urn:oasis:names:tc:SAML:metadata:attribute"
+    xmlns:mdrpi="urn:oasis:names:tc:SAML:metadata:rpi"
+    xmlns:shibmd="urn:mace:shibboleth:metadata:1.0"
+    xmlns:xrd="http://docs.oasis-open.org/ns/xri/xrd-1.0" xmlns:pyff="http://pyff.io/NS"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" xmlns:xs="http://www.w3.org/2001/XMLSchema"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:ser="http://eidas.europa.eu/metadata/servicelist"
+    xmlns:eidas="http://eidas.europa.eu/saml-extensions"
+    xmlns="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:idpdisc="urn:oasis:names:tc:SAML:profiles:SSO:idp-discovery-protocol"
+    xmlns:init="urn:oasis:names:tc:SAML:profiles:SSO:request-init"
+    xmlns:remd="http://refeds.org/metadata" xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:alg="urn:oasis:names:tc:SAML:metadata:algsupport"
+    xmlns:urn="urn:oasis:names:tc:SAML:2.0:metadata" Name="http://www.clarin.eu/spf"
+    ID="_20200129T172616Z" validUntil="2020-02-08T17:26:16Z" cacheDuration="PT5H">
+
+</md:EntitiesDescriptor>
diff --git a/README.md b/README.md
@@ -1,27 +1,31 @@
-[![Travis Status](https://travis-ci.org/clarin-eric/SPF-SPs-metadata.svg?branch=master)](https://travis-ci.org/clarin-eric/SPF-SPs-metadata)
+[![Travis Status](https://travis-ci.com/clarin-eric/SPF-SPs-metadata.svg?branch=master)](https://travis-ci.com/clarin-eric/SPF-SPs-metadata)
+[![Latest Release](https://img.shields.io/github/v/release/clarin-eric/SPF-SPs-metadata)](https://github.com/clarin-eric/SPF-SPs-metadata/releases/latest)
+[![Commits Since Latest Release](https://img.shields.io/github/commits-since/clarin-eric/SPF-SPs-metadata/latest)](https://github.com/clarin-eric/SPF-SPs-metadata/commits/master)
 # Metadata sources for service providers inside the CLARIN Service Provider Federation
 
 ## Notes for service provider operators
 
+:warning: **The structure of this repository has changed on 10/02/2020, please make sure to update your forks before creating new pull requests. See the [repository system changelog](https://github.com/clarin-eric/SPF-SPs-metadata/wiki/Repository-system-changelog) for a detailed description of the changes.**
+
 To update or add SAML metadata for your SP:
 1. Fork this repository.
-2. Make your changes to the *clarin-sp-metadata.xml* file.
+2. Make your changes in the file corresponding to your SP inside the `metadata/` directory. In case you are adding a new SP, add the metadata by creating a new file following the same naming convention: `[New SP File Name] = [SP entityID].replace("http(s)?://", "").replace("/", "%2F") + ".xml"`
 3. Create a pull request to the *master* branch of this repository.
 4. Wait for Travis CI to finish the XSD validation on your pull request.
-5. Make sure your pull request is XSD valid. Fix your code based on the Travis CI output information and update the pull request until XSD validation passes.
-6. Wait for your pull request to be merged into the *master* branch by a CLARIN SPF operator. This will trigger the generation of a [QA report](https://clarin-eric.github.io/SPF-SPs-metadata/page/master_qa_report.html) by Travis CI.
-7. Wait for Travis CI to finish the generation of the [QA report](https://clarin-eric.github.io/SPF-SPs-metadata/page/master_qa_report.html).
-8. Fix the issues concerning your SP described in the QA report and update your pull request accordingly (alternatively you can also submit a new pull request with your QA fixes).
+5. Make sure your pull request is XSD valid. Fix your metadata based on the Travis CI output information and update the pull request until XSD validation passes.
+6. Wait for your pull request to be merged into the *master* branch by a CLARIN SPF operator. This will trigger the generation of an [aggregated quality assurance (QA) report](https://clarin-eric.github.io/SPF-SPs-metadata/web/master_qa_report.html) by Travis CI.
+7. Wait for Travis CI to finish the generation of the [aggregated QA report](https://clarin-eric.github.io/SPF-SPs-metadata/web/master_qa_report.html). The [CLARIN bot](https://github.com/clarin-bot) will write a comment on the pull request once the generation is complete, linking to the new aggregated QA report and to any individual SP report which has changed due to your pull request. At this point, a CLARIN SPF operator might also request futher changes not covered by the QA automated testing.
+8. In your fork, fix all issues concerning your SP described in the QA report and submit a new pull request.
 
-After a pull request is created *(3.)* the [SAML metadata checker script](https://github.com/clarin-eric/SAML-metadata-checker) will automactically run on the pull request code via Travis CI *(4.)*. The result of this check will be visible on the pull request page. Check the [existing pull resquests](https://github.com/clarin-eric/SPF-SPs-metadata/pulls?utf8=%E2%9C%93&q=is%3Apr) on this repository for examples.
+After a pull request is created *(3.)* the [SAML metadata checker script](https://github.com/clarin-eric/SAML-metadata-checker) will automactically perform an XSD validation on the pull request metadata via Travis CI *(4.)*. The result of this check will be visible on the pull request page. Check the [existing pull resquests](https://github.com/clarin-eric/SPF-SPs-metadata/pulls?utf8=%E2%9C%93&q=is%3Apr) on this repository for examples.
 
-When your pull request successfully passes XSD validation *(5.)*, a CLARIN SPF operator will merge it into the *master* branch of original repository for QA assessment *(6.)*. Note: the SPF operators will only consider for merging pull requests which are XSD valid. If you cannot make your file successfully pass the XSD validation or you believe you are hitting a false positive. Please create an [issue](https://github.com/clarin-eric/SPF-SPs-metadata/issues/new) explaining the problem. 
+When your pull request successfully passes XSD validation *(5.)*, a CLARIN SPF operator will merge it into the *master* branch of original repository for QA assessment *(6.)*.  
+Note that the SPF operators will only consider merging pull requests which are XSD valid. If you cannot make your file successfully pass the XSD validation or you believe you are hitting a false positive. Please create an [issue](https://github.com/clarin-eric/SPF-SPs-metadata/issues/new) explaining the problem. 
 
-After your pull request is merged *(6.)*, Travis CI will automatically analyze the latest *master* version and generate a QA report visible in [this table](https://clarin-eric.github.io/SPF-SPs-metadata/page/master_qa_report.html) *(7.)*.
-Please ascertain that you comply with the [SAML metadata guidelines](https://www.clarin.eu/content/guidelines-saml-metadata-about-your-sp). Mind to check and resolve issues in the SAML metadata quality for your SP after your pull request has been merged into the *master* branch, then update your pull request with any necessary fixes or create a new one *(8.)*. Make sure you always update the SAML metadata template of your SP to make it correspond exactly with the SAML metadata you deposit here (see e.g. https://goo.gl/uysudA).
+After your pull request is merged *(6.)*, Travis CI will automatically analyze the latest *master* version and generate a new QA report visible in [this page](https://clarin-eric.github.io/SPF-SPs-metadata/web/master_qa_report.html) *(7.)*. When this operation has finished, the CLARIN bot](https://github.com/clarin-bot) will write a comment on the pull request linking to the new aggregated QA report and to any individual SP report which changed. Please ascertain that you comply with the [SAML metadata guidelines](https://www.clarin.eu/content/guidelines-saml-metadata-about-your-sp). Mind to check and resolve issues in the SAML metadata quality for your SP after your pull request has been merged into the *master* branch, then update your fork and create a new pull request with any necessary fixes *(8.)*. Make sure you always update the SAML metadata template of your SP to make it correspond exactly with the SAML metadata you deposit here (see e.g. https://goo.gl/uysudA).
 
 If you wish that the registration/modification of the SAML metadata about your SP with identity federations is coordinated extra carefully (say, you perform a key rollover), then please create a new [issue](https://github.com/clarin-eric/SPF-SPs-metadata/issues/new) describing the task. Alternatively you can also head over to https://trac.clarin.eu/newticket and create a ticket for the 'AAI' Trac component (requires a CLARIN 'developer' account).
 
-Finally your metadata will be merged into the *production* branch and picked up by an hourly cron job which automatically checks out the latest version and publishes it at https://infra.clarin.eu/aai/md_about_spf_sps.xml (staging feed) and https://infra.clarin.eu/aai/prod_md_about_spf_sps.xml (production feed). 
+Finally your metadata will be released and picked up by an hourly cron job which automatically downloads the latest available release from this repository and publishes it at https://infra.clarin.eu/aai/md_about_spf_sps.xml (staging feed) and https://infra.clarin.eu/aai/prod_md_about_spf_sps.xml (production feed). 
 
-Note: For an SP to be published in the production feed it must be defined first as a production SP [in our configuration](https://github.com/clarin-eric/pyFF_config/blob/master/job_b.fd). This can only be done by CLARIN SPF operators.
+Note: For an SP to be published in the production feed it must be defined first with production status [in the CLARIN CentreRegistry](https://centres.clarin.eu/spf). This can only be done by CLARIN SPF operators.
diff --git a/check-saml-metadata b/check-saml-metadata