-
Notifications
You must be signed in to change notification settings - Fork 1
173 lines (148 loc) · 6.88 KB
/
adp.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: adp
on:
schedule:
- cron: "0 * * * *"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
jobs:
process-reference-files:
runs-on: ubuntu-latest
steps:
# https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions
- name: Extract Branch Name and Configure Runner
shell: bash
run: |
if [ "${GITHUB_REF##*/}" = "adp_test" ]; then
echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV";
echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV";
echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV";
echo "A_API_KEY=${{ secrets.ADP_TEST_API_KEY }}" >> "$GITHUB_ENV";
echo "CVE-API-ORG=secretariat-reference" >> "$GITHUB_ENV";
echo "CVE-API-USER=automated-ingest" >> "$GITHUB_ENV";
echo "SERVICES_URL=https://cveawg-adp-test.mitre.org/api/cve/" >> "$GITHUB_ENV";
elif [ "${GITHUB_REF##*/}" = "main" ]; then
echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV";
echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV";
echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV";
echo "A_API_KEY=${{ secrets.MAIN_API_KEY }}" >> "$GITHUB_ENV";
echo "CVE-API-ORG=CVE" >> "$GITHUB_ENV";
echo "CVE-API-USER=reference-ingest" >> "$GITHUB_ENV";
echo "SERVICES_URL=https://cveawg.mitre.org/api/cve/" >> "$GITHUB_ENV";
fi
- name: check_out_contrib1_repository
uses: actions/checkout@v4
with:
fetch-depth: 0
repository: "CVEProject/cve-reference-ingest-data"
path: "data-contrib1"
- name: process_new_files_from_contrib1
run: |
CONTRIBUTOR="contrib1"
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest-data"
CHECKOUT_PATH="data-$CONTRIBUTOR"
DIRECTORY="references"
cd $CHECKOUT_PATH
curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/create-file-via-api.py > create-file-via-api.py
curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/read-file-via-api.py > read-file-via-api.py
if ! test -e create-file-via-api.py; then
echo "Failed to copy create-file-via-api.py"
exit 1
fi
if ! test -e read-file-via-api.py; then
echo "Failed to copy read-file-via-api.py"
exit 1
fi
CURRENT_SHA=$(git rev-parse HEAD)
PREVIOUS_SHA=$(python read-file-via-api.py $DIR_IN_PRIMARY_REPO last_run_sha)
if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then
echo "PREVIOUS_SHA is valid"
else
echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}, ${PREVIOUS_SHA}"
exit 1
fi
if [ -n "$PREVIOUS_SHA" ]; then
git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do
echo "Processing: $REFERENCE_FILE"
if ! python create-file-via-api.py $REFERENCE_FILE $CONTRIBUTOR; then
echo "Unable To process ${REFERENCE_FILE}"
exit 1
fi
done
fi
- name: check_out_primary_repository
uses: actions/checkout@v4
with:
ref: ${{ env.BRANCH_NAME }}
fetch-depth: 0
repository: "CVEProject/cve-reference-ingest"
path: "data-primary"
- name: process_new_files_from_primary
run: |
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest"
CHECKOUT_PATH="data-primary"
DIRECTORY="references"
pwd
PRIMARY_LAST_RUN_SHA="last_run_shas/$DIR_IN_PRIMARY_REPO/last_run_sha"
cd $CHECKOUT_PATH
CURRENT_SHA=$(git rev-parse HEAD)
PREVIOUS_SHA=$(cat $PRIMARY_LAST_RUN_SHA)
if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then
echo "PREVIOUS_SHA is valid"
else
echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}"
exit 1
fi
git config user.name "GitHub Action"
git config user.email "[email protected]"
git pull
if [ -n "$PREVIOUS_SHA" ]; then
git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do
echo "Processing: $REFERENCE_FILE"
if ! python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $REFERENCE_FILE; then
cp $REFERENCE_FILE retry/$REFERENCE_FILE
git add retry/$REFERENCE_FILE
git commit -m "Adding failed file to retry folder"
git push
if [ $? -ne 0 ]; then
# The push to the retry folder failed, log the information so it can be recovered from the logs
echo "git push failed for File: ${REFERENCE_FILE}"
fi
fi
sleep 0.7
done
fi
echo $CURRENT_SHA > $PRIMARY_LAST_RUN_SHA
git add $PRIMARY_LAST_RUN_SHA
THIS_CONTRIBUTOR="contrib1"
THIS_CONTRIBUTOR_REPO="CVEProject/cve-reference-ingest-data"
CURRENT_SHA_OF_CONTRIBUTOR=$(cd ../data-$THIS_CONTRIBUTOR; git rev-parse HEAD)
CONTRIBUTOR_LAST_RUN_SHA="last_run_shas/$THIS_CONTRIBUTOR_REPO/last_run_sha"
echo $CURRENT_SHA_OF_CONTRIBUTOR > $CONTRIBUTOR_LAST_RUN_SHA
git add $CONTRIBUTOR_LAST_RUN_SHA
if [ -s .rbp ]; then
RBP_BASENAME=$(head -1 .rbp)
RBP_PATHNAME=rbp/$RBP_BASENAME
git add $RBP_PATHNAME
fi
git commit -m "Update last run SHA for primary"
git push
- name: attempt_to_process_retry
if: always()
run: |
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest"
CHECKOUT_PATH="data-primary"
DIRECTORY="retry"
cd $CHECKOUT_PATH
git config user.name "GitHub Action"
git config user.email "[email protected]"
for file in retry/references/*; do
if [ -f "$file" ]; then
echo "Processing: $file"
if python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $file; then
git rm $file
git commit -m "Completed Retry of File: ${file}"
git push
fi
fi
sleep 0.7
done