forked from aptos-labs/aptos-core
-
Notifications
You must be signed in to change notification settings - Fork 0
285 lines (257 loc) · 13.7 KB
/
forge-stable.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
# Continuously run stable forge tests against the latest main branch.
name: Continuous Forge Tests - Stable
permissions:
issues: write
pull-requests: write
contents: read
id-token: write
actions: write #required for workflow cancellation via check-aptos-core
on:
# Allow triggering manually
workflow_dispatch:
inputs:
IMAGE_TAG:
required: false
type: string
description: The docker image tag to test. This may be a git SHA1, or a tag like "<branch>_<git SHA1>". If not specified, Forge will find the latest build based on the git history (starting from GIT_SHA input)
GIT_SHA:
required: false
type: string
description: The git SHA1 to checkout. This affects the Forge test runner that is used. If not specified, the latest main will be used
schedule:
- cron: "0 6 * * *" # The main branch cadence. This runs every day at 6am UTC.
pull_request:
paths:
- ".github/workflows/forge-stable.yaml"
- "testsuite/find_latest_image.py"
push:
branches:
- aptos-release-v* # the aptos release branches
env:
AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
IMAGE_TAG: ${{ inputs.IMAGE_TAG }} # this is only used for workflow_dispatch, otherwise defaults to empty
AWS_REGION: us-west-2
jobs:
# This job determines the image tag and branch to test, and passes them to the other jobs
# NOTE: this may be better as a separate workflow as the logic is quite complex but generalizable
determine-test-metadata:
runs-on: ubuntu-latest
outputs:
IMAGE_TAG: ${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
BRANCH: ${{ steps.determine-test-branch.outputs.BRANCH }}
steps:
- uses: actions/checkout@v3
- name: Determine branch based on cadence
id: determine-test-branch
run: |
if [[ "${{ github.event_name }}" == "schedule" ]]; then
if [[ "${{ github.event.schedule }}" == "0 6 * * *" ]]; then
echo "Branch: main"
echo "BRANCH=main" >> $GITHUB_OUTPUT
else
echo "Unknown schedule: ${{ github.event.schedule }}"
exit 1
fi
elif [[ "${{ github.event_name }}" == "push" ]]; then
echo "Branch: ${{ github.ref_name }}"
echo "BRANCH=${{ github.ref_name }}" >> $GITHUB_OUTPUT
else
echo "Using GIT_SHA"
# on workflow_dispatch, this will simply use the inputs.GIT_SHA given (or the default)
# on pull_request, this will default to null and the following "checkout" step will use the PR's base branch
echo "BRANCH=${{ inputs.GIT_SHA }}" >> $GITHUB_OUTPUT
fi
- uses: aptos-labs/aptos-core/.github/actions/check-aptos-core@main
with:
cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork
# actions/get-latest-docker-image-tag requires docker utilities and having authenticated to internal docker image registries
- uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
id: docker-setup
with:
GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
EXPORT_GCP_PROJECT_VARIABLES: "false"
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }}
GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
- uses: aptos-labs/aptos-core/.github/actions/get-latest-docker-image-tag@main
id: get-docker-image-tag
with:
branch: ${{ steps.determine-test-branch.outputs.BRANCH }}
variants: "failpoints performance"
- name: Write summary
run: |
IMAGE_TAG=${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
BRANCH=${{ steps.determine-test-branch.outputs.BRANCH }}
if [ -n "${BRANCH}" ]; then
echo "BRANCH: [${BRANCH}](https://github.com/${{ github.repository }}/tree/${BRANCH})" >> $GITHUB_STEP_SUMMARY
fi
echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
### Real-world-network tests.
run-forge-realistic-env-max-load-long:
if: ${{ github.event_name != 'pull_request' }}
needs: determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-max-load-long-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: realistic_env_max_load_large
POST_TO_SLACK: true
run-forge-realistic-env-load-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-max-load-long] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-load-sweep-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1500 # Run for 25 minutes (5 tests, each for 300 seconds)
FORGE_TEST_SUITE: realistic_env_load_sweep
POST_TO_SLACK: true
run-forge-realistic-env-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-load-sweep] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-workload-sweep-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1600 # Run for 26 minutes (4 tests, each for 400 seconds)
FORGE_TEST_SUITE: realistic_env_workload_sweep
POST_TO_SLACK: true
run-forge-realistic-env-graceful-overload:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-workload-sweep] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-graceful-overload-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: realistic_env_graceful_overload
POST_TO_SLACK: true
run-forge-realistic-network-tuned-for-throughput:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-graceful-overload] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-network-tuned-for-throughput-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
FORGE_ENABLE_PERFORMANCE: true
POST_TO_SLACK: true
### Forge Correctness/Componenet/Stress tests
run-forge-consensus-stress-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-network-tuned-for-throughput] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-consensus-stress-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 2400 # Run for 40 minutes
FORGE_TEST_SUITE: consensus_stress_test
POST_TO_SLACK: true
run-forge-workload-mix-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-consensus-stress-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-workload-mix-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: workload_mix
POST_TO_SLACK: true
run-forge-single-vfn-perf:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-workload-mix-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-continuous-e2e-single-vfn-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 480 # Run for 8 minutes
FORGE_TEST_SUITE: single_vfn_perf
POST_TO_SLACK: true
run-forge-haproxy:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-single-vfn-perf] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-haproxy-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 600 # Run for 10 minutes
FORGE_ENABLE_HAPROXY: true
FORGE_TEST_SUITE: realistic_env_max_load
POST_TO_SLACK: true
run-forge-fullnode-reboot-stress-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-haproxy] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-fullnode-reboot-stress-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes
FORGE_TEST_SUITE: fullnode_reboot_stress_test
POST_TO_SLACK: true
### Compatibility Forge tests
run-forge-compat:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-fullnode-reboot-stress-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
FORGE_NAMESPACE: forge-compat-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 300 # Run for 5 minutes
# This will upgrade from testnet branch to the latest main
FORGE_TEST_SUITE: compat
IMAGE_TAG: testnet
GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
POST_TO_SLACK: true
### Changing working quorum Forge tests
run-forge-changing-working-quorum-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-compat] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-changing-working-quorum-test-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: changing_working_quorum_test
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true
run-forge-changing-working-quorum-test-high-load:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-changing-working-quorum-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-changing-working-quorum-test-high-load-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: changing_working_quorum_test_high_load
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true
# Measures PFN latencies with a constant TPS (with a realistic environment)
run-forge-pfn-const-tps-realistic-env:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-changing-working-quorum-test-high-load] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-pfn-const-tps-with-realistic-env-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
POST_TO_SLACK: true