-
Notifications
You must be signed in to change notification settings - Fork 17
211 lines (179 loc) · 8.28 KB
/
run-ss3-with-est.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
# Build SS3 and run test models with estimation and hessian
# Only runs manual run or if run-ss3-no-est.yml runs successfully
name: run-ss3-with-est
on:
workflow_dispatch:
# push:
# paths:
# - '**.tpl'
# branches:
# - main
workflow_run:
workflows: ["run-ss3-no-est"]
types:
- completed
# pull_request:
# types: ['opened', 'edited', 'reopened', 'synchronize', 'ready_for_review']
# paths:
# - '**.tpl'
# branches:
# - main
# Run fast running SS3 test models with estimation
jobs:
run-ss3-with-est:
if: ${{ github.event_name == 'workflow_dispatch' || (github.event.pull_request.draft == 'false' && github.event.workflow_run.conclusion == 'success') }}
runs-on: ubuntu-latest
env:
R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"
steps:
- name: Checkout ss3 repo
uses: actions/checkout@v4
- name: Checkout models repo
uses: actions/checkout@v4
with:
repository: 'nmfs-ost/ss3-test-models'
path: test-models-repo
- name: Update Ubuntu packages
run: |
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo apt-get update
sudo apt-get install --only-upgrade libstdc++6
- name: Setup R
uses: r-lib/actions/setup-r@v2
# - name: Get admb and put in path (not using docker), linux
# run: |
# wget https://github.com/admb-project/admb/releases/download/admb-13.2/admb-13.2-linux.zip
# sudo unzip admb-13.2-linux.zip -d /usr/local/bin
# sudo chmod 755 /usr/local/bin/admb-13.2/bin/admb
# echo "/usr/local/bin/admb-13.2/bin" >> $GITHUB_PATH
# - name: Build stock synthesis
# run: |
# rm -rf SS330
# mkdir SS330
# /bin/bash ./Make_SS_330_new.sh -b SS330
- name: Build stock synthesis with admb docker image
run: |
rm -rf SS330
rm -rf ss3_osx.tar
mkdir SS330
chmod 777 SS330
/bin/bash ./Make_SS_330_new.sh --admb docker -b SS330
- name: Move exes and R scripts to needed locations
run: |
mv test-models-repo/models test-models-repo/model_runs
mv SS330/ss3 test-models-repo/model_runs/ss3
- name: Change permissions on ss3 exes
run: sudo chmod a+x test-models-repo/model_runs/ss3
- name: Download R packages to run test models in parallel parallel
run: Rscript -e 'install.packages(c("parallely", "furrr", "future"))'
- name: Run test models
run: |
# Setup parallel
ncores <- parallelly::availableCores(omit = 1)
future::plan(future::multisession, workers = ncores)
# Get model names and file paths
mod_names <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = FALSE, recursive = FALSE)
mod_paths <- list.dirs(file.path("test-models-repo", "model_runs"), full.names = TRUE, recursive = FALSE)
print(mod_names)
# Create function to run models with estimation
run_ss <- function(dir) {
wd <- getwd()
print(wd)
on.exit(system(paste0("cd ", wd)))
# Rename the reference files
file.rename(file.path(dir, "ss_summary.sso"),
file.path(dir, "ss_summary_ref.sso"))
file.rename(file.path(dir, "warning.sso"),
file.path(dir, "warning_ref.sso"))
file.copy(file.path(dir, "ss3.par"), file.path(dir, "ss3_ref.par"))
# Run the models with estimation and see if model finishes without error
message("running ss3 on ", basename(dir))
system(paste0("cd ", dir, " && ../ss3 -nox"))
model_ran <- file.exists(file.path(dir, "control.ss_new"))
return(model_ran)
}
# Run test models in parallel and print out error messages
mod_ran <- furrr::future_map(mod_paths, function(x){tryCatch(run_ss(x),
error = function(e) print(e))})
mod_errors <- mod_names[unlist(lapply(mod_ran, function(x) "simpleError" %in% class(x)))]
success <- TRUE
# Print list of models with errors if there are any
if(length(mod_errors) > 0) {
message("Model code with errors were: ", paste0(mod_errors, collapse = ", "),
". See error list above for more details.")
success <- FALSE
} else {
message("All code ran without error, but model runs may still have failed.")
}
# Print list of models that didn't run if there are any
mod_no_run <- mod_names[unlist(lapply(mod_ran, function(x) isFALSE(x)))] # false means model didn't run
if(length(mod_no_run) > 0) {
message("Models that didn't run are ", paste0(mod_no_run, collapse = ", "))
success <- FALSE
} else {
message("All models ran without error.")
}
# Determine if job fails or passes
if(success == FALSE) {
stop("Job failed due to code with errors or models that didn't run.")
} else {
message("All models successfully ran.")
}
shell: Rscript {0}
- name: Run comparison using compare.R file in test models repo
run: |
source("test-models-repo/.github/r_scripts/compare.R")
orig_wd <- getwd()
setwd("test-models-repo")
on.exit(orig_wd)
dir.create("run_R")
# Get model folder names
mod_fold <- file.path("model_runs")
mod_names <- list.dirs(mod_fold, full.names = FALSE, recursive = FALSE)
message("Will compare ref runs to new results for these models:")
print(mod_names)
message("Notable changes in total likelihood, max gradients, ",
" and number of warnings:")
compare_list <- vector(mode = "list", length = length(mod_names))
for(i in mod_names) {
pos <- which(mod_names == i)
sum_file <- file.path(mod_fold, i, "ss_summary.sso")
if (i == "Simple") {
file.copy(sum_file, file.path("run_R", paste0(i, "_ss_summary.sso")))
}
ref_sum_file <- file.path(mod_fold, i, "ss_summary_ref.sso")
par_file <- file.path(mod_fold, i, "ss3.par")
ref_par_file <- file.path(mod_fold, i, "ss3_ref.par")
warn_file <- file.path(mod_fold, i, "warning.sso")
ref_warn_file <- file.path(mod_fold, i, "warning_ref.sso")
fail_file <- file.path("run_R", "test_failed.csv")
compare_list[[pos]] <- compare_ss_runs(mod_name = i,
sum_file = sum_file, ref_sum_file = ref_sum_file,
par_file = par_file, ref_par_file = ref_par_file,
warn_file = warn_file, ref_warn_file = ref_warn_file,
hessian = TRUE,
new_file = NULL, fail_file = fail_file)
}
# Write out all model results
compare_df <- do.call("rbind", compare_list)
compare_df_print <- format(compare_df, digits = 6, nsmall = 3,
justify = "left")
message("See saved artifact all_results.csv for all compared values and their differences.")
# Write all model comparison results to csv
write.csv(compare_df_print, "run_R/all_results.csv", row.names = FALSE)
# Write all model changes to csv
message("See saved artifact all_changes.csv for only changed values (even if the threshold was too low to fail the job).")
filtered_df <- compare_df[compare_df$diff != 0, ]
filtered_df <- format(filtered_df, digits = 6, nsmall = 3,
justify = "left")
write.csv(filtered_df, "run_R/all_changes.csv", row.names = FALSE)
shell: Rscript {0}
- name: Determine results of test
run: cd test-models-repo && Rscript .github/r_scripts/check_failed.R
- name: Archive results
uses: actions/upload-artifact@main
if: always()
with:
name: 'result_textfiles'
path: test-models-repo/run_R/