-
-
Notifications
You must be signed in to change notification settings - Fork 2
/
run.sh
148 lines (125 loc) · 4.63 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#!/bin/bash
# Welcome to the run.sh tutorial!
# This script solves the challenge of managing multiple tasks that need to run
# at different intervals within a SyftBox application.
# Problem: How do we keep track of when each task was last run?
# Solution: Create a directory to store timestamps for each section
TIMESTAMP_DIR="./script_timestamps"
mkdir -p "$TIMESTAMP_DIR"
mkdir -p "state"
# Problem: How do we determine if enough time has passed to run a task again?
# Solution: Create a function to check the time difference
function should_run() {
local section=$1
local interval=$2
local timestamp_file="$TIMESTAMP_DIR/${section}_last_run"
if [ ! -f "$timestamp_file" ]; then
return 0
fi
last_run=$(cat "$timestamp_file")
current_time=$(date +%s)
time_diff=$((current_time - last_run))
if [ $time_diff -ge $interval ]; then
return 0
else
return 1
fi
}
# Problem: How do we ensure our project always has the latest dependencies?
# Solution: Create a function to update dependencies using 'uv'
function update_dependencies() {
curl -LsSf https://astral.sh/uv/install.sh | sh
#!/bin/sh
if [ ! -d ".venv" ]; then
echo "Virtual environment not found. Creating one..."
uv venv .venv
echo "Virtual environment created successfully."
else
echo "Virtual environment already exists."
fi
uv pip install -r requirements.txt
}
# Problem: How do we record when a task was last run?
# Solution: Create a function to update the timestamp for a section
function update_timestamp() {
local section=$1
local timestamp_file="$TIMESTAMP_DIR/${section}_last_run"
date +%s > "$timestamp_file"
}
# Problem: How do we manage tasks that need to run every 5 seconds?
# Solution: Create a function for 5-second interval tasks
function section_0() {
local section="EXAMPLE PROJECT: section_0"
local interval=5 # 5 seconds
if should_run "$section" $interval; then
echo "Running $section..."
uv run python main_pipeline_setup.py
uv run python main_5_secs.py
echo "Section 0 completed."
update_timestamp "$section"
else
echo "Skipping $section, not enough time has passed."
fi
}
# Problem: How do we handle tasks that should run every 5 minutes?
# Solution: Create a function for 5-minute interval tasks
function section_1() {
local section="EXAMPLE PROJECT: section_1"
local interval=300 # 5 minutes
if should_run "$section" $interval; then
echo "Running $section..."
uv run python main_5_mins.py
echo "Section 1 completed."
update_timestamp "$section"
else
echo "Skipping $section, not enough time has passed."
fi
}
# Problem: What about tasks that need to run hourly?
# Solution: Create a function for hourly tasks
function section_2() {
local section="EXAMPLE PROJECT: section_2"
local interval=3600 # 1 hour
if should_run "$section" $interval; then
echo "Running $section..."
uv run python main_1_hour.py
echo "Section 2 completed."
update_timestamp "$section"
else
echo "Skipping $section, not enough time has passed."
fi
}
# Problem: How do we manage daily tasks and ensure dependencies are up to date?
# Solution: Create a function for daily tasks that also updates dependencies
function section_3() {
local section="EXAMPLE PROJECT: section_3"
local interval=86400 # 1 day
if should_run "$section" $interval; then
echo "Running $section..."
uv run python main_1_day.py
update_dependencies
echo "Section 3 completed."
update_timestamp "$section"
else
echo "Skipping $section, not enough time has passed."
fi
}
# Problem: How do we ensure all our tasks are executed in the correct order?
# Solution: Call all section functions in the desired sequence
section_0
section_1
section_2
section_3
# Congratulations! You've reached the end of the run.sh script.
# This script solves the complex problem of managing multiple tasks
# with different execution intervals in a SyftBox application.
# To customize this for your own project:
# 1. Identify the different intervals at which your tasks need to run
# 2. Create or modify sections for each interval
# 3. Replace the Python script calls with your own task scripts
# 4. Adjust the update_dependencies function if needed
# Remember: This script is typically called by a cron job frequently.
# The 'should_run' function ensures each section only runs when appropriate,
# solving the problem of over-execution.
# Next Steps:
# - Explore main_pipeline_setup.py to understand pipeline folder creation