-
Notifications
You must be signed in to change notification settings - Fork 57
/
install.py
executable file
·429 lines (359 loc) · 16.6 KB
/
install.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
#!/usr/bin/env python3
################################################################
#
# install.py - Quickly set up a new GovReady-Q instance
# from a freshly-cloned repository.
#
# Usage: install.py [--help] [--non-interactive] [--verbose]
#
# Optional arguments:
# -h, --help show this help message and exit
# -n, --non-interactive run without terminal interaction
# -t, --timeout seconds to allow external programs to run
# -u, --user do pip install with --user flag
# -v, --verbose output more information
#
################################################################
# Note: we use print("foo") ; sys.stdout.flush() instead of print("", flush=True)
# to avoid a syntax error crash if run under Python 2.
# parse command-line arguments
import argparse
# system stuff
import os
import platform
import re
import signal
import subprocess
import sys
import time
from subprocess import PIPE
# JSON handling
import json
# Default constants
GOVREADYURL = "http://localhost:8000"
SPACER = "\n====\n"
# Gracefully exit on control-C
signal.signal(signal.SIGINT, lambda signal_number, current_stack_frame: sys.exit(0))
# Define a fatal error handler
class FatalError(Exception):
pass
# Define a halted error handler
class HaltedError(Exception):
pass
# Define a non-zero return code error handler
class ReturncodeNonZeroError(Exception):
def __init__(self, completed_process, msg=None):
if msg is None:
# default message if none set
msg = "An external program or script returned an error."
super(ReturncodeNonZeroError, self).__init__(msg)
self.completed_process = completed_process
# Set up argparse
def init_argparse():
parser = argparse.ArgumentParser(
description='Quickly set up a new GovReady-Q instance from a freshly-cloned repository.')
parser.add_argument('--non-interactive', '-n', action='store_true', help='run without terminal interaction')
parser.add_argument('--timeout', '-t', type=int, default=120,
help='seconds to allow external programs to run (default=120)')
parser.add_argument('--user', '-u', action='store_true', help='do pip install with --user flag')
parser.add_argument('--verbose', '-v', action='count', default=0, help='output more information')
parser.add_argument('--docker', '-d', action='store_true', help='runs with docker installation')
return parser
################################################################
#
# helpers
#
################################################################
def run_optionally_verbose(args, timeout, verbose_flag):
if verbose_flag:
import time
start = time.time()
print(f"Executing: {args}")
p = subprocess.run(args, timeout=timeout)
print("Elapsed time: {:1f} seconds.".format(time.time() - start))
else:
p = subprocess.run(args, timeout=timeout, stdout=PIPE, stderr=PIPE)
return p
def check_has_command(command_array):
try:
# hardcode timeout to 5 seconds; if checking command takes longer than that, something is really wrong
p = subprocess.run(command_array, timeout=5, stdout=PIPE, stderr=PIPE)
return True
except FileNotFoundError as err:
return False
# checks if a package is out of date
# if okay, returns (True, None, None)
# if out of date, returns (False, current_version, latest_version)
# N.B., this routine will return okay if the package is not installed
def check_package_version(package_name):
p = subprocess.run([sys.executable, '-m', 'pip', 'list', '--outdated', '--format', 'json'], stdout=PIPE, stderr=PIPE)
packages = json.loads(p.stdout.decode('utf-8'))
for package in packages:
if package['name'] == package_name:
return False, package['version'], package['latest_version']
return True, None, None
def create_environment_json(path):
import secrets
alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
secret_key = ''.join(secrets.choice(alphabet) for i in range(50))
# NOTE: `environment` here refers to locally-created environment data object and not OS-level environment variables
environment = {
"govready-url": GOVREADYURL,
"static": "static_root",
"secret-key": secret_key,
"test_visible": False,
"debug": True,
"session_security_expire_at_browser_close" : True,
"session_security_warn_after" : 1200,
"session_security_expire_after" : 1800
}
# Create local directory
if not os.path.exists('local'):
os.makedirs('local')
# Create local/envionment.json file
with open(path, 'w') as f:
f.write(json.dumps(environment, sort_keys=True, indent=2))
def main():
print(">>>>>>>>>> Welcome to the GovReady-Q Installer <<<<<<<<<\n")
try:
# Collect command line arguments, print help if necessary
argparser = init_argparse();
args = argparser.parse_args();
python_manage = ['./manage.py']
if args.docker:
python_manage = [sys.executable, "manage.py"]
elif os.name == 'nt':
python_manage = [sys.executable, 'manage.py']
print("Testing environment...\n")
# Print machine information
print("Platform is {} version {} running on {}.".format(platform.system(), platform.release(), platform.machine()))
# Print spacer
print(SPACER)
# Test version of Python
ver = sys.version_info
print("Python version is {}.{}.{}.".format(ver[0], ver[1], ver[2]))
if sys.version_info >= (3, 6):
print("+ Python version is >= 3.6.")
else:
print("! Python version is < 3.6.")
print("GovReady-Q is best run with Python 3.6 or higher.")
print("It is STRONGLY encouraged to run GovReady-Q with Python 3.8 or higher.")
if args.non_interactive:
reply = ''
else:
reply = input("Continue install with Python {}.{}.{} (y/n)? ".format(ver[0], ver[1], ver[2]))
if len(reply) == 0 or reply[0].lower() != "y":
raise HaltedError("Python version is < 3.8")
# Print spacer
print(SPACER)
# Check if inside a virtual environment
if not args.docker:
print("Check for virtual Python environment.")
if sys.prefix != sys.base_prefix:
print("+ Installer is running inside a virtual Python environment.")
else:
print("! Installer is not running inside a virtual Python environment.")
print("It is STRONGLY encouraged to run GovReady-Q inside a Python virtual environment.")
if args.non_interactive:
reply = ''
else:
reply = input("Continue install outside of virtual environment (y/n)? ")
if len(reply) == 0 or reply[0].lower() != "y":
raise HaltedError("Installer is not running inside a virtual Python environment")
# Print spacer
print(SPACER)
# Check for python3 and pip3 (and not 2 or e.g. 'python3.8')
print("Confirming python3 and pip3 commands are available...")
sys.stdout.flush()
if not check_has_command(['python3', '--version']):
raise FatalError("The 'python3' command is not available.")
if not check_has_command(['pip3', '--version']):
raise FatalError("The 'pip3' command is not available.")
print("... done confirming python3 and pip3 commands are available.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Check for updated pip
print("Check that pip is up to date.")
pip_up_to_date, pip_current, pip_latest = check_package_version('pip')
if pip_up_to_date:
print("+ pip is up to date.")
else:
print("! pip is not the latest version ({} vs. {}).".format(pip_current, pip_latest))
print("It is STRONGLY encouraged to ensure pip is updated before continuing, or non-obvious errors may occur.")
if args.non_interactive:
reply = ''
else:
reply = input("Continue install with outdated pip (y/n)? ")
if len(reply) == 0 or reply[0].lower() != "y":
raise HaltedError(
"pip is not up to date ({} vs. {}).\n\nSuggested fix: Run 'pip install --upgrade pip'".format(
pip_current, pip_latest))
# Print spacer
print(SPACER)
# Print mode of interactivity
if args.non_interactive:
print("Installing/updating GovReady-Q in non-interactive mode.")
else:
print("Installing/updating GovReady-Q in interactive mode.")
# Print spacer
print(SPACER)
# Briefly sleep in verbose mode so user can glance at output.
time.sleep(3) if args.verbose else time.sleep(0)
# pip install basic requirements
print("Installing Python libraries via pip (this may take a while)...")
sys.stdout.flush()
if args.user:
pip_install_command = ['pip3', 'install', '--user', '-r', 'requirements.txt']
else:
pip_install_command = ['pip3', 'install', '-r', 'requirements.txt']
if args.docker:
pip_install_command.append('--ignore-installed')
p = run_optionally_verbose(pip_install_command, args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
print("... done installing Python libraries via pip.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# # Retrieve static assets
if args.docker:
print("Fetching static resource files from Internet...")
sys.stdout.flush()
p = run_optionally_verbose(['./fetch-vendor-resources.sh'], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
print("... done fetching resource files from Internet.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Create the local/environment.json file, if it is missing (it generally will be)
# NOTE: `environment` here refers to locally-created environment data object and not OS-level environment variables
print("Creating local/environment.json file...")
sys.stdout.flush()
environment_path = 'local/environment.json'
if os.path.exists(environment_path):
# confirm that environment.json is JSON
try:
environment = json.load(open(environment_path))
print("environment.json file already exists, proceeding")
except json.decoder.JSONDecodeError as e:
print("'{}' is not in JSON format:".format(environment_path))
print(">>>>>>>>>>")
print(open(environment_path).read())
print("<<<<<<<<<<")
raise FatalError("'{}' is not in JSON format.".format(environment_path))
else:
create_environment_json(environment_path)
print("... done creating local/environment.json file.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Configure database (migrate, load_modules)
print("Initializing/migrating database...")
sys.stdout.flush()
print(python_manage)
p = run_optionally_verbose([*python_manage, "migrate"], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
p = run_optionally_verbose([*python_manage, "load_modules"], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
print("... done initializing/migrating database.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Collect files into static directory
print("Collecting files into static directory...")
sys.stdout.flush()
if args.non_interactive:
p = run_optionally_verbose([*python_manage, 'collectstatic', '--no-input'], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
else:
p = run_optionally_verbose([*python_manage, 'collectstatic', '--no-input'], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
print("... done collecting files into static directory.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Run first_run non-interactively
print("Setting up system and creating Administrator user if none exists...")
sys.stdout.flush()
p = subprocess.run([*python_manage, "first_run", "--non-interactive"], timeout=args.timeout, stdout=PIPE,
stderr=PIPE)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
if args.verbose:
print(p.stdout.decode('utf-8'), p.stderr.decode('utf-8'))
# save admin account details
admin_details = ''
if p.stdout:
m1 = re.search('\n(Created administrator account.+)\n', p.stdout.decode('utf-8'))
m2 = re.search('\n(Skipping create admin account.+)\n', p.stdout.decode('utf-8'))
m3 = re.search('\n(\[INFO\] Superuser.+)\n', p.stdout.decode('utf-8'))
if m1:
admin_details = m1.group(1)
elif m2:
admin_details = "Administrator account(s) previously created."
elif m3:
admin_details = "Administrator account(s) previously created."
else:
admin_details = "Administrator account details not found."
print("... done setting up system and creating Administrator user.")
sys.stdout.flush()
# Print spacer
print(SPACER)
# Load GovReady sample SSP
print("Setting up GovReady-Q sample project if none exists...")
sys.stdout.flush()
p = run_optionally_verbose([*python_manage, "load_govready_ssp"], args.timeout, args.verbose)
if p.returncode != 0:
raise ReturncodeNonZeroError(p)
print("... done setting up GovReady-Q sample project.")
sys.stdout.flush()
# Print spacer
print(SPACER)
print("""\
***********************************
* GovReady-Q Server configured... *
***********************************
To start GovReady-Q, run:
./manage.py runserver
""")
if len(admin_details):
if "Created administrator account" in admin_details:
print("Log in using the administrator credentials printed below.\n\nWRITE THIS DOWN:\n")
print(admin_details, "\n")
print("When GovReady-Q is running, visit http://localhost:8000/ with your web browser.\n")
except ReturncodeNonZeroError as err:
p = err.completed_process
sys.stderr.write("\n\nFatal error, exiting: external program or script {} returned error code {}.\n\n".format(p.args,
p.returncode))
# diagnose stdout and stdout to see if we can find an obvious problem
# (add more checks here as appropriate)
# check for missing Xcode Command Line Tools (macOS)
if p.stderr and 'xcrun: error: invalid active developer path (/Library/Developer/CommandLineTools), missing xcrun at: /Library/Developer/CommandLineTools/usr/bin/xcrun' in p.stderr.decode(
'utf-8'):
sys.stderr.write("Suggested fix (see documentation): You need to do 'xcode-select --install'.\n\n")
sys.exit(1)
except subprocess.TimeoutExpired as err:
sys.stderr.write(
"\n\nFatal error, exiting: external program or script {} took longer than {:.1f} seconds.\n\n".format(err.cmd,
err.timeout))
sys.stderr.write("Suggested fix: run again with '--timeout {}'.\n\n".format(max(args.timeout + 120, 600)))
sys.exit(1)
except HaltedError as err:
sys.stderr.write("\n\nInstall halted because: {}.\n\n".format(err));
sys.exit(1)
except FatalError as err:
sys.stderr.write("\n\nFatal error, exiting: {}.\n\n".format(err));
sys.exit(1)
# catch all errors
except Exception as err:
sys.stderr.write(
'\n\nFatal error, exiting: unrecognized error on line {}, "{}".\n\n'.format(sys.exc_info()[2].tb_lineno, err));
sys.exit(1)
if __name__ == "__main__":
exit(main())