Improve build into master

.dockerignore

@@ -0,0 +1,10 @@
+/solr-*
+/unpacked-war/
+/build/
+/specify_exports/*
+!/specify_exports/README
+
+/.lastupdate
+/example.crontab
+
+.DS_Store

.gitignore

@@ -5,4 +5,6 @@
 !/specify_exports/README
 
 /.lastupdate
-/example.crontab
+/example.crontab
+
+.DS_Store

Dockerfile

@@ -0,0 +1,60 @@
+# Build it like this:
+# docker build --tag webportal-service:improve-build .
+
+# Run it like this:
+# docker run -p 80:80 -v /absolute/location/of/your/export.zip:/home/specify/webportal-installer/specify_exports/export.zip webportal-service:improve-build
+
+FROM ubuntu:18.04
+
+LABEL maintainer="Specify Collections Consortium <github.com/specify>"
+
+# Get Ubuntu packages
+RUN apt-get update && apt-get -y install \
+        nginx \
+        unzip \
+        curl \
+        wget \
+        python \
+        python-lxml \
+        make \
+        lsof \
+        default-jre \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create a user and group for the application
+RUN groupadd -g 999 specify && \
+    useradd -r -u 999 -g specify specify
+
+# Create the application directory and set ownership
+RUN mkdir -p /home/specify/webportal-installer && chown specify:specify -R /home/specify
+
+# Switch to the specify user
+USER specify
+
+# Copy the application files into the container
+COPY --chown=specify:specify . /home/specify/webportal-installer
+WORKDIR /home/specify/webportal-installer
+
+# Expose the port for the web portal
+EXPOSE 80
+
+# Switch back to root user for further configuration
+USER root
+
+# Configure nginx to proxy the Solr requests and serve the static files
+COPY webportal-nginx.conf /etc/nginx/sites-available/webportal-nginx.conf
+
+# Disable the default nginx site and enable the portal site
+RUN rm /etc/nginx/sites-enabled/default \
+        && ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
+        && service nginx stop
+
+# Redirect nginx logs to stdout and stderr
+RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log
+
+# Build the Solr app
+# Run Solr in foreground
+# Wait for Solr to load
+# Import data from the .zip file
+# Run Docker in foreground
+CMD ["sh", "-c", "make clean-all && make build-all && ./build/bin/solr start -force && sleep 20 && curl -v \"http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\\\"&escape=\\&header=true\" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' && nginx -g 'daemon off;'"]

Makefile

@@ -1,32 +1,218 @@
 # Mirror for downloading Apache Solr.
-SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr
+SOLR_MIRROR := https://archive.apache.org/dist/lucene/solr
+
+# Use 'schema.xml' if solr will be used to create the core
+# Use 'managed-schema' if pre-configuring core
+SCHEMA_FILE := managed-schema
+
+#location of default settings files in solr dist
+DEFAULT_SETS := server/solr/configsets/_default
+
+PYTHON := python2
 
 # Use latest available version of Solr 4.
-export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr_vers.py)
+SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py)
 
-export SOLR_DIST := solr-$(SOLR_VERSION)
-export TOPDIR := $(shell pwd)
+SOLR_DIST := solr-$(SOLR_VERSION)
 
-all: build
+INPUTS := $(wildcard specify_exports/*.zip)
+COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
+PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
+WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
+SETTING_TEMPLATES := $(addprefix build/setting_templates/, $(COLLECTIONS))
+SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
-clean:
-	rm -rf build/
+###### Usage Information #####
 
-realclean: clean
-	rm -rf solr-* unpacked-war
+.PHONY: usage
+usage:
+	@echo Usage:
+	@echo
+	@echo make build-all    -- Same as build-html build-cores.
+	@echo make build-html   -- Builds the web app part of the web portal.
+	@echo make build-cores  -- Builds the solr cores for all collections.
+	@echo make load-data    -- Loads the collection data into solr.
+	@echo make load-data-COLLECTION  -- Loads the data for COLLECTION into solr.
+	@echo make force-load-data    -- Loads the collection data into solr even if up-to-date.
+	@echo make force-load-data-COLLECTION  -- Loads the data for COLLECTION into solr even if up-to-date.
+	@echo
+	@echo The build-all and build-cores targets should only be used when solr is
+	@echo not running. The build-html and load-data* targets maybe used freely.
+	@echo All build and load targets will only have effect if there are changes in
+	@echo the specify_exports or custom_settings directories.
 
-build: $(SOLR_DIST) build.make specify_exports specify_exports/*.zip
-	cp -r $(SOLR_DIST)/ build
-	$(MAKE) -f $(TOPDIR)/build.make -C build
-	touch $@
+##### Main targets #####
+
+.PHONY: build-all build-cores build-html build-setting-templates load-data
+build-all: build-cores build-html build-setting-templates
+build-cores: $(SOLR_CORES)
+build-html: $(WEBAPPS) build/html/index.html
+build-setting-templates: $(SETTING_TEMPLATES)
+load-data: $(addprefix load-data-, $(COLLECTIONS))
+force-load-data: $(addprefix force-load-data-, $(COLLECTIONS))
+
+##### Cleaning targets #####
+
+.PHONY: clean-all clean-build clean-solr clean-cores clean-html clean-*-html clean-*-core clean-setting-templates
+
+clean-all: clean-build clean-solr
+
+clean-build:
+	rm -rf build
+
+clean-solr:
+	rm -rf solr-*
+
+clean-cores:
+	rm -rf $(SOLR_CORES)
+
+clean-html:
+	rm -rf build/html
+
+clean-%-html:
+	rm -rf build/html/$*
+
+clean-%-core:
+	rm -rf build/server/solr/$*
+
+clean-setting-templates:
+	rm -rf build/setting-templates
+
+##### Common building steps #####
 
 $(SOLR_DIST).tgz:
-	# Fetching Solr distribution tar ball.
+	@printf "\n\n### Fetching Solr distribution tar ball.\n\n"
 	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
 
 $(SOLR_DIST): $(SOLR_DIST).tgz
-	# Unpacking Solr distribution.
+	@printf "\n\n### Unpacking Solr distribution.\n\n"
 	rm -rf $@
 	tar -zxf $<
+	touch $@
+
+build: $(SOLR_DIST)
+	@printf "\n\n### Copying solr to build directory.\n\n"
+	cp -r $(SOLR_DIST)/ build
+	$(PYTHON) patch_web_xml.py \
+		$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml\
+		> $@/server/solr-webapp/webapp/WEB-INF/web.xml
+
+build/col: | build
+	@printf "\n\n"
+	mkdir build/col
+
+.PRECIOUS: build/col/%/PortalFiles
+build/col/%/PortalFiles: specify_exports/%.zip | build/col
+	@printf "\n\n### Extracting $@.\n\n"
+	unzip -DD -qq -o -d build/col/$* $^
+
+##### Solr core building #####
+
+build/server/solr/%: build/col/%/SolrFldSchema.xml | build
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@/conf $@/data
 
-$(SOLR_DIST)/%: $(SOLR_DIST)
+	$(PYTHON) patch_solrconfig_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml \
+		> $@/conf/solrconfig.xml
+
+	$(PYTHON) patch_schema_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \
+		build/col/$*/SolrFldSchema.xml \
+		> $@/conf/$(SCHEMA_FILE)
+
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt $@/conf/
+	cp -r $(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ $@/conf/
+
+	echo 'dataDir=data' > $@/core.properties
+	echo 'name=$*' >> $@/core.properties
+	echo 'config=conf/solrconfig.xml' >> $@/core.properties
+
+	touch $@
+
+build/col/%/SolrFldSchema.xml: build/col/%/PortalFiles
+	@printf "\n\n### Generating $@.\n\n"
+	echo '<?xml version="1.0" encoding="UTF-8" ?>' > $@
+	echo "<fields>" >> $@
+	cat $</SolrFldSchema.xml >> $@
+	echo "</fields>" >> $@
+
+##### Website building #####
+
+build/html: | build
+	@printf "\n\n"
+	mkdir build/html
+
+build/html/index.html: $(WEBAPPS) | build/html
+	@printf "\n\n### Generating $@.\n\n"
+	$(PYTHON) make_toplevel_index.py index_skel.html \
+		$(addsuffix /resources/config/settings.json, $(WEBAPPS)) \
+		> $@
+
+build/html/%: build/col/%/PortalFiles custom_settings/%/fldmodel.json custom_settings/%/settings.json | build/html
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	cp -r PortalApp/* $@
+	$(PYTHON) make_fldmodel_json.py \
+		build/col/$*/PortalFiles/*flds.json \
+		custom_settings/$*/fldmodel.json \
+		> $@/resources/config/fldmodel.json
+	$(PYTHON) patch_settings_json.py \
+		PortalApp/resources/config/settings.json \
+		custom_settings/$*/settings.json \
+		$* \
+		build/col/$*/PortalFiles/*Setting.json \
+		> $@/resources/config/settings.json
+	touch $@
+
+custom_settings/%/fldmodel.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
+custom_settings/%/settings.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
+##### Loading data #####
+
+.PHONY: load-data-%
+load-data-%: build/html/%/load-timestamp.txt ;
+
+.PRECIOUS: build/html/%/load-timestamp.txt
+build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
+	@printf "\n\n### Loading data into $*.\n\n"
+	curl -X POST "http://localhost:8983/solr/$*/update" \
+		-d '{ "delete": {"query":"*:*"} }' \
+		-H 'Content-Type: application/json' \
+		| grep '"status":0'
+	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
+		--data-binary @build/col/$*/PortalFiles/PortalData.csv \
+		-H 'Content-type:application/csv' \
+		| grep '"status":0'
+	date > $@
+
+.PHONY: force-load-data-%
+force-load-data-%:
+	rm -f build/html/$*/load-timestamp.txt
+	$(MAKE) build/html/$*/load-timestamp.txt
+
+##### Settings templates #####
+
+build/setting_templates: | build
+	@printf "\n\n"
+	mkdir $@
+
+build/setting_templates/%: build/html/% | build/setting_templates
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	$(PYTHON) make_settings_template.py \
+		PortalApp/resources/config/settings.json \
+		> $@/settings.json
+	$(PYTHON) make_fields_template.py \
+		build/html/$*/resources/config/fldmodel.json \
+		> $@/fldmodel.json
+	touch $@