From d87b1eae56efa327017a01f3ef28e0e11e6d7cc7 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Wed, 29 Apr 2020 11:37:45 -0500
Subject: [PATCH 01/38] do all building in the build dir and add update-data
 target.

---
 Makefile   | 15 +++++++++++++++
 build.make | 33 +++++++++++++++------------------
 core.make  |  9 ++-------
 3 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index a95b9ec..8f8d995 100644
--- a/Makefile
+++ b/Makefile
@@ -7,8 +7,21 @@ export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr
 export SOLR_DIST := solr-$(SOLR_VERSION)
 export TOPDIR := $(shell pwd)
 
+.PHONY: all clean realclean update-data
+
 all: build
 
+update-data: build
+	for core in build/cores/* ; do \
+		corename=`basename "$$core"` ; \
+		curl -X POST 'http://localhost:8983/solr/vimsfish/update?commit=true' \
+	 		-d '{ "delete": {"query":"*:*"} }' \
+			-H 'Content-Type: application/json' ; \
+		curl "http://localhost:8983/solr/$$corename/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
+			--data-binary @$$core/PortalFiles/PortalData.csv \
+			-H 'Content-type:application/csv' ; \
+	done
+
 clean:
 	rm -rf build/
 
@@ -17,6 +30,8 @@ realclean: clean
 
 build: $(SOLR_DIST) build.make specify_exports specify_exports/*.zip
 	cp -r $(SOLR_DIST)/ build
+	# The following file gets replaced by a patched version:
+	rm build/server/solr-webapp/webapp/WEB-INF/web.xml
 	$(MAKE) -f $(TOPDIR)/build.make -C build
 	touch $@
 
diff --git a/build.make b/build.make
index 1485a91..27f1549 100644
--- a/build.make
+++ b/build.make
@@ -1,6 +1,14 @@
 
-#all: solr-home setting_templates html html/index.html
-all: setting_templates html html/index.html
+.PHONY: link-cores
+
+all: setting_templates html html/index.html link-cores server/solr-webapp/webapp/WEB-INF/web.xml
+
+
+server/solr-webapp/webapp/WEB-INF/web.xml: $(TOPDIR)/patch_web_xml.py \
+					   $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
+	# Patching solr server app for cross-domain access to enable extjs ajax stores to POST solr query params.
+	python2 $^ > $@
+
 
 cores: $(TOPDIR)/core.make $(TOPDIR)/$(SOLR_DIST) \
 		 $(TOPDIR)/specify_exports  $(TOPDIR)/specify_exports/*.zip
@@ -40,20 +48,9 @@ html: cores
 		cp -r $$core/webapp html/`basename $$core` ; \
 	done
 
-#solr-home: $(TOPDIR)/$(SOLR_DIST) cores solr.xml
-#	# Build the Solr home directory.
-#	rm -rf solr-home
-#	cp -r $(TOPDIR)/$(SOLR_DIST)/example/multicore solr-home
-	# Copy each core into place.
-#	rm -rf solr-home/core*
-#	for core in cores/* ; do \
-#		cp -r $$core/core solr-home/`basename $$core` ; \
-#	done
-	# Copy top level Solr configuration into place.
-#	cp solr.xml solr-home/
-
-#solr.xml: $(TOPDIR)/make_solr_xml.py $(TOPDIR)/$(SOLR_DIST)/example/multicore/solr.xml cores
-	# Generate top level Solr config that defines the available cores.
-	#python $(TOPDIR)/make_solr_xml.py $(TOPDIR)/$(SOLR_DIST)/example/multicore/solr.xml \
-	#	cores/* > $@
+link-cores: cores
+	# Link the cores into the solr home folder.
+	for core in cores/* ; do \
+		ln -s ../../$$core server/solr/ ; \
+	done
 
diff --git a/core.make b/core.make
index ebd627b..f18e562 100644
--- a/core.make
+++ b/core.make
@@ -29,12 +29,10 @@ $(SCHEMA_FILE): $(TOPDIR)/patch_schema_xml.py \
 	python2 $^ > $@
 
 web.xml: $(TOPDIR)/patch_web_xml.py \
-		$(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml 
+		$(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
 	# Patching solr server app for cross-domain access to enable extjs ajax stores to POST solr query params.
-	#python $^ > $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
 	python2 $^ > $@
-	sudo cp $@ $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
- 
+
 solrconfig.xml: $(TOPDIR)/patch_solrconfig_xml.py \
 		$(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml
 	# Patching Solr config for use with Specify.
@@ -57,9 +55,6 @@ webapp: $(TOPDIR)/PortalApp settings.json fldmodel.json
 	# Copy patched settings into place.
 	cp settings.json webapp/resources/config/
 
-	# Fix Solr URL format in WebApp.
-	#sed -i "s,solrURL + ':' + solrPort + '/',solrURL," webapp/app/store/MainSolrStore.js
-
 core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE) web.xml
 	# Setup solr-home subdir for this core.
 	mkdir -p core/conf

From 948ef0e3350f82dd9fe4102105de215e5f04355d Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Wed, 29 Apr 2020 14:29:18 -0500
Subject: [PATCH 02/38] web.xml file doesn't seem to be needed in each core.

---
 core.make | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/core.make b/core.make
index f18e562..aaa8b39 100644
--- a/core.make
+++ b/core.make
@@ -28,11 +28,6 @@ $(SCHEMA_FILE): $(TOPDIR)/patch_schema_xml.py \
 	# Patching Solr schema with fields from Specify export.
 	python2 $^ > $@
 
-web.xml: $(TOPDIR)/patch_web_xml.py \
-		$(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
-	# Patching solr server app for cross-domain access to enable extjs ajax stores to POST solr query params.
-	python2 $^ > $@
-
 solrconfig.xml: $(TOPDIR)/patch_solrconfig_xml.py \
 		$(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml
 	# Patching Solr config for use with Specify.
@@ -55,11 +50,10 @@ webapp: $(TOPDIR)/PortalApp settings.json fldmodel.json
 	# Copy patched settings into place.
 	cp settings.json webapp/resources/config/
 
-core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE) web.xml
+core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE)
 	# Setup solr-home subdir for this core.
 	mkdir -p core/conf
 	cp solrconfig.xml $(SCHEMA_FILE)  core/conf/
-	#cp -r PortalFiles/solr core/data/index
 	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt core/conf/
 	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt core/conf/
 	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt core/conf/

From ea5768a12d32754808c4e7fd0006b82521a0b455 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Wed, 29 Apr 2020 17:12:46 -0500
Subject: [PATCH 03/38] update instructions.

---
 Makefile               |   4 +-
 README.md              | 203 +++++++++++++++++++++--------------------
 webportal-nginx.conf   |  17 ++++
 webportal-solr.service |  14 +++
 4 files changed, 135 insertions(+), 103 deletions(-)
 create mode 100644 webportal-nginx.conf
 create mode 100644 webportal-solr.service

diff --git a/Makefile b/Makefile
index 8f8d995..d975f32 100644
--- a/Makefile
+++ b/Makefile
@@ -7,11 +7,11 @@ export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr
 export SOLR_DIST := solr-$(SOLR_VERSION)
 export TOPDIR := $(shell pwd)
 
-.PHONY: all clean realclean update-data
+.PHONY: all clean realclean load-data
 
 all: build
 
-update-data: build
+load-data: build
 	for core in build/cores/* ; do \
 		corename=`basename "$$core"` ; \
 		curl -X POST 'http://localhost:8983/solr/vimsfish/update?commit=true' \
diff --git a/README.md b/README.md
index 8502beb..acf8bf9 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,13 @@
 Specify Web Portal (Version 2.0)
 ================================
 
-The Specify Collections Consortium is funded by its member institutions. The Consortium web site is: [http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/)
+The Specify Collections Consortium is funded by its member
+institutions. The Consortium web site is:
+[http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/)
 
-Specify Web Portal Copyright © 2020 Specify Collections Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free software licensed under GNU General Public License 2 (GPL2).
+Specify Web Portal Copyright © 2020 Specify Collections
+Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free
+software licensed under GNU General Public License 2 (GPL2).
 
 ```
 Specify Collections Consortium
@@ -15,9 +19,14 @@ Lawrence, KS 66045 USA
 
 ## Developer Instructions
 
-After completing these instructions you will be able to run Specify Web Portal 2.0.
+After completing these instructions you will be able to run Specify
+Web Portal 2.0.
 
-These instructions are for deployment on a server running Ubuntu. An export file for a single collection is required for setting up the Specify Web Portal. This can be accomplished using the Schema Mappging tool tool within the Specify 6 application together with the stand alone Specify Data Export tool.
+These instructions are for deployment on a server running Ubuntu. An
+export file for a single collection is required for setting up the
+Specify Web Portal. This can be accomplished using the Schema Mappging
+tool tool within the Specify 6 application together with the stand
+alone Specify Data Export tool.
 
 Install system dependencies
 ------------
@@ -35,60 +44,75 @@ Installation Instructions
 -------------------------
 
 These instructions illustrate the fewest steps needed to install the
-web portal. For updating existing installations, se the 'Data Only Updates' section below.
+web portal. 
 
-1. Clone the Web Portal 2.0 repository by clicking the green button (Clone or Download) at the top of the page and unpack this repository on your server.
+1. These instructions assume the Web Portal is being setup under a
+   user account with the name `specify`. If a different account is
+   used, the directory paths in these instructions will need to be
+   updated accordingly.
 
-   This will install Solr on the server.
+1. Clone the Web Portal 2.0 repository using git:
 
-2. Use the Specify Data Export tool to create a Web Portal export zip
-   file (see the Specify 6 Data Export documentation) for each collection
-   to be hosted in the portal. If aggregated collections are desired, replace the single colleciton with the aggregated collections file after the initial Web Portal installation.
-
-3. Create a  `specify_exports` directory in the web portal directory and copy the zip files (step 2) into it. The copied files should be given names that are suitable for use in URLs; so no spaces, capital letters, slashes or
-   other problematic characters. E.g. `kufish.zip`
+    `specify:~$ git clone https://github.com/specify/webportal-installer.git`
 
-4. Build the Solr app: `make clean && make`.
+    This will create the directory `webportal-installer`.
+    
+1. Configure nginx to proxy the Solr requests and serve the static
+   files by copying `webportal-nginx.conf` to
+   `/etc/nginx/sites-available/`:
+   
+   `specify:~/webportal-installer$ sudo cp webportal-nginx.conf /etc/nginx/sites-available/`
 
-5. Copy the Solr core to the Solr installation:
-
-   [CORENAME] - the name of the exported archive without the file extension. (E.g. `kufish`)
+   N.B. this file will require changes if the `webportal-installer` is
+   in a different location.
+   
+1. Disable the default nginx site and enable the portal site:
+   ```
+   specify:~/webportal-installer$ sudo rm /etc/nginx/sites-enabled/default
+   specify:~/webportal-installer$ sudo ln -s /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/
+   specify:~/webportal-installer$ sudo systemctl restart nginx
+   ```
 
-   [SOLRVERSION] - the version of Solr. (E.g. `7.5.0`)
+2. Use the Specify Data Export tool to create a Web Portal export zip
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be hosted in the portal. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
 
-```
-    mkdir solr-[SOLRVERSION]/server/solr/[CORENAME]
-    cp -r build/cores/[CORENAME]/core/* solr-[SOLRVERSION]/server/solr/[CORENAME]
-    cp build/cores/[CORENAME]/web.xml solr-[SOLRVERSION]/server/solr-webapp/webapp/WEB-INF/web.xml 
-    # Only necessary for the first core.
-```
-6. Restrict access to the solr admin web page. This can be done in solr 7.5 by editing solr:
+3. Copy the zip files from the Specify Data Export into the
+   `specify_exports` directory under `webportal-installer`. The
+   copied files should be given names that are suitable for use in
+   URLs; so no spaces, capital letters, slashes or other problematic
+   characters. E.g. `kufish.zip`
 
-   `[SOLRVERSION]/server/etc/jetty-http.xml`
+4. Build the Solr app: 
 
-   In the ServerConnector section replace: 
+    `specify:~/webportal-installer$ make clean && make`.
 
-   `<Set name="host"><Property name="jetty.host" /></Set>` with `<Set name="host">127.0.0.1</Set>`
+5. Copy the `webportal-solr.service` systemd unit file to
+   `/etc/systemd/system/webportal-solr.service` and reload the
+   systemd daemon.
+   
+   ```
+   specify:~/webportal-installer$ sudo cp webportal-solr.service /etc/systemd/system/
+   specify:~/webportal-installer$ sudo systemctl daemon-reload
+   ```
+   N.B. this file will require changes if the `webportal-installer` is
+   in a different location.
 
-7. Start solr
-   `solr-[SOLRVERSION]/bin/solr start`
+6. Start and enable the service:
 
-   When completing this step the following warnings may be issued and can be safely ignored:
+    ```
+    specify:~/webportal-installer$ sudo systemctl start webportal-solr.service
+    specify:~/webportal-installer$ sudo systemctl enable webportal-solr.service
+    ```
+    
+7. Load the Specify data into Solr:
 
-   `*** [WARN] *** Your open file limit is currently 1024.`
-  
-    `It should be set to 65000 to avoid operational disruption.` 
-    `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to  false in your profile or solr.in.sh` 
-   `*** [WARN] *** Your Max Processes Limit is currently 63590.` 
-    `It should be set to 65000 to avoid operational disruption.` 
-    `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to  false in your profile or solr.in.sh` 
-   `Waiting up to 180 seconds to see Solr running on port 8983 [|]` 
-   `Started Solr server on port 8983 (pid=15422). Happy searching!`
+    `specify:~/webportal-installer$ make load-data`
 
-8. Import the csv data:
-  `curl 'http://localhost:8983/solr/[CORENAME]/update/csv?commit=true&encapsulator="&escape=\&header=true' --data-binary @build/cores/[CORENAME]/PortalFiles/PortalData.csv -H 'Content-type:application/csv'`
 
-  When completing this step you may receive output similar to the following: 
+  When completing this step you will receive output similar to the following: 
 
   `{`
 
@@ -97,75 +121,52 @@ web portal. For updating existing installations, se the 'Data Only Updates' sect
   `"status":0,`
 
   `"QTime":1153}}`
+  
+8. The Web Portal is now running and can be tested by visiting the
+   server's address with a web browser.
+   
+9. Be aware that the Solr admin page is available by default on
+   port 8983. This can be useful for troubleshooting but should be
+   blocked in production using a firewall rule.
 
-9. Move the built webportal to convenient location:
-
-   [WPPATH] - the directory where the web portal fiels reside. (E.g. /home/specify/webportal)
-
-    `mv build [WPPATH]`
-
-10. Create the Nginx configuration file to serve the portal:
-
-     `sudo nano /etc/nginx/sites-available/webportal.conf`
-   ```
-   server {
-       listen 80 default_server;
-
-       rewrite ^/([^/]+)/select /solr/$1/select;
-
-       location ~ ^/solr/([^/]+)/select {
-         proxy_pass http://localhost:8983;
-       }
-
-       location / {
-                root [WPPATH]/html;
-        }
-   }
-   ```
-11. Remove the default Nginx site and enable the portal site:
-   ```
-   sudo rm /etc/nginx/sites-enabled/default
-   sudo ln -L /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/
-   ```
-12. Restart Nginx: `sudo systemctl restart nginx`
-
-    The Web Portal is now running and can be tested.
-
-    The Web Portal can be customized by editing the settings in the settings.json file found at here: [WPPATH]/html/corename/resources/config/settings.json 
-
+Customization
+-------------
 
-Data Only Updates
------------------
+TODO
 
-If the fields used in a portal are unchanged and only data is being updated, delete the current contents of the solr core with:
+Updating
+--------
 
-`curl 'http://localhost:8983/solr/hollow/update?commit=true&stream.body=<delete><query>*%3A*</query></delete>'`
-(You will probably need to add/edit a requestParsers block in the `solr-[SOLRVERSION]/server/solr/[CORENAME]/conf/solrconfig.xml` file for the core. Add it to the requestDispatcher block:
+To update the Specify data in the webportal follow these steps.
 
-```
-<requestDispatcher>
-    <requestParsers enableRemoteStreaming="true"
-                enableStreamBody="true"
-                multipartUploadLimitInKB="2048"
-                formdataUploadLimitInKB="2048"
-                addHttpRequestToContext="false" />
+1. Stop the webportal-solr systemd service:
 
+    `specify:~/webportal-installer$ sudo systemctl stop webportal-solr.service`
 
-    <httpCaching never304="true" />
-</requestDispatcher>
-```
-(See http://lucene.apache.org/solr/guide/requestdispatcher-in-solrconfig.html)
+2. Use the Specify Data Export tool to create a Web Portal export zip
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be hosted in the portal. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
 
-Then use the curl CSV import command above to add the new data.
+3. Copy the zip files from the Specify Data Export into the
+   `specify_exports` directory under `webportal-installer`. The
+   copied files should be given names that are suitable for use in
+   URLs; so no spaces, capital letters, slashes or other problematic
+   characters. E.g. `kufish.zip`
 
+4. Build the Solr app: 
 
-Schema Definition Updates
--------------------------
+    `specify:~/webportal-installer$ make clean && make`.
 
-In this case, you will need to stop Solr `(solr-[SOLRVERSION]/bin/solr stop)`, remove the cores to be updated from your Solr server directory, and follow all the installation steps besides the HTTP configuration.
+6. Start the webportal-solr service:
 
+    `specify:~/webportal-installer$ sudo systemctl start webportal-solr.service`
+    
+7. Load the Specify data into Solr:
 
-Web Portal Application Updates
-------------------------------
+    `specify:~/webportal-installer$ make load-data`
+  
+8. The Web Portal is now running and can be tested by visiting the
+   server's address with a web browser.
 
-If a new version of the Web Portal is being installed it will be necessary to perform step 8 after building.
diff --git a/webportal-nginx.conf b/webportal-nginx.conf
new file mode 100644
index 0000000..f93ece6
--- /dev/null
+++ b/webportal-nginx.conf
@@ -0,0 +1,17 @@
+server {
+    listen 80 default_server;
+
+    rewrite ^/([^/]+)/select /solr/$1/select;
+
+    location ~ ^/solr/([^/]+)/select {
+      proxy_pass http://localhost:8983;
+    }
+
+    location / {
+      root /home/specify/webportal-installer/build/html;
+    }
+
+    location /custom-images/ {
+      alias /home/specify/custom-images/;
+    }
+}
diff --git a/webportal-solr.service b/webportal-solr.service
new file mode 100644
index 0000000..7dceb7d
--- /dev/null
+++ b/webportal-solr.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Solr server for Specify Web Portal
+After=network.target
+
+[Service]
+User=specify
+Group=specify
+PIDFile=/home/specify/webportal-installer/build/bin/solr-8983.pid
+ExecStart=/home/specify/webportal-installer/build/bin/solr start
+ExecStop=/home/specify/webportal-installer/build/bin/solr stop
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

From dd341e5cdcafec12095f7e8bcf990cdce5a03eeb Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Wed, 29 Apr 2020 17:19:49 -0500
Subject: [PATCH 04/38] styling instructions.

---
 README.md | 73 +++++++++++++++++++++++++++++++++----------------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/README.md b/README.md
index acf8bf9..27b4446 100644
--- a/README.md
+++ b/README.md
@@ -53,21 +53,25 @@ web portal.
 
 1. Clone the Web Portal 2.0 repository using git:
 
-    `specify:~$ git clone https://github.com/specify/webportal-installer.git`
-
+    ```console
+    specify:~$ git clone https://github.com/specify/webportal-installer.git
+    ```
+    
     This will create the directory `webportal-installer`.
     
 1. Configure nginx to proxy the Solr requests and serve the static
    files by copying `webportal-nginx.conf` to
    `/etc/nginx/sites-available/`:
    
-   `specify:~/webportal-installer$ sudo cp webportal-nginx.conf /etc/nginx/sites-available/`
-
+   ```console
+   specify:~/webportal-installer$ sudo cp webportal-nginx.conf /etc/nginx/sites-available/
+   ```
+   
    N.B. this file will require changes if the `webportal-installer` is
    in a different location.
    
 1. Disable the default nginx site and enable the portal site:
-   ```
+   ```console
    specify:~/webportal-installer$ sudo rm /etc/nginx/sites-enabled/default
    specify:~/webportal-installer$ sudo ln -s /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/
    specify:~/webportal-installer$ sudo systemctl restart nginx
@@ -80,20 +84,21 @@ web portal.
    collections file after the initial Web Portal installation.
 
 3. Copy the zip files from the Specify Data Export into the
-   `specify_exports` directory under `webportal-installer`. The
-   copied files should be given names that are suitable for use in
-   URLs; so no spaces, capital letters, slashes or other problematic
+   `webportal-installer/specify_exports` directory. The copied files
+   should be given names that are suitable for use in URLs; so no
+   spaces, capital letters, slashes or other problematic
    characters. E.g. `kufish.zip`
 
 4. Build the Solr app: 
-
-    `specify:~/webportal-installer$ make clean && make`.
+   ```console
+   specify:~/webportal-installer$ make clean && make
+   ```
 
 5. Copy the `webportal-solr.service` systemd unit file to
    `/etc/systemd/system/webportal-solr.service` and reload the
    systemd daemon.
    
-   ```
+   ```console
    specify:~/webportal-installer$ sudo cp webportal-solr.service /etc/systemd/system/
    specify:~/webportal-installer$ sudo systemctl daemon-reload
    ```
@@ -102,25 +107,26 @@ web portal.
 
 6. Start and enable the service:
 
-    ```
+    ```console
     specify:~/webportal-installer$ sudo systemctl start webportal-solr.service
     specify:~/webportal-installer$ sudo systemctl enable webportal-solr.service
     ```
     
 7. Load the Specify data into Solr:
 
-    `specify:~/webportal-installer$ make load-data`
+    ```console
+    specify:~/webportal-installer$ make load-data
+    ```
 
 
   When completing this step you will receive output similar to the following: 
 
-  `{`
-
-  `"responseHeader":{`
-
-  `"status":0,`
-
-  `"QTime":1153}}`
+    ```json
+    {
+    "responseHeader":{
+    "status":0,
+    "QTime":1153}}
+    ```
   
 8. The Web Portal is now running and can be tested by visiting the
    server's address with a web browser.
@@ -141,7 +147,9 @@ To update the Specify data in the webportal follow these steps.
 
 1. Stop the webportal-solr systemd service:
 
-    `specify:~/webportal-installer$ sudo systemctl stop webportal-solr.service`
+    ```console
+    specify:~/webportal-installer$ sudo systemctl stop webportal-solr.service
+    ```
 
 2. Use the Specify Data Export tool to create a Web Portal export zip
    file (see the Specify 6 Data Export documentation) for each
@@ -149,24 +157,31 @@ To update the Specify data in the webportal follow these steps.
    are desired, replace the single colleciton with the aggregated
    collections file after the initial Web Portal installation.
 
+
 3. Copy the zip files from the Specify Data Export into the
-   `specify_exports` directory under `webportal-installer`. The
-   copied files should be given names that are suitable for use in
-   URLs; so no spaces, capital letters, slashes or other problematic
+   `webportal-installer/specify_exports` directory. The copied files
+   should be given names that are suitable for use in URLs; so no
+   spaces, capital letters, slashes or other problematic
    characters. E.g. `kufish.zip`
-
+   
 4. Build the Solr app: 
 
-    `specify:~/webportal-installer$ make clean && make`.
+    ```console
+    specify:~/webportal-installer$ make clean && make
+    ```
 
 6. Start the webportal-solr service:
 
-    `specify:~/webportal-installer$ sudo systemctl start webportal-solr.service`
+    ```console
+    specify:~/webportal-installer$ sudo systemctl start webportal-solr.service
+    ```
     
 7. Load the Specify data into Solr:
 
-    `specify:~/webportal-installer$ make load-data`
-  
+    ```console
+    specify:~/webportal-installer$ make load-data
+    ```
+    
 8. The Web Portal is now running and can be tested by visiting the
    server's address with a web browser.
 

From 636e267fb9febe8bf4ad8d576c0085b620b7e174 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Wed, 29 Apr 2020 17:22:30 -0500
Subject: [PATCH 05/38] no json formatter in markdown?

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 27b4446..74ff2b7 100644
--- a/README.md
+++ b/README.md
@@ -121,7 +121,7 @@ web portal.
 
   When completing this step you will receive output similar to the following: 
 
-    ```json
+    ```
     {
     "responseHeader":{
     "status":0,

From b89e120ed475f4148881f714a24ee735baf48a93 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Thu, 30 Apr 2020 12:54:30 -0500
Subject: [PATCH 06/38] fix load-data command.

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d975f32..6ea9e38 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ all: build
 load-data: build
 	for core in build/cores/* ; do \
 		corename=`basename "$$core"` ; \
-		curl -X POST 'http://localhost:8983/solr/vimsfish/update?commit=true' \
+		curl -X POST 'http://localhost:8983/solr/$$corename/update?commit=true' \
 	 		-d '{ "delete": {"query":"*:*"} }' \
 			-H 'Content-Type: application/json' ; \
 		curl "http://localhost:8983/solr/$$corename/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \

From 07e173be3d4dcfc39139a61c47a824d16b753784 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Thu, 30 Apr 2020 12:55:56 -0500
Subject: [PATCH 07/38] fix load-data command again.

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6ea9e38..f16a0cd 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ all: build
 load-data: build
 	for core in build/cores/* ; do \
 		corename=`basename "$$core"` ; \
-		curl -X POST 'http://localhost:8983/solr/$$corename/update?commit=true' \
+		curl -X POST "http://localhost:8983/solr/$$corename/update?commit=true" \
 	 		-d '{ "delete": {"query":"*:*"} }' \
 			-H 'Content-Type: application/json' ; \
 		curl "http://localhost:8983/solr/$$corename/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \

From cbea0c77d8dd22a4f77c1edb891648f93a969ab7 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Thu, 30 Apr 2020 15:40:20 -0500
Subject: [PATCH 08/38] more instruction updates.

---
 README.md | 57 +++++++++++++++++++++++++++++--------------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index 74ff2b7..3d2cd3c 100644
--- a/README.md
+++ b/README.md
@@ -54,17 +54,17 @@ web portal.
 1. Clone the Web Portal 2.0 repository using git:
 
     ```console
-    specify:~$ git clone https://github.com/specify/webportal-installer.git
+    specify@wp:~$ git clone https://github.com/specify/webportal-installer.git
     ```
     
     This will create the directory `webportal-installer`.
     
 1. Configure nginx to proxy the Solr requests and serve the static
-   files by copying `webportal-nginx.conf` to
+   files by copying the provided `webportal-nginx.conf` to
    `/etc/nginx/sites-available/`:
    
    ```console
-   specify:~/webportal-installer$ sudo cp webportal-nginx.conf /etc/nginx/sites-available/
+   root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-nginx.conf /etc/nginx/sites-available/
    ```
    
    N.B. this file will require changes if the `webportal-installer` is
@@ -72,9 +72,9 @@ web portal.
    
 1. Disable the default nginx site and enable the portal site:
    ```console
-   specify:~/webportal-installer$ sudo rm /etc/nginx/sites-enabled/default
-   specify:~/webportal-installer$ sudo ln -s /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/
-   specify:~/webportal-installer$ sudo systemctl restart nginx
+   root@wp:~# rm /etc/nginx/sites-enabled/default
+   root@wp:~# ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/
+   root@wp:~# systemctl restart nginx
    ```
 
 2. Use the Specify Data Export tool to create a Web Portal export zip
@@ -91,16 +91,16 @@ web portal.
 
 4. Build the Solr app: 
    ```console
-   specify:~/webportal-installer$ make clean && make
+   specify@wp:~/webportal-installer$ make clean && make
    ```
 
-5. Copy the `webportal-solr.service` systemd unit file to
+5. Copy the provided `webportal-solr.service` systemd unit file to
    `/etc/systemd/system/webportal-solr.service` and reload the
    systemd daemon.
    
    ```console
-   specify:~/webportal-installer$ sudo cp webportal-solr.service /etc/systemd/system/
-   specify:~/webportal-installer$ sudo systemctl daemon-reload
+   root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-solr.service /etc/systemd/system/
+   root@wp:~# systemctl daemon-reload
    ```
    N.B. this file will require changes if the `webportal-installer` is
    in a different location.
@@ -108,32 +108,35 @@ web portal.
 6. Start and enable the service:
 
     ```console
-    specify:~/webportal-installer$ sudo systemctl start webportal-solr.service
-    specify:~/webportal-installer$ sudo systemctl enable webportal-solr.service
+    root@wp:~# systemctl start webportal-solr.service
+    root@wp:~# systemctl enable webportal-solr.service
     ```
     
 7. Load the Specify data into Solr:
 
     ```console
-    specify:~/webportal-installer$ make load-data
+    specify@wp:~/webportal-installer$ make load-data
     ```
 
-
-  When completing this step you will receive output similar to the following: 
-
-    ```
-    {
-    "responseHeader":{
-    "status":0,
-    "QTime":1153}}
-    ```
+  When completing this step you will receive output consisting of
+  blocks of JSON. This is normal unless one of the blocks includes an
+  error message.
   
 8. The Web Portal is now running and can be tested by visiting the
    server's address with a web browser.
    
 9. Be aware that the Solr admin page is available by default on
    port 8983. This can be useful for troubleshooting but should be
-   blocked in production using a firewall rule.
+   blocked in production using firewall rules such as the following:
+   
+    ```console
+    root@wp:~# ufw default deny incoming
+    root@wp:~# ufw default allow outgoing
+    root@wp:~# ufw allow ssh
+    root@wp:~# ufw allow http
+    root@wp:~# ufw --force enable
+    ```
+   
 
 Customization
 -------------
@@ -148,7 +151,7 @@ To update the Specify data in the webportal follow these steps.
 1. Stop the webportal-solr systemd service:
 
     ```console
-    specify:~/webportal-installer$ sudo systemctl stop webportal-solr.service
+    root@wp:~# systemctl stop webportal-solr.service
     ```
 
 2. Use the Specify Data Export tool to create a Web Portal export zip
@@ -167,19 +170,19 @@ To update the Specify data in the webportal follow these steps.
 4. Build the Solr app: 
 
     ```console
-    specify:~/webportal-installer$ make clean && make
+    specify@wp:~/webportal-installer$ make clean && make
     ```
 
 6. Start the webportal-solr service:
 
     ```console
-    specify:~/webportal-installer$ sudo systemctl start webportal-solr.service
+    root@wp:~# systemctl start webportal-solr.service
     ```
     
 7. Load the Specify data into Solr:
 
     ```console
-    specify:~/webportal-installer$ make load-data
+    specify@wp:~/webportal-installer$ make load-data
     ```
     
 8. The Web Portal is now running and can be tested by visiting the

From ef0f3f9918d8d7d7b5675519a3ac550d2441892a Mon Sep 17 00:00:00 2001
From: Ben Anhalt <anhalt@ku.edu>
Date: Thu, 30 Apr 2020 15:44:23 -0500
Subject: [PATCH 09/38] fix indentation.

---
 README.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 3d2cd3c..b1a3216 100644
--- a/README.md
+++ b/README.md
@@ -114,13 +114,13 @@ web portal.
     
 7. Load the Specify data into Solr:
 
-    ```console
-    specify@wp:~/webportal-installer$ make load-data
-    ```
-
-  When completing this step you will receive output consisting of
-  blocks of JSON. This is normal unless one of the blocks includes an
-  error message.
+   ```console
+   specify@wp:~/webportal-installer$ make load-data
+   ```
+   
+   When completing this step you will receive output consisting of
+   blocks of JSON. This is normal unless one of the blocks includes an
+   error message.
   
 8. The Web Portal is now running and can be tested by visiting the
    server's address with a web browser.

From 768609df16258677ceebb0e8c4cb031d205319be Mon Sep 17 00:00:00 2001
From: Ben Anhalt <anhalt@ku.edu>
Date: Fri, 8 May 2020 11:22:53 -0500
Subject: [PATCH 10/38] add ulimit adjustments

---
 webportal-solr.service | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/webportal-solr.service b/webportal-solr.service
index 7dceb7d..52b1640 100644
--- a/webportal-solr.service
+++ b/webportal-solr.service
@@ -8,6 +8,8 @@ Group=specify
 PIDFile=/home/specify/webportal-installer/build/bin/solr-8983.pid
 ExecStart=/home/specify/webportal-installer/build/bin/solr start
 ExecStop=/home/specify/webportal-installer/build/bin/solr stop
+LimitNOFILE=65000
+LimitNPROC=65000
 Restart=always
 
 [Install]

From 9af1ceac42f874b1d04fd9f07485521ba256f780 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 5 May 2020 15:50:20 -0500
Subject: [PATCH 11/38] add helper commands.

---
 Makefile   | 24 ++++++++++++++++--------
 build.make |  6 +++---
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index f16a0cd..94a2bf5 100644
--- a/Makefile
+++ b/Makefile
@@ -11,15 +11,9 @@ export TOPDIR := $(shell pwd)
 
 all: build
 
-load-data: build
+load-data:
 	for core in build/cores/* ; do \
-		corename=`basename "$$core"` ; \
-		curl -X POST "http://localhost:8983/solr/$$corename/update?commit=true" \
-	 		-d '{ "delete": {"query":"*:*"} }' \
-			-H 'Content-Type: application/json' ; \
-		curl "http://localhost:8983/solr/$$corename/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
-			--data-binary @$$core/PortalFiles/PortalData.csv \
-			-H 'Content-type:application/csv' ; \
+		make $$core/load-data; \
 	done
 
 clean:
@@ -45,3 +39,17 @@ $(SOLR_DIST): $(SOLR_DIST).tgz
 	tar -zxf $<
 
 $(SOLR_DIST)/%: $(SOLR_DIST)
+
+build/cores/%/rebuild-webapp:
+	make CORENAME=$* -B -f $(TOPDIR)/core.make -C build/cores/$* webapp
+
+build/cores/%/unpack:
+	unzip -o -d build/cores/$* specify_exports/$*.zip
+
+build/cores/%/load-data:
+	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
+		-d '{ "delete": {"query":"*:*"} }' \
+		-H 'Content-Type: application/json'
+	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
+		--data-binary @build/cores/$*/PortalFiles/PortalData.csv \
+		-H 'Content-type:application/csv'
diff --git a/build.make b/build.make
index 27f1549..70ec952 100644
--- a/build.make
+++ b/build.make
@@ -42,15 +42,15 @@ html/index.html: $(TOPDIR)/make_toplevel_index.py $(TOPDIR)/index_skel.html core
 		cores/*/webapp/resources/config/settings.json > $@
 
 html: cores
-	# Put the webapps in the html folder.
+	# Link the webapps into the html folder.
 	mkdir -p html
 	for core in cores/* ; do \
-		cp -r $$core/webapp html/`basename $$core` ; \
+		ln -sfT ../$$core/webapp html/`basename $$core` ; \
 	done
 
 link-cores: cores
 	# Link the cores into the solr home folder.
 	for core in cores/* ; do \
-		ln -s ../../$$core server/solr/ ; \
+		ln -sf ../../$$core server/solr/ ; \
 	done
 

From 4ecdc887abe4b522925be3c6b65543fe16f0cf22 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Mon, 11 May 2020 16:28:52 -0500
Subject: [PATCH 12/38] track update times of collections.

---
 Makefile               |  5 +++--
 index_skel.html        | 34 ++++++++++++++++++++++++++++++----
 make_toplevel_index.py |  3 ---
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 94a2bf5..c7d6e50 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ all: build
 
 load-data:
 	for core in build/cores/* ; do \
-		make $$core/load-data; \
+		make $$core/webapp/load-data; \
 	done
 
 clean:
@@ -46,10 +46,11 @@ build/cores/%/rebuild-webapp:
 build/cores/%/unpack:
 	unzip -o -d build/cores/$* specify_exports/$*.zip
 
-build/cores/%/load-data:
+build/cores/%/webapp/load-data: build/cores/%/unpack specify_exports/%.zip
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
 		-H 'Content-Type: application/json'
 	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
 		--data-binary @build/cores/$*/PortalFiles/PortalData.csv \
 		-H 'Content-type:application/csv'
+	date > $@
diff --git a/index_skel.html b/index_skel.html
index 4176e36..7388d19 100644
--- a/index_skel.html
+++ b/index_skel.html
@@ -3,15 +3,41 @@
   <head>
     <title>Specify Web Portal</title>
     <meta name="robots" content="noindex" />
+    <style>
+      body {
+          font-size: 100%;
+          font-family: sans-serif;
+      }
+      #collections span {
+          font-size: 66%;
+          margin-left: 2em;
+      }
+      #collections span::before {
+          content: "last update: "
+      }
+    </style>
   </head>
   <body>
     <h2>The following collections are available:</h2>
     <ul id="collections">
       <li>Nothing here</li>
     </ul>
-    <p>
-      Last update:
-      <span id="update-time">N/A</span>
-    </p>
+    <script type="text/javascript">
+      document.querySelectorAll('#collections li').forEach(function(li) {
+          var a = li.querySelector('a');
+          if (a == null) return;
+
+          var xhr = new XMLHttpRequest();
+          xhr.open('GET', a.getAttribute('href') + '/load-data');
+          xhr.onload = function() {
+              if (xhr.status === 200) {
+                  var update = document.createElement('span');
+                  update.textContent = xhr.responseText;
+                  li.append(update);
+              }
+          };
+          xhr.send();
+      });
+    </script>
   </body>
 </html>
diff --git a/make_toplevel_index.py b/make_toplevel_index.py
index 2343f42..d11f597 100644
--- a/make_toplevel_index.py
+++ b/make_toplevel_index.py
@@ -58,7 +58,4 @@ def splitall(path):
     a.set('href', core_dir)
     a.text = core_name
 
-update_time = skel.find('.//span[@id="update-time"]')
-update_time.text = datetime.now().isoformat()
-
 skel.write(sys.stdout)

From ae121861853b4ebca05ab3ecabb1b3de930563e9 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 12:05:12 -0500
Subject: [PATCH 13/38] wip.

---
 Makefile  | 84 ++++++++++++++++++++++++++++++++++++++++++-------------
 core.make |  2 +-
 2 files changed, 66 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index c7d6e50..524a961 100644
--- a/Makefile
+++ b/Makefile
@@ -1,52 +1,98 @@
 # Mirror for downloading Apache Solr.
 SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr
 
+export PYTHON := python2
+
 # Use latest available version of Solr 4.
-export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr_vers.py)
+export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py)
 
 export SOLR_DIST := solr-$(SOLR_VERSION)
 export TOPDIR := $(shell pwd)
 
-.PHONY: all clean realclean load-data
+INPUTS := $(wildcard specify_exports/*.zip)
+COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
+CORES := $(addprefix build/cores/, $(COLLECTIONS))
+PORTALFILES := $(addsuffix /PortalFiles, $(CORES))
+WEBAPPS := $(addsuffix /webapp, $(CORES))
+CCORES := $(addsuffix /core, $(CORES))
+
+.SILENT:
+
+.PHONY: usage
+usage:
+	@echo Usage:
+	@echo make build-all  -- Do everything.
 
-all: build
+.PHONY: build-all build-cores build-html load-data
+build-all: build-cores build-html
+build-cores: $(CCORES)
+build-html: $(WEBAPPS) build/html/index.html
+load-data: $(addprefix load-data-, $(COLLECTIONS))
 
-load-data:
-	for core in build/cores/* ; do \
-		make $$core/webapp/load-data; \
-	done
+.PHONY: start-solr stop-solr
+start-solr:
+	build/bin/solr start
 
+stop-solr:
+	build/bin/solr stop
+
+.PHONY: clean realclean
 clean:
 	rm -rf build/
 
 realclean: clean
-	rm -rf solr-* unpacked-war
+	rm -rf solr-*
 
-build: $(SOLR_DIST) build.make specify_exports specify_exports/*.zip
+build: $(SOLR_DIST)
+	@echo Copying solr to build directory.
 	cp -r $(SOLR_DIST)/ build
-	# The following file gets replaced by a patched version:
-	rm build/server/solr-webapp/webapp/WEB-INF/web.xml
-	$(MAKE) -f $(TOPDIR)/build.make -C build
+	$(PYTHON) patch_web_xml.py $(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml > $@/server/solr-webapp/webapp/WEB-INF/web.xml
+
+build/cores: | build
+	mkdir build/cores
+
+build/cores/%/PortalFiles: specify_exports/%.zip | build/cores
+	@echo Extracting $@.
+	unzip -DD -qq -o -d build/cores/$* $^
+
+build/cores/%/core: build/cores/%/PortalFiles
+	@echo Generating $@.
+	$(MAKE) CORENAME=$* -f $(TOPDIR)/core.make -C build/cores/$* core
+	ln -sfT ../../cores/$*/core build/server/solr/$*
+	touch $@
+
+build/cores/%/webapp: build/cores/%/PortalFiles | build/html
+	@echo Generating $@.
+	$(MAKE) CORENAME=$* -B -f $(TOPDIR)/core.make -C build/cores/$* webapp
+	ln -sfT ../../$@ build/html/$*
 	touch $@
 
+build/html: | build
+	mkdir build/html
+
+build/html/index.html: $(WEBAPPS) | build/html
+	@echo Generating $@.
+	$(PYTHON) make_toplevel_index.py index_skel.html \
+		build/cores/*/webapp/resources/config/settings.json > $@
+
 $(SOLR_DIST).tgz:
-	# Fetching Solr distribution tar ball.
+	@echo Fetching Solr distribution tar ball.
 	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
 
 $(SOLR_DIST): $(SOLR_DIST).tgz
-	# Unpacking Solr distribution.
+	@echo Unpacking Solr distribution.
 	rm -rf $@
 	tar -zxf $<
 
 $(SOLR_DIST)/%: $(SOLR_DIST)
 
-build/cores/%/rebuild-webapp:
-	make CORENAME=$* -B -f $(TOPDIR)/core.make -C build/cores/$* webapp
 
-build/cores/%/unpack:
-	unzip -o -d build/cores/$* specify_exports/$*.zip
+.PHONY: load-data-%
+load-data-%: build/cores/%/webapp/load-data ;
 
-build/cores/%/webapp/load-data: build/cores/%/unpack specify_exports/%.zip
+.PRECIOUS: build/cores/%/webapp/load-data
+build/cores/%/webapp/load-data: build/cores/%/PortalFiles
+	@echo Loading data into $*.
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
 		-H 'Content-Type: application/json'
diff --git a/core.make b/core.make
index aaa8b39..24c25cc 100644
--- a/core.make
+++ b/core.make
@@ -61,4 +61,4 @@ core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE)
 	echo 'dataDir=data' > core/core.properties
 	echo 'name=$(CORENAME)' >> core/core.properties
 	echo 'config=conf/solrconfig.xml' >> core/core.properties
-	mkdir core/data
+	mkdir -p core/data

From 2b1ea946d1a7568c9a01cd88fb6f8df5b6f9cfa4 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 12:25:29 -0500
Subject: [PATCH 14/38] wip.

---
 Makefile  | 111 ++++++++++++++++++++++++++++++++++++++----------------
 core.make |  64 -------------------------------
 2 files changed, 78 insertions(+), 97 deletions(-)
 delete mode 100644 core.make

diff --git a/Makefile b/Makefile
index 524a961..06b0fec 100644
--- a/Makefile
+++ b/Makefile
@@ -1,22 +1,29 @@
 # Mirror for downloading Apache Solr.
 SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr
 
-export PYTHON := python2
+# Use 'schema.xml' if solr will be used to create the core
+# Use 'managed-schema' if pre-configuring core
+SCHEMA_FILE := managed-schema
+
+#location of default settings files in solr dist
+DEFAULT_SETS := server/solr/configsets/_default
+
+
+PYTHON := python2
 
 # Use latest available version of Solr 4.
-export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py)
+SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py)
 
-export SOLR_DIST := solr-$(SOLR_VERSION)
-export TOPDIR := $(shell pwd)
+SOLR_DIST := solr-$(SOLR_VERSION)
 
 INPUTS := $(wildcard specify_exports/*.zip)
 COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
 CORES := $(addprefix build/cores/, $(COLLECTIONS))
 PORTALFILES := $(addsuffix /PortalFiles, $(CORES))
-WEBAPPS := $(addsuffix /webapp, $(CORES))
-CCORES := $(addsuffix /core, $(CORES))
+WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
+CCORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
-.SILENT:
+# .SILENT:
 
 .PHONY: usage
 usage:
@@ -29,11 +36,11 @@ build-cores: $(CCORES)
 build-html: $(WEBAPPS) build/html/index.html
 load-data: $(addprefix load-data-, $(COLLECTIONS))
 
-.PHONY: start-solr stop-solr
-start-solr:
+.PHONY: solr-start solr-stop
+solr-start:
 	build/bin/solr start
 
-stop-solr:
+solr-stop:
 	build/bin/solr stop
 
 .PHONY: clean realclean
@@ -44,55 +51,93 @@ realclean: clean
 	rm -rf solr-*
 
 build: $(SOLR_DIST)
-	@echo Copying solr to build directory.
+	@printf "\n\n### Copying solr to build directory.\n\n"
 	cp -r $(SOLR_DIST)/ build
-	$(PYTHON) patch_web_xml.py $(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml > $@/server/solr-webapp/webapp/WEB-INF/web.xml
+	$(PYTHON) patch_web_xml.py \
+		$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml\
+		> $@/server/solr-webapp/webapp/WEB-INF/web.xml
 
 build/cores: | build
+	@printf "\n\n"
 	mkdir build/cores
 
+.PRECIOUS: build/cores/%/PortalFiles
 build/cores/%/PortalFiles: specify_exports/%.zip | build/cores
-	@echo Extracting $@.
+	@printf "\n\n### Extracting $@.\n\n"
 	unzip -DD -qq -o -d build/cores/$* $^
 
-build/cores/%/core: build/cores/%/PortalFiles
-	@echo Generating $@.
-	$(MAKE) CORENAME=$* -f $(TOPDIR)/core.make -C build/cores/$* core
-	ln -sfT ../../cores/$*/core build/server/solr/$*
-	touch $@
+build/server/solr/%: build/cores/%/SolrFldSchema.xml | build
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@/conf $@/data
+
+	$(PYTHON) patch_solrconfig_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml \
+		> $@/conf/solrconfig.xml
+
+	$(PYTHON) patch_schema_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \
+		build/cores/$*/SolrFldSchema.xml \
+		> $@/conf/$(SCHEMA_FILE)
+
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt $@/conf/
+	cp -r $(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ $@/conf/
+
+	echo 'dataDir=data' > $@/core.properties
+	echo 'name=$*' >> $@/core.properties
+	echo 'config=conf/solrconfig.xml' >> $@/core.properties
 
-build/cores/%/webapp: build/cores/%/PortalFiles | build/html
-	@echo Generating $@.
-	$(MAKE) CORENAME=$* -B -f $(TOPDIR)/core.make -C build/cores/$* webapp
-	ln -sfT ../../$@ build/html/$*
 	touch $@
 
+build/cores/%/SolrFldSchema.xml: build/cores/%/PortalFiles
+	@printf "\n\n### Generating $@.\n\n"
+	echo '<?xml version="1.0" encoding="UTF-8" ?>' > $@
+	echo "<fields>" >> $@
+	cat $</SolrFldSchema.xml >> $@
+	echo "</fields>" >> $@
+
 build/html: | build
+	@printf "\n\n"
 	mkdir build/html
 
 build/html/index.html: $(WEBAPPS) | build/html
-	@echo Generating $@.
+	@printf "\n\n### Generating $@.\n\n"
 	$(PYTHON) make_toplevel_index.py index_skel.html \
-		build/cores/*/webapp/resources/config/settings.json > $@
+		$(addsuffix /resources/config/settings.json, $(WEBAPPS)) \
+		> $@
+
+build/html/%: build/cores/%/PortalFiles | build/html
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	cp -r PortalApp/* $@
+	$(PYTHON) make_fldmodel_json.py \
+		build/cores/$*/PortalFiles/*flds.json \
+		custom_settings/$*/fldmodel.json \
+		> $@/resources/config/fldmodel.json
+	$(PYTHON) patch_settings_json.py \
+		PortalApp/resources/config/settings.json \
+		custom_settings/$*/settings.json \
+		$* \
+		build/cores/$*/PortalFiles/*Setting.json \
+		> $@/resources/config/settings.json
+	touch $@
 
 $(SOLR_DIST).tgz:
-	@echo Fetching Solr distribution tar ball.
+	@printf "\n\n### Fetching Solr distribution tar ball.\n\n"
 	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
 
 $(SOLR_DIST): $(SOLR_DIST).tgz
-	@echo Unpacking Solr distribution.
+	@printf "\n\n### Unpacking Solr distribution.\n\n"
 	rm -rf $@
 	tar -zxf $<
 
-$(SOLR_DIST)/%: $(SOLR_DIST)
-
-
 .PHONY: load-data-%
-load-data-%: build/cores/%/webapp/load-data ;
+load-data-%: build/html/%/load-data ;
 
-.PRECIOUS: build/cores/%/webapp/load-data
-build/cores/%/webapp/load-data: build/cores/%/PortalFiles
-	@echo Loading data into $*.
+.PRECIOUS: build/html/%/load-data
+build/html/%/load-data: build/cores/%/PortalFiles
+	@printf "\n\n### Loading data into $*.\n\n"
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
 		-H 'Content-Type: application/json'
diff --git a/core.make b/core.make
deleted file mode 100644
index 24c25cc..0000000
--- a/core.make
+++ /dev/null
@@ -1,64 +0,0 @@
-# 
-# Use 'schema.xml' if solr will be used to create the core
-# Use 'managed-schema' if pre-configuring core 
-SCHEMA_FILE := managed-schema
-#location of default settings files in solr dist
-DEFAULT_SETS := server/solr/configsets/_default
-
-all: webapp core
-
-# The custom setting file should really be a dependency here,
-# but I don't know how to handle the case that it doesn't exist.
-settings.json: $(TOPDIR)/patch_settings_json.py \
-		$(TOPDIR)/PortalApp/resources/config/settings.json
-	# Patch web app settings.
-	python2 $^ $(TOPDIR)/custom_settings/$(CORENAME)/settings.json \
-		 $(CORENAME) PortalFiles/*Setting.json > $@
-
-SolrFldSchema.xml: PortalFiles/SolrFldSchema.xml
-	# Add a root element to the schema field list.
-	echo '<?xml version="1.0" encoding="UTF-8" ?>' > $@
-	echo "<fields>" >> $@
-	cat $< >> $@
-	echo "</fields>" >> $@
-
-$(SCHEMA_FILE): $(TOPDIR)/patch_schema_xml.py \
-		$(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \
-		SolrFldSchema.xml
-	# Patching Solr schema with fields from Specify export.
-	python2 $^ > $@
-
-solrconfig.xml: $(TOPDIR)/patch_solrconfig_xml.py \
-		$(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml
-	# Patching Solr config for use with Specify.
-	python2 $^ > $@
-
-# The custom setting file should really be a dependency here,
-# but I don't know how to handle the case that it doesn't exist.
-fldmodel.json: $(TOPDIR)/make_fldmodel_json.py PortalFiles/*flds.json
-	# Patch any custom settings into the field definitions.
-	python2 $^ $(TOPDIR)/custom_settings/$(CORENAME)/fldmodel.json > $@
-
-webapp: $(TOPDIR)/PortalApp settings.json fldmodel.json
-	# Setup web app instance for this core.
-	mkdir -p webapp
-	cp -r $(TOPDIR)/PortalApp/* webapp/
-
-	# Copy WebPortal field specs into place.
-	cp fldmodel.json webapp/resources/config/fldmodel.json
-
-	# Copy patched settings into place.
-	cp settings.json webapp/resources/config/
-
-core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE)
-	# Setup solr-home subdir for this core.
-	mkdir -p core/conf
-	cp solrconfig.xml $(SCHEMA_FILE)  core/conf/
-	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt core/conf/
-	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt core/conf/
-	cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt core/conf/
-	cp -r $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ core/conf/
-	echo 'dataDir=data' > core/core.properties
-	echo 'name=$(CORENAME)' >> core/core.properties
-	echo 'config=conf/solrconfig.xml' >> core/core.properties
-	mkdir -p core/data

From 8047d9b081080b3970a0bc3283718e6097fdccfc Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 13:00:55 -0500
Subject: [PATCH 15/38] renaming.

---
 Makefile | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 06b0fec..cb0e7a5 100644
--- a/Makefile
+++ b/Makefile
@@ -18,10 +18,9 @@ SOLR_DIST := solr-$(SOLR_VERSION)
 
 INPUTS := $(wildcard specify_exports/*.zip)
 COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
-CORES := $(addprefix build/cores/, $(COLLECTIONS))
-PORTALFILES := $(addsuffix /PortalFiles, $(CORES))
+PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
 WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
-CCORES := $(addprefix build/server/solr/, $(COLLECTIONS))
+SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
 # .SILENT:
 
@@ -32,7 +31,7 @@ usage:
 
 .PHONY: build-all build-cores build-html load-data
 build-all: build-cores build-html
-build-cores: $(CCORES)
+build-cores: $(SOLR_CORES)
 build-html: $(WEBAPPS) build/html/index.html
 load-data: $(addprefix load-data-, $(COLLECTIONS))
 
@@ -57,16 +56,16 @@ build: $(SOLR_DIST)
 		$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml\
 		> $@/server/solr-webapp/webapp/WEB-INF/web.xml
 
-build/cores: | build
+build/col: | build
 	@printf "\n\n"
-	mkdir build/cores
+	mkdir build/col
 
-.PRECIOUS: build/cores/%/PortalFiles
-build/cores/%/PortalFiles: specify_exports/%.zip | build/cores
+.PRECIOUS: build/col/%/PortalFiles
+build/col/%/PortalFiles: specify_exports/%.zip | build/col
 	@printf "\n\n### Extracting $@.\n\n"
-	unzip -DD -qq -o -d build/cores/$* $^
+	unzip -DD -qq -o -d build/col/$* $^
 
-build/server/solr/%: build/cores/%/SolrFldSchema.xml | build
+build/server/solr/%: build/col/%/SolrFldSchema.xml | build
 	@printf "\n\n### Generating $@.\n\n"
 	mkdir -p $@/conf $@/data
 
@@ -76,7 +75,7 @@ build/server/solr/%: build/cores/%/SolrFldSchema.xml | build
 
 	$(PYTHON) patch_schema_xml.py \
 		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \
-		build/cores/$*/SolrFldSchema.xml \
+		build/col/$*/SolrFldSchema.xml \
 		> $@/conf/$(SCHEMA_FILE)
 
 	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt $@/conf/
@@ -90,7 +89,7 @@ build/server/solr/%: build/cores/%/SolrFldSchema.xml | build
 
 	touch $@
 
-build/cores/%/SolrFldSchema.xml: build/cores/%/PortalFiles
+build/col/%/SolrFldSchema.xml: build/col/%/PortalFiles
 	@printf "\n\n### Generating $@.\n\n"
 	echo '<?xml version="1.0" encoding="UTF-8" ?>' > $@
 	echo "<fields>" >> $@
@@ -107,19 +106,19 @@ build/html/index.html: $(WEBAPPS) | build/html
 		$(addsuffix /resources/config/settings.json, $(WEBAPPS)) \
 		> $@
 
-build/html/%: build/cores/%/PortalFiles | build/html
+build/html/%: build/col/%/PortalFiles | build/html
 	@printf "\n\n### Generating $@.\n\n"
 	mkdir -p $@
 	cp -r PortalApp/* $@
 	$(PYTHON) make_fldmodel_json.py \
-		build/cores/$*/PortalFiles/*flds.json \
+		build/col/$*/PortalFiles/*flds.json \
 		custom_settings/$*/fldmodel.json \
 		> $@/resources/config/fldmodel.json
 	$(PYTHON) patch_settings_json.py \
 		PortalApp/resources/config/settings.json \
 		custom_settings/$*/settings.json \
 		$* \
-		build/cores/$*/PortalFiles/*Setting.json \
+		build/col/$*/PortalFiles/*Setting.json \
 		> $@/resources/config/settings.json
 	touch $@
 
@@ -136,12 +135,12 @@ $(SOLR_DIST): $(SOLR_DIST).tgz
 load-data-%: build/html/%/load-data ;
 
 .PRECIOUS: build/html/%/load-data
-build/html/%/load-data: build/cores/%/PortalFiles
+build/html/%/load-data: build/col/%/PortalFiles
 	@printf "\n\n### Loading data into $*.\n\n"
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
 		-H 'Content-Type: application/json'
 	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
-		--data-binary @build/cores/$*/PortalFiles/PortalData.csv \
+		--data-binary @build/col/$*/PortalFiles/PortalData.csv \
 		-H 'Content-type:application/csv'
 	date > $@

From e34b2e0a2634d287568b178ce34cc74ac7cc1469 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 13:07:58 -0500
Subject: [PATCH 16/38] section headings.

---
 Makefile | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index cb0e7a5..f12b5b0 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,6 @@ SCHEMA_FILE := managed-schema
 #location of default settings files in solr dist
 DEFAULT_SETS := server/solr/configsets/_default
 
-
 PYTHON := python2
 
 # Use latest available version of Solr 4.
@@ -22,25 +21,22 @@ PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
 WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
 SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
-# .SILENT:
+###### Usage Information #####
 
 .PHONY: usage
 usage:
 	@echo Usage:
 	@echo make build-all  -- Do everything.
 
+##### Main build targets ####
+
 .PHONY: build-all build-cores build-html load-data
 build-all: build-cores build-html
 build-cores: $(SOLR_CORES)
 build-html: $(WEBAPPS) build/html/index.html
 load-data: $(addprefix load-data-, $(COLLECTIONS))
 
-.PHONY: solr-start solr-stop
-solr-start:
-	build/bin/solr start
-
-solr-stop:
-	build/bin/solr stop
+##### Cleaning targets #####
 
 .PHONY: clean realclean
 clean:
@@ -49,6 +45,18 @@ clean:
 realclean: clean
 	rm -rf solr-*
 
+##### Common building steps #####
+
+$(SOLR_DIST).tgz:
+	@printf "\n\n### Fetching Solr distribution tar ball.\n\n"
+	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
+
+$(SOLR_DIST): $(SOLR_DIST).tgz
+	@printf "\n\n### Unpacking Solr distribution.\n\n"
+	rm -rf $@
+	tar -zxf $<
+	touch $@
+
 build: $(SOLR_DIST)
 	@printf "\n\n### Copying solr to build directory.\n\n"
 	cp -r $(SOLR_DIST)/ build
@@ -65,6 +73,8 @@ build/col/%/PortalFiles: specify_exports/%.zip | build/col
 	@printf "\n\n### Extracting $@.\n\n"
 	unzip -DD -qq -o -d build/col/$* $^
 
+##### Solr core building #####
+
 build/server/solr/%: build/col/%/SolrFldSchema.xml | build
 	@printf "\n\n### Generating $@.\n\n"
 	mkdir -p $@/conf $@/data
@@ -96,6 +106,8 @@ build/col/%/SolrFldSchema.xml: build/col/%/PortalFiles
 	cat $</SolrFldSchema.xml >> $@
 	echo "</fields>" >> $@
 
+##### Website building #####
+
 build/html: | build
 	@printf "\n\n"
 	mkdir build/html
@@ -122,14 +134,7 @@ build/html/%: build/col/%/PortalFiles | build/html
 		> $@/resources/config/settings.json
 	touch $@
 
-$(SOLR_DIST).tgz:
-	@printf "\n\n### Fetching Solr distribution tar ball.\n\n"
-	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
-
-$(SOLR_DIST): $(SOLR_DIST).tgz
-	@printf "\n\n### Unpacking Solr distribution.\n\n"
-	rm -rf $@
-	tar -zxf $<
+##### Loading data #####
 
 .PHONY: load-data-%
 load-data-%: build/html/%/load-data ;

From dacadd7dd4b9d4a4028e9c807d1ce51bd72afa2d Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 13:32:31 -0500
Subject: [PATCH 17/38] integrate setting_templates building.

---
 Makefile   | 24 ++++++++++++++++++++---
 build.make | 56 ------------------------------------------------------
 2 files changed, 21 insertions(+), 59 deletions(-)
 delete mode 100644 build.make

diff --git a/Makefile b/Makefile
index f12b5b0..251a02d 100644
--- a/Makefile
+++ b/Makefile
@@ -19,6 +19,7 @@ INPUTS := $(wildcard specify_exports/*.zip)
 COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
 PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
 WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
+SETTING_TEMPLATES := $(addprefix build/setting_templates/$c, $(COLLECTIONS))
 SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
 ###### Usage Information #####
@@ -28,12 +29,13 @@ usage:
 	@echo Usage:
 	@echo make build-all  -- Do everything.
 
-##### Main build targets ####
+##### Main build targets #####
 
 .PHONY: build-all build-cores build-html load-data
-build-all: build-cores build-html
+build-all: build-cores build-html build-setting-templates
 build-cores: $(SOLR_CORES)
 build-html: $(WEBAPPS) build/html/index.html
+build-setting-templates: $(SETTING_TEMPLATES)
 load-data: $(addprefix load-data-, $(COLLECTIONS))
 
 ##### Cleaning targets #####
@@ -140,7 +142,7 @@ build/html/%: build/col/%/PortalFiles | build/html
 load-data-%: build/html/%/load-data ;
 
 .PRECIOUS: build/html/%/load-data
-build/html/%/load-data: build/col/%/PortalFiles
+build/html/%/load-data: build/col/%/PortalFiles | build/html/%
 	@printf "\n\n### Loading data into $*.\n\n"
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
@@ -149,3 +151,19 @@ build/html/%/load-data: build/col/%/PortalFiles
 		--data-binary @build/col/$*/PortalFiles/PortalData.csv \
 		-H 'Content-type:application/csv'
 	date > $@
+
+##### Settings templates #####
+
+build/setting_templates: | build
+	@printf "\n\n"
+	mkdir $@
+
+build/setting_templates/%: build/html/% | build/setting_templates
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	$(PYTHON) make_settings_template.py \
+		PortalApp/resources/config/settings.json \
+		> $@/settings.json
+	$(PYTHON) make_fields_template.py \
+		build/html/$*/resources/config/fldmodel.json \
+		> $@/fldmodel.json
diff --git a/build.make b/build.make
deleted file mode 100644
index 70ec952..0000000
--- a/build.make
+++ /dev/null
@@ -1,56 +0,0 @@
-
-.PHONY: link-cores
-
-all: setting_templates html html/index.html link-cores server/solr-webapp/webapp/WEB-INF/web.xml
-
-
-server/solr-webapp/webapp/WEB-INF/web.xml: $(TOPDIR)/patch_web_xml.py \
-					   $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml
-	# Patching solr server app for cross-domain access to enable extjs ajax stores to POST solr query params.
-	python2 $^ > $@
-
-
-cores: $(TOPDIR)/core.make $(TOPDIR)/$(SOLR_DIST) \
-		 $(TOPDIR)/specify_exports  $(TOPDIR)/specify_exports/*.zip
-	# We build a Solr core and webapp instance for
-	# each subdir in specify_exports.
-	rm -rf cores
-	for zipfile in $(TOPDIR)/specify_exports/*.zip ; do \
-		zipfile_name=`basename "$$zipfile"` ; \
-		corename="$${zipfile_name%.*}" ; \
-		mkdir -p "cores/$$corename" ; \
-		unzip -d "cores/$$corename" "$$zipfile" ; \
-		$(MAKE) CORENAME="$$corename" -f $(TOPDIR)/core.make -C "cores/$$corename" ; \
-	done
-
-setting_templates: $(TOPDIR)/make_settings_template.py $(TOPDIR)/make_fields_template.py cores
-	mkdir -p $@
-	for core in cores/* ; do \
-		corename=`basename "$$core"` ; \
-		mkdir -p "$@/$$corename" ; \
-		python2 $(TOPDIR)/make_settings_template.py \
-			$(TOPDIR)/PortalApp/resources/config/settings.json \
-			> "$@/$$corename/settings.json" ; \
-		python2 $(TOPDIR)/make_fields_template.py \
-			"cores/$$corename/webapp/resources/config/fldmodel.json" \
-			> "$@/$$corename/fldmodel.json" ; \
-	done
-
-
-html/index.html: $(TOPDIR)/make_toplevel_index.py $(TOPDIR)/index_skel.html cores html
-	python2 $(TOPDIR)/make_toplevel_index.py $(TOPDIR)/index_skel.html \
-		cores/*/webapp/resources/config/settings.json > $@
-
-html: cores
-	# Link the webapps into the html folder.
-	mkdir -p html
-	for core in cores/* ; do \
-		ln -sfT ../$$core/webapp html/`basename $$core` ; \
-	done
-
-link-cores: cores
-	# Link the cores into the solr home folder.
-	for core in cores/* ; do \
-		ln -sf ../../$$core server/solr/ ; \
-	done
-

From 720bf8e22cd434b3822c7a9cf5fe32dfa7768874 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 13:49:46 -0500
Subject: [PATCH 18/38] cleaning targets.

---
 Makefile | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 251a02d..8d59148 100644
--- a/Makefile
+++ b/Makefile
@@ -29,9 +29,9 @@ usage:
 	@echo Usage:
 	@echo make build-all  -- Do everything.
 
-##### Main build targets #####
+##### Main targets #####
 
-.PHONY: build-all build-cores build-html load-data
+.PHONY: build-all build-cores build-html build-setting-templates load-data
 build-all: build-cores build-html build-setting-templates
 build-cores: $(SOLR_CORES)
 build-html: $(WEBAPPS) build/html/index.html
@@ -40,13 +40,31 @@ load-data: $(addprefix load-data-, $(COLLECTIONS))
 
 ##### Cleaning targets #####
 
-.PHONY: clean realclean
-clean:
-	rm -rf build/
+.PHONY: clean-all clean-build clean-solr clean-cores clean-html clean-*-html clean-*-core clean-setting-templates
 
-realclean: clean
+clean-all: clean-build clean-solr
+
+clean-build:
+	rm -rf build
+
+clean-solr:
 	rm -rf solr-*
 
+clean-cores:
+	rm -rf $(SOLR_CORES)
+
+clean-html:
+	rm -rf build/html
+
+clean-%-html:
+	rm -rf build/html/$*
+
+clean-%-core:
+	rm -rf build/server/solr/$*
+
+clean-setting-templates:
+	rm -rf build/setting-templates
+
 ##### Common building steps #####
 
 $(SOLR_DIST).tgz:

From 0aad0836dbc7398108c90495190ad1d9893409e0 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 13:59:33 -0500
Subject: [PATCH 19/38] usage.

---
 Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 8d59148..82bf87e 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,17 @@ SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 .PHONY: usage
 usage:
 	@echo Usage:
-	@echo make build-all  -- Do everything.
+	@echo
+	@echo make build-all    -- Same as build-html build-cores.
+	@echo make build-html   -- Builds the web app part of the web portal.
+	@echo make build-cores  -- Builds the solr cores for all collections.
+	@echo make load-data    -- Loads the collection data into solr.
+	@echo make load-data-COLLECTION  -- Loads the data for COLLECTION into solr.
+	@echo
+	@echo The build-all and build-cores targets should only be used when solr is
+	@echo not running. The build-html and load-data* targets maybe used freely.
+	@echo All build and load targets will only have effect if there are changes in
+	@echo the specify_exports or custom_settings directories.
 
 ##### Main targets #####
 

From d5440c8f8694f0f214d3d67bbdc382da83fbbb90 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 14:14:59 -0500
Subject: [PATCH 20/38] fix html and rename load-data timestamp file.

---
 Makefile               | 7 ++++---
 index_skel.html        | 2 +-
 make_toplevel_index.py | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 82bf87e..7db3c1b 100644
--- a/Makefile
+++ b/Makefile
@@ -167,10 +167,10 @@ build/html/%: build/col/%/PortalFiles | build/html
 ##### Loading data #####
 
 .PHONY: load-data-%
-load-data-%: build/html/%/load-data ;
+load-data-%: build/html/%/load-timestamp.txt ;
 
-.PRECIOUS: build/html/%/load-data
-build/html/%/load-data: build/col/%/PortalFiles | build/html/%
+.PRECIOUS: build/html/%/load-timestamp.txt
+build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
 	@printf "\n\n### Loading data into $*.\n\n"
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
@@ -195,3 +195,4 @@ build/setting_templates/%: build/html/% | build/setting_templates
 	$(PYTHON) make_fields_template.py \
 		build/html/$*/resources/config/fldmodel.json \
 		> $@/fldmodel.json
+	touch $@
diff --git a/index_skel.html b/index_skel.html
index 7388d19..f92f626 100644
--- a/index_skel.html
+++ b/index_skel.html
@@ -28,7 +28,7 @@ <h2>The following collections are available:</h2>
           if (a == null) return;
 
           var xhr = new XMLHttpRequest();
-          xhr.open('GET', a.getAttribute('href') + '/load-data');
+          xhr.open('GET', a.getAttribute('href') + '/load-timestamp.txt');
           xhr.onload = function() {
               if (xhr.status === 200) {
                   var update = document.createElement('span');
diff --git a/make_toplevel_index.py b/make_toplevel_index.py
index d11f597..b1b3ae1 100644
--- a/make_toplevel_index.py
+++ b/make_toplevel_index.py
@@ -50,7 +50,7 @@ def splitall(path):
     with open(settings_file) as f:
         settings = json.load(f)
 
-    core_dir = splitall(settings_file)[1]
+    core_dir = splitall(settings_file)[2]
     core_name = settings[0]['collectionName'] or core_dir
 
     li = ElementTree.SubElement(collections, 'li')

From 48b4356e7057944daf5052e5d6a64cff8dfa10fa Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 14:17:28 -0500
Subject: [PATCH 21/38] fix.

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7db3c1b..5784093 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ INPUTS := $(wildcard specify_exports/*.zip)
 COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
 PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
 WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
-SETTING_TEMPLATES := $(addprefix build/setting_templates/$c, $(COLLECTIONS))
+SETTING_TEMPLATES := $(addprefix build/setting_templates/, $(COLLECTIONS))
 SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
 ###### Usage Information #####

From dba262ee8899dbbc2387b8cc707eec629e0a58d5 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 14:33:17 -0500
Subject: [PATCH 22/38] make custom_settings into deps for html building.

---
 Makefile               | 12 +++++++++++-
 make_fldmodel_json.py  |  6 +++++-
 patch_settings_json.py |  6 +++++-
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 5784093..30341d1 100644
--- a/Makefile
+++ b/Makefile
@@ -148,7 +148,7 @@ build/html/index.html: $(WEBAPPS) | build/html
 		$(addsuffix /resources/config/settings.json, $(WEBAPPS)) \
 		> $@
 
-build/html/%: build/col/%/PortalFiles | build/html
+build/html/%: build/col/%/PortalFiles custom_settings/%/fldmodel.json custom_settings/%/settings.json | build/html
 	@printf "\n\n### Generating $@.\n\n"
 	mkdir -p $@
 	cp -r PortalApp/* $@
@@ -164,6 +164,16 @@ build/html/%: build/col/%/PortalFiles | build/html
 		> $@/resources/config/settings.json
 	touch $@
 
+custom_settings/%/fldmodel.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
+custom_settings/%/settings.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
 ##### Loading data #####
 
 .PHONY: load-data-%
diff --git a/make_fldmodel_json.py b/make_fldmodel_json.py
index 59193cc..679e87e 100644
--- a/make_fldmodel_json.py
+++ b/make_fldmodel_json.py
@@ -27,7 +27,11 @@
 
 if os.path.isfile(sys.argv[2]):
     with open(sys.argv[2]) as f:
-        custom = json.load(f, object_pairs_hook=OrderedDict)
+        data = f.read().strip()
+        if data:
+            custom = json.loads(data, object_pairs_hook=OrderedDict)
+        else:
+            custom = []
 else:
     custom = []
 
diff --git a/patch_settings_json.py b/patch_settings_json.py
index 8314e88..81069bd 100644
--- a/patch_settings_json.py
+++ b/patch_settings_json.py
@@ -27,7 +27,11 @@
 
 if os.path.isfile(sys.argv[2]):
     with open(sys.argv[2]) as f:
-        custom_settings = json.load(f)
+        data = f.read().strip()
+        if data:
+            custom_settings = json.loads(data)
+        else:
+            custom_settings = {}
 else:
     custom_settings = {}
 

From e737ce033a0d2c9c816cd90ec2cfbec14bc6d67b Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 16:02:13 -0500
Subject: [PATCH 23/38] check solr response for success.

---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 30341d1..0957d59 100644
--- a/Makefile
+++ b/Makefile
@@ -184,10 +184,12 @@ build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
 	@printf "\n\n### Loading data into $*.\n\n"
 	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
 		-d '{ "delete": {"query":"*:*"} }' \
-		-H 'Content-Type: application/json'
+		-H 'Content-Type: application/json' \
+		| grep '"status":0'
 	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
 		--data-binary @build/col/$*/PortalFiles/PortalData.csv \
-		-H 'Content-type:application/csv'
+		-H 'Content-type:application/csv' \
+		| grep '"status":0'
 	date > $@
 
 ##### Settings templates #####

From a82c6f0b5e934b35abe625f69b67a513365793f6 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 16:06:36 -0500
Subject: [PATCH 24/38] add force-load-data target.

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Makefile b/Makefile
index 0957d59..d519498 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,8 @@ usage:
 	@echo make build-cores  -- Builds the solr cores for all collections.
 	@echo make load-data    -- Loads the collection data into solr.
 	@echo make load-data-COLLECTION  -- Loads the data for COLLECTION into solr.
+	@echo make force-load-data    -- Loads the collection data into solr even if up-to-date.
+	@echo make force-load-data-COLLECTION  -- Loads the data for COLLECTION into solr even if up-to-date.
 	@echo
 	@echo The build-all and build-cores targets should only be used when solr is
 	@echo not running. The build-html and load-data* targets maybe used freely.
@@ -47,6 +49,7 @@ build-cores: $(SOLR_CORES)
 build-html: $(WEBAPPS) build/html/index.html
 build-setting-templates: $(SETTING_TEMPLATES)
 load-data: $(addprefix load-data-, $(COLLECTIONS))
+force-load-data: $(addprefix force-load-data-, $(COLLECTIONS))
 
 ##### Cleaning targets #####
 
@@ -192,6 +195,10 @@ build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
 		| grep '"status":0'
 	date > $@
 
+force-load-data-%:
+	rm -f build/html/$*/load-timestamp.txt
+	$(MAKE) build/html/$*/load-timestamp.txt
+
 ##### Settings templates #####
 
 build/setting_templates: | build

From df083df5d92a7b98709b2d0ed159edd2bfb790c2 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 16:18:57 -0500
Subject: [PATCH 25/38] commit only after data is loaded.

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d519498..08b286f 100644
--- a/Makefile
+++ b/Makefile
@@ -185,7 +185,7 @@ load-data-%: build/html/%/load-timestamp.txt ;
 .PRECIOUS: build/html/%/load-timestamp.txt
 build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
 	@printf "\n\n### Loading data into $*.\n\n"
-	curl -X POST "http://localhost:8983/solr/$*/update?commit=true" \
+	curl -X POST "http://localhost:8983/solr/$*/update" \
 		-d '{ "delete": {"query":"*:*"} }' \
 		-H 'Content-Type: application/json' \
 		| grep '"status":0'

From 11238a7f4bd52af938464f8d30cc4b2fc980559d Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 16:41:35 -0500
Subject: [PATCH 26/38] update instructions.

---
 README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index b1a3216..d263f38 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ web portal.
 
 4. Build the Solr app: 
    ```console
-   specify@wp:~/webportal-installer$ make clean && make
+   specify@wp:~/webportal-installer$ make clean-all && make build-all
    ```
 
 5. Copy the provided `webportal-solr.service` systemd unit file to
@@ -136,18 +136,62 @@ web portal.
     root@wp:~# ufw allow http
     root@wp:~# ufw --force enable
     ```
-   
 
 Customization
 -------------
 
-TODO
+Certain customizations of the webportal are possible such as
+background images. To enable customizations, after the webportal is
+built with `make build-all`, copy the folders inside
+`webportal-installer/build/setting_templates/` into
+`webportal-installer/custom_settings/` and edit the files in each
+folder as desired. Afterwards, `make build-html` will incorporate the
+customizations into the portal without restarting any services.
+
+The provided `webportal-nginx.conf` serves the directory
+`/home/specify/custom-images/` at `/custom-images/` allowing files
+place there to be used in customization.
 
-Updating
+Data-only Updates
 --------
 
+The portal can be updated with new data from exported collections
+without restarting the Solr service if the export mappings are the
+same.
+
 To update the Specify data in the webportal follow these steps.
 
+
+2. Use the Specify Data Export tool to create a Web Portal export zip
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be updated. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
+
+
+3. Copy the zip files from the Specify Data Export into the
+   `webportal-installer/specify_exports` directory. The copied files
+   in the `specify_exports` directory must have the same names as the
+   original exports.
+   
+7. Load the Specify data into Solr:
+
+    ```console
+    specify@wp:~/webportal-installer$ make load-data
+    ```
+    
+8. Only export files newer than the last update will be loaded. The
+   target `make force-load-data` will force all collections to be
+   reloaded.
+
+Full Updates
+--------
+
+If new collections are to be added to the portal or mappings of
+existing collections change, the Solr service has to be stopped and
+the cores rebuilt.
+
+
 1. Stop the webportal-solr systemd service:
 
     ```console
@@ -170,7 +214,7 @@ To update the Specify data in the webportal follow these steps.
 4. Build the Solr app: 
 
     ```console
-    specify@wp:~/webportal-installer$ make clean && make
+    specify@wp:~/webportal-installer$ make build-all
     ```
 
 6. Start the webportal-solr service:

From c3a09f63dd8c3b70b32365aa84f2413ee5b812f8 Mon Sep 17 00:00:00 2001
From: Ben Anhalt <ben.anhalt@gmail.com>
Date: Tue, 12 May 2020 16:56:13 -0500
Subject: [PATCH 27/38] make force-load-data-% PHONY.

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 08b286f..86ffb7a 100644
--- a/Makefile
+++ b/Makefile
@@ -195,6 +195,7 @@ build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
 		| grep '"status":0'
 	date > $@
 
+.PHONY: force-load-data-%
 force-load-data-%:
 	rm -f build/html/$*/load-timestamp.txt
 	$(MAKE) build/html/$*/load-timestamp.txt

From 6b7102cb24e3e13b878d2ec00e921ae0682a1f54 Mon Sep 17 00:00:00 2001
From: Maksym Patiiuk <40512816+maxxxxxdlp@users.noreply.github.com>
Date: Wed, 16 Dec 2020 20:39:49 +0200
Subject: [PATCH 28/38] Adds a Dockerfile

---
 .dockerignore | 10 +++++++++
 .gitignore    |  4 +++-
 Dockerfile    | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 .dockerignore
 create mode 100644 Dockerfile

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..6d5988d
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,10 @@
+/solr-*
+/unpacked-war/
+/build/
+/specify_exports/*
+!/specify_exports/README
+
+/.lastupdate
+/example.crontab
+
+.DS_Store
diff --git a/.gitignore b/.gitignore
index b880781..6d5988d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,6 @@
 !/specify_exports/README
 
 /.lastupdate
-/example.crontab
\ No newline at end of file
+/example.crontab
+
+.DS_Store
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..935d065
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,59 @@
+FROM ubuntu:18.04
+
+LABEL maintainer="Specify Collections Consortium <github.com/specify>"
+
+# Get Ubuntu packages
+RUN apt-get update && apt-get -y install \
+	nginx \
+	unzip \
+	curl \
+	wget \
+	python \
+	python-lxml \
+	make \
+	lsof \
+	default-jre
+
+# Clean Up
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 999 specify && \
+    useradd -r -u 999 -g specify specify
+
+RUN mkdir -p /home/specify/webportal-installer && chown specify.specify -R /home/specify
+
+USER specify
+
+# Get Web Portal
+COPY --chown=specify:specify . /home/specify/webportal-installer
+WORKDIR /home/specify/webportal-installer
+
+EXPOSE 80
+
+USER root
+
+# Configure nginx to proxy the Solr requests and serve the static files by copying the provided webportal-nginx.conf to /etc/nginx/sites-available/
+RUN install -o root -g root -m644 ./webportal-nginx.conf /etc/nginx/sites-available/
+
+# Disable the default nginx site and enable the portal site
+RUN rm /etc/nginx/sites-enabled/default \
+	&& ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
+	&& service nginx stop
+
+# Copy the zip files from the Specify Data Export into the webportal-installer/specify_exports
+COPY data/export.zip ./specify_exports/
+
+# Build the Solr app
+RUN make clean-all && make build-all
+# TODO: test what folders we can remove from here
+
+RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log
+
+# Run Solr in foreground
+# Wait for Solr to load
+# Import data from the .zip file
+# Run Docker in foreground
+CMD ./build/bin/solr start -force -p 8983 \
+	&& sleep 10 \
+	&& make load-data \
+	&& nginx -g 'daemon off;'
\ No newline at end of file

From 98d847e47c8037c4a8087168ca502676c041df07 Mon Sep 17 00:00:00 2001
From: Maksym Patiiuk <40512816+maxxxxxdlp@users.noreply.github.com>
Date: Thu, 17 Dec 2020 00:31:07 +0200
Subject: [PATCH 29/38] Adds instructions for running the container

---
 Dockerfile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 935d065..a15ed65 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,9 @@
+# Build it like this:
+# docker build --tag webportal-service:improve-build .
+
+# Run it like this:
+# docker run -p 80:80 -v /absolute/location/of/your/export.zip:/home/specify/webportal-installer/specify_exports/export.zip webportal-service:improve-build
+
 FROM ubuntu:18.04
 
 LABEL maintainer="Specify Collections Consortium <github.com/specify>"

From a0fb946561b46f5fc3d5aaa4e6ac1ab8679f6941 Mon Sep 17 00:00:00 2001
From: Maksym Patiiuk <40512816+maxxxxxdlp@users.noreply.github.com>
Date: Thu, 17 Dec 2020 17:53:37 +0200
Subject: [PATCH 30/38] Fixes Docker Build error

---
 Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a15ed65..6575795 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,9 +46,6 @@ RUN rm /etc/nginx/sites-enabled/default \
 	&& ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
 	&& service nginx stop
 
-# Copy the zip files from the Specify Data Export into the webportal-installer/specify_exports
-COPY data/export.zip ./specify_exports/
-
 # Build the Solr app
 RUN make clean-all && make build-all
 # TODO: test what folders we can remove from here

From c7caff0b230ac98522c8e4e862f9c98420364007 Mon Sep 17 00:00:00 2001
From: Maksym Patiiuk <40512816+maxxxxxdlp@users.noreply.github.com>
Date: Thu, 17 Dec 2020 19:16:35 +0200
Subject: [PATCH 31/38] Fixes docker run errors

---
 Dockerfile | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6575795..1794b74 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,17 +46,15 @@ RUN rm /etc/nginx/sites-enabled/default \
 	&& ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
 	&& service nginx stop
 
-# Build the Solr app
-RUN make clean-all && make build-all
-# TODO: test what folders we can remove from here
-
 RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log
 
+# Build the Solr app
 # Run Solr in foreground
 # Wait for Solr to load
 # Import data from the .zip file
 # Run Docker in foreground
-CMD ./build/bin/solr start -force -p 8983 \
+CMD make clean-all && make build-all \
+	&& ./build/bin/solr start -force \
 	&& sleep 10 \
 	&& make load-data \
 	&& nginx -g 'daemon off;'
\ No newline at end of file

From 7770655374de0fd4c738819dd98516e5d9ca9f82 Mon Sep 17 00:00:00 2001
From: Maksym Patiiuk <40512816+maxxxxxdlp@users.noreply.github.com>
Date: Thu, 17 Dec 2020 21:06:16 +0200
Subject: [PATCH 32/38] Fixes occasional Solr errors

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1794b74..bdb10f6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,6 +55,6 @@ RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/n
 # Run Docker in foreground
 CMD make clean-all && make build-all \
 	&& ./build/bin/solr start -force \
-	&& sleep 10 \
-	&& make load-data \
+	&& sleep 20 \
+	&& curl -v "http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\"&escape=\&header=true" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' \
 	&& nginx -g 'daemon off;'
\ No newline at end of file

From 388c071d622b182e11feadc51a0b536175d03af2 Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Tue, 18 Apr 2023 07:45:33 -0500
Subject: [PATCH 33/38] Add configuration instructions

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index d263f38..161c789 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,12 @@ University of Kansas
 Lawrence, KS 66045 USA
 ```
 
+## Configuration Instructions
+
+Instructions on customizing the Web Portal configuration are available on the Specify Communtiy Forum:
+
+[Web Portal Configuration Instructions](https://discourse.specifysoftware.org/t/web-portal-configuration-instructions/1144)
+
 ## Developer Instructions
 
 After completing these instructions you will be able to run Specify

From 5ac3062c9d48ae9e294ca242a0e0c62a89bade72 Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Wed, 2 Aug 2023 10:34:09 -0500
Subject: [PATCH 34/38] Remove log4j references

Following @sglim2 here: https://github.com/specify/webportal-installer/commit/bed607a5c55bd66ce5f7b3bd7444efced1ec4cbd#diff-40598dffc26314434ab8958b01a8fab57075476864ba864aaf6c276486ff6313
---
 .DS_Store        | Bin 0 -> 6148 bytes
 log4j.properties |  24 ------------------------
 2 files changed, 24 deletions(-)
 create mode 100644 .DS_Store
 delete mode 100644 log4j.properties

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6
GIT binary patch
literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

literal 0
HcmV?d00001

diff --git a/log4j.properties b/log4j.properties
deleted file mode 100644
index 1d7c991..0000000
--- a/log4j.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-#  Logging level
-solr.log=/var/log/tomcat7/
-log4j.rootLogger=INFO, file, CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n
-
-#- size rotation with log cleanup.
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.MaxFileSize=4MB
-log4j.appender.file.MaxBackupIndex=9
-
-#- File to log to and log format
-log4j.appender.file.File=${solr.log}/solr.log
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.hadoop=WARN
-
-# set to INFO to enable infostream log messages
-log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF

From d7994bd2e64ace738ce4bc07ff505ad165a92687 Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Wed, 2 Aug 2023 10:35:40 -0500
Subject: [PATCH 35/38] Delete .DS_Store

---
 .DS_Store | Bin 6148 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 .DS_Store

diff --git a/.DS_Store b/.DS_Store
deleted file mode 100644
index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I


From 5cfeb388eec0e98e16b195d4c9a04c632d5c5409 Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Mon, 4 Dec 2023 14:54:55 -0600
Subject: [PATCH 36/38] Update README.md

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 161c789..4ef7296 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,9 @@ web portal.
    used, the directory paths in these instructions will need to be
    updated accordingly.
 
+   You will also need to update the `webportal-solr.service` file to use
+   the correct user and group.
+
 1. Clone the Web Portal 2.0 repository using git:
 
     ```console

From 4c3b90e541d871c0d4f58718b9ec87d26c4513eb Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Tue, 13 Feb 2024 09:12:58 -0600
Subject: [PATCH 37/38] Update Makefile

Insecure requests are now rejected causing build failure
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 86ffb7a..a374f01 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # Mirror for downloading Apache Solr.
-SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr
+SOLR_MIRROR := https://archive.apache.org/dist/lucene/solr
 
 # Use 'schema.xml' if solr will be used to create the core
 # Use 'managed-schema' if pre-configuring core

From c830d23a303ac681863ba1962deeda9aca9c0c6a Mon Sep 17 00:00:00 2001
From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com>
Date: Wed, 16 Oct 2024 14:39:50 -0500
Subject: [PATCH 38/38] Improve Dockerfile

---
 Dockerfile | 48 ++++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index bdb10f6..1f2039e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,42 +10,46 @@ LABEL maintainer="Specify Collections Consortium <github.com/specify>"
 
 # Get Ubuntu packages
 RUN apt-get update && apt-get -y install \
-	nginx \
-	unzip \
-	curl \
-	wget \
-	python \
-	python-lxml \
-	make \
-	lsof \
-	default-jre
-
-# Clean Up
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-
+        nginx \
+        unzip \
+        curl \
+        wget \
+        python \
+        python-lxml \
+        make \
+        lsof \
+        default-jre \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create a user and group for the application
 RUN groupadd -g 999 specify && \
     useradd -r -u 999 -g specify specify
 
-RUN mkdir -p /home/specify/webportal-installer && chown specify.specify -R /home/specify
+# Create the application directory and set ownership
+RUN mkdir -p /home/specify/webportal-installer && chown specify:specify -R /home/specify
 
+# Switch to the specify user
 USER specify
 
-# Get Web Portal
+# Copy the application files into the container
 COPY --chown=specify:specify . /home/specify/webportal-installer
 WORKDIR /home/specify/webportal-installer
 
+# Expose the port for the web portal
 EXPOSE 80
 
+# Switch back to root user for further configuration
 USER root
 
-# Configure nginx to proxy the Solr requests and serve the static files by copying the provided webportal-nginx.conf to /etc/nginx/sites-available/
-RUN install -o root -g root -m644 ./webportal-nginx.conf /etc/nginx/sites-available/
+# Configure nginx to proxy the Solr requests and serve the static files
+COPY webportal-nginx.conf /etc/nginx/sites-available/webportal-nginx.conf
 
 # Disable the default nginx site and enable the portal site
 RUN rm /etc/nginx/sites-enabled/default \
-	&& ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
-	&& service nginx stop
+        && ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
+        && service nginx stop
 
+# Redirect nginx logs to stdout and stderr
 RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log
 
 # Build the Solr app
@@ -53,8 +57,4 @@ RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/n
 # Wait for Solr to load
 # Import data from the .zip file
 # Run Docker in foreground
-CMD make clean-all && make build-all \
-	&& ./build/bin/solr start -force \
-	&& sleep 20 \
-	&& curl -v "http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\"&escape=\&header=true" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' \
-	&& nginx -g 'daemon off;'
\ No newline at end of file
+CMD ["sh", "-c", "make clean-all && make build-all && ./build/bin/solr start -force && sleep 20 && curl -v \"http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\\\"&escape=\\&header=true\" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' && nginx -g 'daemon off;'"]
\ No newline at end of file