Merge pull request #1789 from forumone/uat-forumone

Sync f1's uat-forumone branch

bin/xml-datasets/files/.gitignore

@@ -0,0 +1 @@
+*

bin/xml-datasets/xml-dataset-download.sh

@@ -0,0 +1,101 @@
+#!/usr/local/bin/bash
+start=$SECONDS
+set -B
+
+# Generate xml dataset by year
+# USAGE: bash xml-dataset-download.sh <year> <api_key>
+# EXAMPLE: bash xml-dataset-download.sh 2000 N4aCuDuJO8Ucf1FTR2EzVPZqo8NsSl1c7YLYOk8N
+# CURL: curl -H "X-API-Key: N4aCuDuJO8Ucf1FTR2EzVPZqo8NsSl1c7YLYOk8N" https://api.foia.gov/api/annual-report-xml/ibwc/2022 -o test.xml
+
+YEAR=$1
+API_KEY=$2
+
+case $YEAR in
+    2011 | 2010 | 2009 | 2008)
+    declare -a agencies=(usibwc abmc acus nrpc usagm ceq cftc cia cigie cncs cppbsd cpsc csb csosa dhs dnfsb doc dod doe doi doj dol dot ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gsa hhs hud iaf ibwc imls lsc mcc mspb nara nasa ncpc ncua nea neh nigc nmb nrc nlrb nsf ntsb odni oge omb ondcp onhir opic opm oshrc ostp pbgc pc prc usrrb sba sec sigar ssa sss dos stb treasury tva usadf usaid usccr co usda usitc usps ustda ustr va osc sigir)
+    ;;
+  2012)
+    declare -a agencies=(usibwc ratb afrh abmc acus nrpc usagm ceq cftc cia cigie cncs cppbsd cpsc csb csosa dhs dnfsb doc dod doe doi doj dol dot ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gsa hhs hud iaf ibwc imls lsc mcc mspb nara nasa ncpc ncua nea neh nigc nmb nrc nlrb nsf ntsb odni oge omb ondcp onhir opic opm oshrc ostp pbgc pc prc usrrb sba sec sigar ssa sss dos stb treasury tva usadf usaid usccr co usda usitc usps ustda ustr va osc sigir)
+    ;;
+  2013)
+    declare -a agencies=(opic usibwc abmc acus nrpc afrh usagm ceq cfpb cftc cia cigie cncs cppbsd cpsc csb csosa dhs dnfsb doc dod doe doi doj dol dot ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gsa hhs hud iaf ibwc imls lsc mcc mspb nara nasa ncpc ncua nea neh nigc nmb nrc nlrb nsf ntsb odni oge omb ondcp onhir dfc opm oshrc ostp pbgc pc prc usrrb sba sec sigar ssa sss dos stb treasury tva usadf usaid usccr co usda usitc usps ustda ustr va ratb osc)
+    ;;
+  2014)
+    declare -a agencies=(opic usibwc abmc acus afrh usagm ceq cfpb cftc cia cigie cncs cppbsd cpsc csb csosa dhs dnfsb doc dod doe doi doj dol dot ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gsa hhs hud iaf ibwc imls lsc mcc mspb nara nasa ncpc ncua nea neh nigc nmb nrc nlrb nsf ntsb odni oge omb ondcp onhir dfc opm oshrc ostp pbgc pclob pc prc usrrb sba sec sigar ssa sss dos stb treasury tva usadf usaid usccr co usda usitc usps ustda ustr va ratb)
+    ;;
+  2015)
+    declare -a agencies=(opic usibwc abmc acus afrh usagm ceq cfpb cftc cia cigie cncs cppbsd cpsc csb csosa dhs dnfsb doc dod doe doi doj dol dot ed eeoc epa  fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gsa hhs hud iaf ibwc imls lsc mcc mspb nara nasa ncpc ncua nea neh nigc nmb nrc nrcp nlrb nsf ntsb odni oge omb ondcp onhir dfc opm osc oshrc ostp pbgc pclob pc prc usrrb sba sec sigar ssa sss dos stb treasury tva usab usadf usaid usccr co usda usitc usps ustda ustr va)
+    ;;
+  2016 | 2017)
+    declare -a agencies=(opic abmc acus afrh asc ceq cfa cfpb cftc cia cigie co cncs cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi doj dol dos dot eac ed eeoc epa  fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmshrc fmcs fomc frb frtib ftc tva gcerc gsa hhs hstsf hud iaf imls jmmff lsc mcc mkuf mmc mspb nara nasa ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt sba sec sigar ssa ssab sss stb treasury usab usadf usagm usaid usccr usda usibwc usich usip usitc usps usrrb ustda ustr va)
+    ;;
+  2018)
+    declare -a agencies=(opic abmc achp acus afrh asc ceq cfa cfpb cftc cia cigie co cncs cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi doj dol dos dot eac ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmshrc fmcs fomc frb frtib ftc tva gcerc gsa hhs hstsf hud iaf imls jmmff lsc mcc mkuf mmc mspb nara nasa ncmnps ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt sba sec sigar ssa ssab sss stb treasury usab usadf usagm usaid usccr usda usibwc usich usip usitc usps usrrb ustda ustr va)
+    ;;
+  2019)
+    declare -a agencies=(opic abmc achp acus afrh asc ceq cfa cfpb cftc cia cigie co cncs cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi doj dol dos dot eac ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmshrc fmcs fomc frb frtib ftc tva gcerc gsa ha hhs hstsf hud iaf imls jmmff lsc mcc mkuf mmc mspb nara nasa ncmnps ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt sba sec sigar ssa ssab sss stb treasury usab usadf usagm usaid usccr usda usibwc usich usip usitc usps usrrb ustda ustr va)
+    ;;
+  2020)
+    declare -a agencies=(abmc achp acus afrh asc ceq cfa cfpb cftc cia cigie co cncs cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi doj dol dos dot eac ed eeoc epa ex-im bank fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmshrc fmcs fomc frb frtib ftc tva gcerc gsa ha hhs hstsf hud iaf imls jmmff lsc mcc mkuf mmc mspb nara nasa ncmnps ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nscai nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt sba sec sigar ssa ssab sss stb treasury usab usadf usagm usaid usccr usda usibwc usich usip usitc usps usrrb ustda ustr va)
+    ;;
+  2021)
+    declare -a agencies=(abmc doj achp acus afrh asc ceq cfa cfpb cftc cia cigie cncs co cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi dol dos dot eac ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gcerc gsa ha hhs hstsf hud iaf imls ipec fpisc jmmff lsc mcc mkuf mmc mspb nara nasa ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt usrrb sba sec sigar ssa ssab sss stb treasury tva usab usadf usagm usaid usccr usda usibwc usich usip usitc usps ustda ustr va)
+    ;;
+  2022)
+    declare -a agencies=(abmc doj achp acus afrh asc ceq cfa cfpb cftc cia cigie cncs co cppbsd cpsc csb csosa dc dfc dhs dnfsb doc dod doe doi dol dos dot eac ed eeoc epa fca fcc fcsic fdic fec ferc ffiec fhfa flra fmc fmcs fmshrc fomc frb frtib ftc gcerc gsa ha hhs hstsf hud iaf imls ipec fpisc jmmff lsc mcc mkuf mmc mspb nara nasa ncd ncpc ncua nea neh nigc nlrb nmb nrc nrpc nsf ntsb nw nwtrb odni oge omb ondcp onhir opm osc oshrc ostp pbgc pc pclob prc pt usrrb sba sec sigar ssa ssab sss stb treasury tva usab usadf usagm usaid usccr usda usibwc usich usip usitc usps ustda ustr va)
+    ;;
+  *)
+    echo -n "Year unavailable: ${YEAR}"
+    exit;
+    ;;
+esac
+
+agencies+=("exim%20bank") # hardcode exim bank which exists in all years
+
+# Exit if there aren't two arguments
+if [ $# != 2 ]
+  then
+    echo "Please supply the year and api key."
+    echo "USAGE: bash xml-dataset-download.sh <year> <api_key>"
+    exit;
+fi
+
+echo "This script will output a zip file in the public directory: bin/xml-datasets/files/$YEAR-FOIASetFull.zip"
+
+newDir="files/${YEAR}"
+
+if ! [[ -d "$newDir" ]] ; then
+  echo -e "Creating new directory $newDir \n"
+   mkdir -p "$newDir"
+fi
+
+for AGENCY in "${agencies[@]}"
+do
+   :
+  echo -e "Getting XML data for ${AGENCY}... \n"
+  sleep 1
+  CLEAN=${AGENCY//%20/-}
+  FILE=files/${YEAR}/${CLEAN}-${YEAR}'.xml'
+  curl -H "X-API-Key: $API_KEY" 'https://api.foia.gov/api/annual-report-xml/'$AGENCY'/'$YEAR -o $FILE
+  if grep -Fxq "Report not found." $FILE
+  then
+    echo -e "Report not found for this agency: $AGENCY $YEAR \n"
+    rm -r $FILE
+  fi
+
+  # Unpublished agency report causes this error
+  if grep -Fxq "Report not complete." $FILE
+  then
+    echo -e "Report not complete: $AGENCY $YEAR \n"
+    rm -r $FILE
+  fi
+done
+
+echo -e "Zipping all XML files for the year ${YEAR}... \n"
+zip -r -j zips/$YEAR-FOIASetFull.zip files/$YEAR/*.xml
+echo -e "Removing all XML files from this directory... \n"
+#rm -r files/$YEAR/*.xml # leave files but use gitignore for testing
+duration=$(( SECONDS - start ))
+echo -e "You can now manually place the zip files into the www.foia.gov directory..."
+echo -e "Finished in ${duration} seconds"
+echo -e "**********\n"

bin/xml-datasets/zips/.gitignore

@@ -0,0 +1 @@
+*.zip

docs/request-forms.md

@@ -0,0 +1,101 @@
+# Request forms
+
+This application (the NFP) takes "webforms" from the Drupal back-end and then renders
+them on the page using front-end code. Normally, Drupal renders webforms
+directly, but in the decoupled architecture of the NFP, the webforms are
+passed from the back-end to the front-end as JSON data, and that JSON is then
+converted into a form. So, this architecture relies on communication and
+conversion between two libraries:
+
+1. [The Drupal "Webform" module](https://www.drupal.org/project/webform)
+2. [The React JSON Schema Forms (RJSF) library](https://rjsf-team.github.io/react-jsonschema-form/docs/)
+
+## Communication
+
+Content in the Drupal back-end is communicated to the front-end via
+Drupal's JSONAPI service. As such, the front-end makes a request to the
+back-end to retrieve a JSON response that details the configuration of each
+individual webform.
+
+## Conversion
+
+The configuration of the webform must then be converted into the data structures
+needed by RJSF. This conversion is done in the webform_to_json_schema.js file.
+
+## Supported features
+
+The Webform module and the RJSF library but have large feature sets, but in
+general the Webform module has more features. In addition, each feature must
+be intentionally converted (as mentioned above) which means that we need custom
+code to support any given feature of the Webform module.
+
+This means that there are things you can configure in a Drupal webform that will
+NOT survive the conversion to RJSF. For example, at the time of this writing,
+webform elements can be configured to be "radio buttons", but our conversion
+code does not correctly support this feature, and so they will appear on the
+front-end as a drop-down. (Though this is a desired feature that we're working
+on.)
+
+Here is a current list of the supported features -- ie, things you can do in
+the Webform module that will convert to RJSF in the expected way:
+
+### Supported elements
+
+These are the "elements" in Webform that will appear as expected in RJSF:
+
+* Text field
+* Email
+* Telephone
+* Select
+* Text area
+* File (see below)
+
+Note that the file element is a special case. Currently a single file element is
+supported but additional file elements cannot be added.
+
+### Conditions
+
+In the Webform module it is possible to set "conditions" on elements, so that
+their behavior can depend on user input in other elements. These are partially
+supported, as detailed below:
+
+#### "Require" condition
+
+The most commonly-used conditional is making a field required based on the input
+in other elements. This is supported in two ways: by a "required" label
+appearing dynamically on the appropriate form elements, and by the return of
+error messages after form submission.
+
+1. "Required" label appearing dynamically: This is the best user experience,
+because the "Required" label makes it immediately clear to the user what is
+required, and because it will not even let the user submit the form until the
+input is provided. However, this method is only supported when the webform
+"condition" uses a logic operator of "any" and a trigger of "value is". Also,
+this method only works on custom fields (fields that are not part of the
+starting webform "template").
+2. Return of error messages after form submission: If a condition is setup to
+make a field required, it will always result in error messages when required
+input is missing. This is a slightly worse user experience, because the user
+will not be aware that the field is required until after they have attempted
+to submit the form. Because this is a worse user experience, it is recommended
+to always use a logic operator of "any" and a trigger of "value is", so that
+the dynamic label (described above) will appear as well.
+
+#### "Visible" condition
+
+Sometimes there are fields that are optional, and would ideally be visible under
+only certain circumstances. This is partially supported. However, it is similar
+to the dynamic "required" labels: it will work only if the logic operator is
+"any" and the trigger is "value is", and it only works on custom fields (fields
+that are not part of the starting webform "template").
+
+#### Other conditions
+
+No other conditions are supported at this time.
+
+## Element positioning and custom fields
+
+Each webform has a core set of fields that appear in a particular order on
+the front-end. This order cannot currently be customized. Additional custom
+fields can be added to webforms, but they will always appear in the "Additional
+information" section.

js/util/request_form/webform_to_json_schema.js

@@ -129,6 +129,48 @@ function toUiSchemaProperty(webformField) {
   return { [webformField.name]: uiSchemaProperty };
 }
 
+/**
+ * Gets information about webform states in fields.
+ */
+function conditionalStates(webformField, stateType) {
+  if (webformField.states && webformField.states[stateType]) {
+    const conditions = (Array.isArray(webformField.states[stateType])) ? webformField.states[stateType] : [webformField.states[stateType]];
+    const results = conditions.map((condition) => {
+      if (typeof condition === 'object') {
+        return Object.keys(condition).map((key) => {
+          const parentMatch = key.match(/:input\[name="(.*)"\]/);
+          const parent = (parentMatch && parentMatch.length > 1) ? parentMatch[1] : false;
+          const value = condition[key].value;
+          if (parent && value) {
+            return {
+              child: webformField.name,
+              parent,
+              value,
+            };
+          }
+          return false;
+        }).filter((v) => v);
+      }
+      return false;
+    }).filter((v) => v);
+    return results;
+  }
+  return [];
+}
+/**
+ * Gets information about conditionally visible fields.
+ */
+function conditionalVisibility(webformField) {
+  return conditionalStates(webformField, 'visible');
+}
+
+/**
+ * Gets information about conditionally required fields.
+ */
+function conditionalRequirement(webformField) {
+  return conditionalStates(webformField, 'required');
+}
+
 /**
  * Translates agency components' Drupal Webform fields into JSON schema and uiSchema
  * for use with react-jsonschema-form.
@@ -148,6 +190,98 @@ function webformFieldsToJsonSchema(formFields = [], { title, description, id } =
     .map(toJsonSchemaProperty)
     .reduce((properties, property) => Object.assign(properties, property), {});
 
+  const conditionallyVisible = formFields
+    .map(conditionalVisibility)
+    .filter((v) => v.length);
+
+  const conditionallyRequired = formFields
+    .map(conditionalRequirement)
+    .filter((v) => v.length);
+
+  const dependencies = {};
+  conditionallyVisible.forEach((conditions) => {
+    conditions.forEach((condition) => {
+      condition.forEach((item) => {
+        const dependencyKey = [item.parent, item.value].join('|');
+        if (typeof dependencies[dependencyKey] === 'undefined') {
+          const properties = {};
+          properties[item.parent] = {
+            enum: [item.value],
+          };
+          dependencies[dependencyKey] = {
+            properties,
+          };
+        }
+        const clonedElement = { ...jsonSchema.properties[item.child] };
+        dependencies[dependencyKey].properties[item.child] = clonedElement;
+      });
+    });
+  });
+  conditionallyRequired.forEach((conditions) => {
+    conditions.forEach((condition) => {
+      condition.forEach((item) => {
+        const dependencyKey = [item.parent, item.value].join('|');
+        if (typeof dependencies[dependencyKey] === 'undefined') {
+          const properties = {};
+          properties[item.parent] = {
+            enum: [item.value],
+          };
+          dependencies[dependencyKey] = {
+            properties,
+          };
+        }
+        const clonedElement = { ...jsonSchema.properties[item.child] };
+        dependencies[dependencyKey].properties[item.child] = clonedElement;
+        if (typeof dependencies[dependencyKey].required === 'undefined') {
+          dependencies[dependencyKey].required = [];
+        }
+        if (!(dependencies[dependencyKey].required.includes(item.child))) {
+          dependencies[dependencyKey].required.push(item.child);
+        }
+      });
+    });
+  });
+
+  if (typeof jsonSchema.dependencies === 'undefined') {
+    jsonSchema.dependencies = {};
+  }
+  Object.keys(dependencies).forEach((dependencyKey) => {
+    const parent = dependencyKey.split('|')[0];
+    if (typeof jsonSchema.dependencies[parent] === 'undefined') {
+      jsonSchema.dependencies[parent] = { oneOf: [] };
+    }
+    jsonSchema.dependencies[parent].oneOf.push(dependencies[dependencyKey]);
+  });
+
+  // Make sure that the dependencies have all available options.
+  Object.keys(dependencies).forEach((dependencyKey) => {
+    const parent = dependencyKey.split('|')[0];
+    if (jsonSchema.properties[parent]) {
+      const enumOptions = jsonSchema.properties[parent].enum;
+      enumOptions.forEach((enumOption) => {
+        const oneOfItems = jsonSchema.dependencies[parent].oneOf;
+        const match = oneOfItems.find((oneOfItem) => oneOfItem.properties[parent] && oneOfItem.properties[parent].enum.includes(enumOption));
+        if (!match) {
+          const emptyDependency = {
+            properties: {},
+          };
+          emptyDependency.properties[parent] = {
+            enum: [enumOption],
+          };
+          oneOfItems.push(emptyDependency);
+        }
+      });
+    }
+  });
+
+  conditionallyVisible.forEach((conditions) => {
+    conditions.forEach((condition) => {
+      condition.forEach((item) => {
+        delete jsonSchema.properties[item.child];
+      });
+    });
+  });
+
   // Add required fields to the `required` property
   jsonSchema.required = formFields
     .filter((formField) => formField.required)