Use @osmcha/osmchange-parser and @osmcha/osm-changeset-xml-parser; remove lib/xml.js

lib/get-changesets.js

@@ -2,21 +2,23 @@
 const _ = require('lodash');
 const moment = require('moment');
 
-const { parseOsmChangeXml, parseChangesetXml, parseAugmentedDiff } = require('../lib/xml');
+const parseOsmChangeXml = require("@osmcha/osmchange-parser");
+const parseChangesetXml = require("@osmcha/osm-changeset-xml-parser");
+const parseAugmentedDiff = require("@osmcha/osm-adiff-parser");
+
 const { getStateForMinute } = require('../util/get-states');
 const { request } = require('../util/request');
 const {
   OSM_CHANGESET_API,
   CHANGE_STATES,
-  TYPES,
   OVERPASS_PRIMARY_URL,
   OVERPASS_SECONDARY_URL
 } = require('./constants');
 
 const getChangesets = async (xml) => {
-  const jsonData = parseOsmChangeXml(xml);
+  const jsonData = await parseOsmChangeXml(xml);
 
-  if (!jsonData.osmChange || !jsonData.osmChange[0]) {
+  if (!jsonData) {
     throw new Error('OSM data missing from XML file');
   }
 
@@ -49,7 +51,7 @@ const getChangesets = async (xml) => {
         getDataParam(
           meta.created_at,
           meta.closed_at ? meta.closed_at : meta.created_at,
-          meta.open === 'false'
+          !meta.open
         ),
         getBboxParam(meta.bbox)
       );
@@ -91,7 +93,7 @@ const getChangesets = async (xml) => {
   const diffFeatureMap = {};
   changesetIds.forEach((cid) => {
     if (featureMap[cid] && realFeatureMap[cid]) {
-      diffFeatureMap[cid] = _.difference(featureMap[cid], realFeatureMap[cid]);
+      diffFeatureMap[cid] = _.difference(featureMap[cid], realFeatureMap[cid].map(s => +s));
       if (diffFeatureMap[cid].length) {
         results[cid].metadata.incomplete = true;
         console.log('# Incomplete changeset', cid);
@@ -101,42 +103,37 @@ const getChangesets = async (xml) => {
 
   console.log('# Feature diff', JSON.stringify(diffFeatureMap));
 
+  for (let realChangeset of Object.values(results)) {
+    makeBackwardsCompatible(realChangeset);
+  }
+
   return results;
 };
 
-const parseJsonData = (jsonData) => {
+const parseJsonData = (osmChange) => {
   const featureMap = {};
   let timestamps = [];
   let changesetIds = [];
 
-  CHANGE_STATES.map((changeState) => {
-    if(jsonData.osmChange[0][changeState]) {
-      jsonData.osmChange[0][changeState].forEach((stateSection) => {
-        TYPES.forEach((type) => {
-          if(stateSection[type]) {
-            stateSection[type].forEach(({ timestamp, changeset, id }) => {
-              timestamps.push(timestamp);
-              changesetIds.push(changeset);
-
-              if (!featureMap[changeset]) {
-                featureMap[changeset] = [];
-              }
-              featureMap[changeset].push(id);
-            })
-          }
-        });
-      });
+  for (let changeState of CHANGE_STATES) {
+    if (osmChange[changeState]) {
+      for (let element of osmChange[changeState]) {
+        let { timestamp, changeset, id } = element;
+        timestamps.push(timestamp);
+        changesetIds.push(changeset);
+
+        if (!featureMap[changeset]) {
+          featureMap[changeset] = [];
+        }
+        featureMap[changeset].push(id);
+      }
     }
-  });
+  }
 
   timestamps = _.uniq(timestamps);
   changesetIds = _.uniq(changesetIds);
 
-  return {
-    timestamps,
-    changesetIds,
-    featureMap,
-  };
+  return { timestamps, changesetIds, featureMap };
 };
 
 const getTimestampsStates = (timestamps) => {
@@ -164,9 +161,9 @@ const queryOverpass = async (changesetId, data, bbox) => {
 const getChangesetMetadata = async (changesetId) => {
   try {
     const body = await request(`${OSM_CHANGESET_API}/${changesetId}`).then(res => res.text());
-    const changesetData = parseChangesetXml(body);
+    const changesetData = await parseChangesetXml(body);
 
-    const meta = changesetData.osm[0].changeset[0];
+    const meta = changesetData.changeset;
 
     const bbox = {
       left: meta.min_lon ? meta.min_lon : -180,
@@ -232,6 +229,41 @@ const getBboxParam = (bbox) => {
   return [bbox.left, bbox.bottom, bbox.right, bbox.top].join(',');
 }
 
+/*
+ * Convert some field types and object structures from the format returned by
+ * osm-changeset-xml-parser to the format expected by current consumers of the
+ * real-changesets dataset.
+ */
+const makeBackwardsCompatible = (realChangeset) => {
+  // these fields need to be converted from Numbers (or Booleans) to strings
+  let fields = [
+    "metadata.min_lon",
+    "metadata.min_lat",
+    "metadata.max_lon",
+    "metadata.max_lat",
+    "metadata.bbox.left",
+    "metadata.bbox.bottom",
+    "metadata.bbox.right",
+    "metadata.bbox.top",
+    "metadata.id",
+    "metadata.uid",
+    "metadata.changes_count",
+    "metadata.comments_count",
+    "metadata.open",
+  ];
+
+  for (let field of fields) {
+    let value = _.get(realChangeset, field);
+    if (value !== undefined) {
+      _.set(realChangeset, field, JSON.stringify(value));
+    }
+  }
+
+  // tags need to be converted from { foo: "bar" } to [{ k: "foo", v: "bar" }] form
+  realChangeset.metadata.tag = Object.entries(realChangeset.metadata.tags).map(([k, v]) => ({ k, v }));
+  delete realChangeset.metadata.tags;
+}
+
 module.exports = {
   getChangesets,
   getChangesetMetadata,

package.json

@@ -25,9 +25,10 @@
   "homepage": "https://github.com/OSMCha/osm-adiff-service",
   "dependencies": {
     "@osmcha/osm-adiff-parser": "^2.0.0",
+    "@osmcha/osm-changeset-xml-parser": "^1.0.0",
+    "@osmcha/osmchange-parser": "^1.0.0",
     "aws-sdk": "^2.625.0",
     "changetags": "^0.1.2",
-    "htmlparser2": "^4.1.0",
     "lodash": "^4.17.15",
     "minimist": "^1.2.0",
     "moment": "^2.25.3",