Improve KML support

package.json

@@ -28,7 +28,7 @@
 		"lint": "eslint --ext mjs src/",
 		"prepublishOnly": "npm lint; npm test; ./pre-publish",
 		"postpublish": "./release_web_ui; ./release_github_version",
-		"browserify": "browserify -r sync-request -r mproj -r buffer -r iconv-lite -r fs -r flatbush -r rw -r path -r kdbush -r @tmcw/togeojson -o www/modules.js",
+		"browserify": "browserify -r sync-request -r mproj -r buffer -r iconv-lite -r fs -r flatbush -r rw -r path -r kdbush -r @tmcw/togeojson -r @placemarkio/tokml -o www/modules.js",
 		"dev": "rollup --config --watch"
 	},
 	"main": "./mapshaper.js",

src/cli/mapshaper-cli-utils.mjs

@@ -19,6 +19,13 @@ cli.isFile = function(path, cache) {
   return ss && ss.isFile() || false;
 };
 
+cli.checkCommandEnv = function(cname) {
+  var blocked = ['i', 'include', 'require', 'external'];
+  if (runningInBrowser() && blocked.includes(cname)) {
+    stop('The -' + cname + ' command cannot be run in the browser');
+  }
+};
+
 // cli.fileSize = function(path) {
 //   var ss = cli.statSync(path);
 //   return ss && ss.size || 0;

src/cli/mapshaper-parse-commands.mjs

@@ -2,6 +2,7 @@ import { getOptionParser } from '../cli/mapshaper-options';
 import { splitShellTokens } from '../cli/mapshaper-option-parsing-utils';
 import { stop } from '../utils/mapshaper-logging';
 import utils from '../utils/mapshaper-utils';
+import cli from './mapshaper-cli-utils';
 
 // Parse an array or a string of command line tokens into an array of
 // command objects.
@@ -23,12 +24,12 @@ export function standardizeConsoleCommands(raw) {
   var parser = getOptionParser();
   // support multiline string of commands pasted into console
   str = str.split(/\n+/g).map(function(str) {
-    var match = /^[a-z][\w-]*/.exec(str = str.trim());
+    var match = /^[a-z][\w-]*/i.exec(str = str.trim());
     //if (match && parser.isCommandName(match[0])) {
     if (match) {
       // add hyphen prefix to bare command
-       // also add hyphen to non-command strings, for a better error message
-       // ("unsupported command" instead of "The -i command cannot be run in the browser")
+      // also add hyphen to non-command strings, for a better error message
+      // ("unsupported command" instead of "The -i command cannot be run in the browser")
       str = '-' + str;
     }
     return str;
@@ -38,15 +39,10 @@ export function standardizeConsoleCommands(raw) {
 
 // Parse a command line string for the browser console
 export function parseConsoleCommands(raw) {
-  var blocked = ['i', 'include', 'require', 'external'];
   var str = standardizeConsoleCommands(raw);
-  var parsed;
-  parsed = parseCommands(str);
+  var parsed = parseCommands(str);
   parsed.forEach(function(cmd) {
-    var i = blocked.indexOf(cmd.name);
-    if (i > -1) {
-      stop("The -" + blocked[i] + " command cannot be run in the browser");
-    }
+    cli.checkCommandEnv(cmd.name);
   });
   return parsed;
 }

src/gui/gui-export-control.mjs

@@ -214,7 +214,7 @@ export var ExportControl = function(gui) {
   }
 
   function initFormatMenu() {
-    var defaults = ['shapefile', 'geojson', 'topojson', 'json', 'dsv', 'svg'];
+    var defaults = ['shapefile', 'geojson', 'topojson', 'json', 'dsv', 'kml', 'svg'];
     var formats = utils.uniq(defaults.concat(getInputFormats()));
     var items = formats.map(function(fmt) {
       return utils.format('<div><label><input type="radio" name="format" value="%s"' +

src/gui/gui-import-control.mjs

@@ -268,12 +268,14 @@ export function ImportControl(gui, opts) {
   async function expandFiles(files) {
     var files2 = [], expanded;
     for (var f of files) {
-      if (internal.isZipFile(f.name) || /\.kmz$/.test(f.name)) {
+      if (internal.isZipFile(f.name)) {
         expanded = await readZipFile(f);
-        files2 = files2.concat(expanded);
+      } else if (internal.isKmzFile(f.name)) {
+        expanded = await readKmzFile(f);
       } else {
-        files2.push(f);
+        expanded = [f];
       }
+      files2 = files2.concat(expanded);
     }
     return files2;
   }
@@ -430,6 +432,15 @@ export function ImportControl(gui, opts) {
     });
   }
 
+  async function readKmzFile(file) {
+    var files = await readZipFile(file);
+    var name = files[0] && files[0].name;
+    if (name == 'doc.kml') {
+      files[0].name = internal.replaceFileExtension(file.name, 'kml');
+    }
+    return files;
+  }
+
   async function readZipFile(file) {
     var files = [];
     await wait(35); // pause a beat so status message can display

src/io/mapshaper-export.mjs

@@ -1,5 +1,6 @@
 import { getLayerBounds } from '../dataset/mapshaper-layer-utils';
 import { exportSVG } from '../svg/mapshaper-svg';
+import { exportKML } from '../kml/kml-export';
 import { exportDbf } from '../shapefile/dbf-export';
 import { exportDelim } from '../text/mapshaper-delim-export';
 import { exportShapefile } from '../shapefile/shp-export';
@@ -31,7 +32,7 @@ export function exportTargetLayers(targets, opts) {
 function exportDatasets(datasets, opts) {
   var format = getOutputFormat(datasets[0], opts);
   var files;
-  if (format == 'svg' || format == 'topojson' || format == 'geojson' && opts.combine_layers) {
+  if (format == 'kml' || format == 'svg' || format == 'topojson' || format == 'geojson' && opts.combine_layers) {
     // multi-layer formats: combine multiple datasets into one
     if (datasets.length > 1) {
       datasets = [mergeDatasetsForExport(datasets)];
@@ -85,8 +86,8 @@ export function exportFileContent(dataset, opts) {
   }, dataset);
 
   // Adjust layer names, so they can be used as output file names
-  // (except for multi-layer formats TopoJSON and SVG)
-  if (opts.file && outFmt != 'topojson' && outFmt != 'svg') {
+  // (except for multi-layer formats TopoJSON, SVG, KML)
+  if (opts.file && outFmt != 'topojson' && outFmt != 'svg'&& outFmt != 'kml') {
     dataset.layers.forEach(function(lyr) {
       lyr.name = getFileBase(opts.file);
     });
@@ -131,7 +132,8 @@ var exporters = {
   dsv: exportDelim,
   dbf: exportDbf,
   json: exportJSON,
-  svg: exportSVG
+  svg: exportSVG,
+  kml: exportKML
 };
 
 

src/io/mapshaper-file-import.mjs

@@ -1,43 +1,97 @@
 import { importContent } from '../io/mapshaper-import';
-import { isSupportedBinaryInputType, guessInputContentType, guessInputFileType } from '../io/mapshaper-file-types';
+import {
+  isSupportedBinaryInputType,
+  guessInputContentType,
+  guessInputFileType,
+  isZipFile,
+  isKmzFile } from '../io/mapshaper-file-types';
 import cmd from '../mapshaper-cmd';
 import cli from '../cli/mapshaper-cli-utils';
 import utils from '../utils/mapshaper-utils';
 import { message, verbose, stop } from '../utils/mapshaper-logging';
-import { getFileExtension, replaceFileExtension } from '../utils/mapshaper-filename-utils';
+import { parseLocalPath, getFileExtension, replaceFileExtension } from '../utils/mapshaper-filename-utils';
 import { trimBOM, decodeString } from '../text/mapshaper-encodings';
 import { extractFiles } from './mapshaper-zip';
 import { buildTopology } from '../topology/mapshaper-topology';
 import { cleanPathsAfterImport } from '../paths/mapshaper-path-import';
 import { mergeDatasets } from '../dataset/mapshaper-merging';
 
 cmd.importFiles = function(opts) {
-  var files = opts.files || [],
-      dataset;
+  var files = opts.files || [];
+  var dataset;
 
+  cli.checkCommandEnv('i');
   if (opts.stdin) {
     return importFile('/dev/stdin', opts);
   }
 
+
   if (files.length > 0 === false) {
     stop('Missing input file(s)');
   }
 
   verbose("Importing: " + files.join(' '));
-  if (files.length == 1 && /\.zip/i.test(files[0])) {
-    dataset = importZipFile(files[0], opts);
-  } else if (files.length == 1) {
+
+  // copy opts, so parameters can be modified within this command
+  opts = Object.assign({}, opts);
+  opts.input = Object.assign({}, opts.input); // make sure we have a cache
+
+  files = expandFiles(files, opts.input);
+
+  if (files.length === 0) {
+    stop('Missing importable files');
+  }
+
+  if (files.length == 1) {
     dataset = importFile(files[0], opts);
-  } else if (opts.merge_files) {
-    // TODO: deprecate and remove this option (use -merge-layers cmd instead)
-    dataset = importFilesTogether(files, opts);
-    dataset.layers = cmd.mergeLayers(dataset.layers);
   } else {
     dataset = importFilesTogether(files, opts);
   }
+
+  if (opts.merge_files && files.length > 1) {
+    // TODO: deprecate and remove this option (use -merge-layers cmd instead)
+    dataset.layers = cmd.mergeLayers(dataset.layers);
+  }
   return dataset;
 };
 
+function expandFiles(files, cache) {
+  var files2 = [];
+  files.forEach(function(file) {
+    var expanded;
+    if (isZipFile(file)) {
+      expanded = expandZipFile(file, cache);
+    } else if (isKmzFile(file)) {
+      expanded = expandKmzFile(file, cache);
+    } else {
+      expanded = [file]; // ordinary file, no change
+    }
+    files2 = files2.concat(expanded);
+  });
+  return files2;
+}
+
+function expandKmzFile(file, cache) {
+  var files = expandZipFile(file, cache);
+  var name = replaceFileExtension(parseLocalPath(file).filename, 'kml');
+  if (files[0] == 'doc.kml') {
+    files[0] = name;
+    cache[name] = cache['doc.kml'];
+  }
+  return files;
+}
+
+function expandZipFile(file, cache) {
+  var input;
+  if (file in cache) {
+    input = cache[file];
+  } else {
+    input = file;
+    cli.checkFileExists(file);
+  }
+  return extractFiles(input, cache);
+}
+
 // Let the web UI replace importFile() with a browser-friendly version
 export function replaceImportFile(func) {
   _importFile = func;
@@ -56,6 +110,7 @@ var _importFile = function(path, opts) {
       content;
 
   cli.checkFileExists(path, cache);
+
   if ((fileType == 'shp' || fileType == 'json' || fileType == 'text' || fileType == 'dbf') && !cached) {
     // these file types are read incrementally
     content = null;
@@ -78,6 +133,7 @@ var _importFile = function(path, opts) {
       stop('Unrecognized file type:', path);
     }
     // TODO: consider using a temp file, to support incremental reading
+    // read to buffer, even for string-based types (to support larger input files)
     content = require('zlib').gunzipSync(cli.readFile(pathgz));
 
   } else { // type can't be inferred from filename -- try reading as text
@@ -116,8 +172,8 @@ function importZipFile(path, opts) {
     input = path;
   }
   var files = extractFiles(input, cache);
-  var zipOpts = Object.assign({}, opts, {input: cache});
-  return importFilesTogether(files, zipOpts);
+  var zipOpts = Object.assign({}, opts, {input: cache, files: files});
+  return cmd.importFiles(zipOpts);
 }
 
 // Import multiple files to a single dataset

src/io/mapshaper-file-types.mjs

@@ -49,8 +49,12 @@ export function isZipFile(file) {
   return /\.zip$/i.test(file);
 }
 
+export function isKmzFile(file) {
+  return /\.kmz$/i.test(file);
+}
+
 export function isSupportedOutputFormat(fmt) {
-  var types = ['geojson', 'topojson', 'json', 'dsv', 'dbf', 'shapefile', 'svg'];
+  var types = ['geojson', 'topojson', 'json', 'dsv', 'dbf', 'shapefile', 'svg', 'kml'];
   return types.indexOf(fmt) > -1;
 }
 
@@ -61,6 +65,8 @@ export function getFormatName(fmt) {
     json: 'JSON records',
     dsv: 'CSV',
     dbf: 'DBF',
+    kml: 'KML',
+    kmz: 'KMZ',
     shapefile: 'Shapefile',
     svg: 'SVG'
   }[fmt] || '';
@@ -74,6 +80,6 @@ export function isSupportedBinaryInputType(path) {
 
 // Detect extensions of some unsupported file types, for cmd line validation
 export function filenameIsUnsupportedOutputType(file) {
-  var rxp = /\.(shx|prj|xls|xlsx|gdb|sbn|sbx|xml|kml)$/i;
+  var rxp = /\.(shx|prj|xls|xlsx|gdb|sbn|sbx|xml)$/i;
   return rxp.test(file);
 }

src/kml/kml-export.mjs

@@ -0,0 +1,17 @@
+import { exportDatasetAsGeoJSON } from '../geojson/geojson-export';
+import { getOutputFileBase } from '../utils/mapshaper-filename-utils';
+// import { isKmzFile } from '../io/mapshaper-file-types';
+
+export function exportKML(dataset, opts) {
+  var toKML = require("@placemarkio/tokml").toKML;
+  var geojsonOpts = Object.assign({combine_layers: true, geojson_type: 'FeatureCollection'}, opts);
+  var geojson = exportDatasetAsGeoJSON(dataset, geojsonOpts);
+  var kml = toKML(geojson);
+  // TODO: add KMZ output
+  // var useKmz = opts.file && isKmzFile(opts.file);
+  var ofile = opts.file || getOutputFileBase(dataset) + '.kml';
+  return [{
+    content: kml,
+    filename: ofile
+  }];
+}

test/kml-test.mjs

@@ -0,0 +1,24 @@
+import api from '../';
+import assert from 'assert';
+import fs from 'fs';
+
+describe('kml i/o', function () {
+
+  it('kmz -> kml -> .geojson', async function() {
+    var file = 'test/data/kml/Albania.kmz';
+    var cmd = `-i ${file} -o points.kml`;
+    var out = await api.applyCommands(cmd);
+    var cmd2 = '-i points.kml -o format=geojson';
+    var out2 = await api.applyCommands(cmd2, {'points.kml': out['points.kml']});
+    var geojson = JSON.parse(out2['points.json']);
+    assert.equal(geojson.features[0].geometry.type, 'Point');
+  })
+
+  it('-o format=kml syntax', async function() {
+    var file = 'test/data/kml/Albania.kmz';
+    var cmd = `-i ${file} -filter-fields -o format=kml`;
+    var out = await api.applyCommands(cmd);
+    assert(/^<kml/.test(out['Albania.kml']));
+  })
+
+});

test/option-parsing-utils-test.mjs

@@ -90,7 +90,6 @@ describe('mapshaper-option-parsing-utils.js', function () {
   describe('splitShellTokens()', function () {
     var split = api.internal.splitShellTokens;
     function test(src, dest) {
-      // assert.deepEqual(await import('shell-quote').parse(src), split(src));
      assert.deepEqual(split(src), dest);
     }
 

test/parse-commands-test.mjs

@@ -6,17 +6,17 @@ var internal = api.internal;
 describe('mapshaper-parse-commands.js', function () {
 
   describe('parseConsoleCommands()', function () {
-    it('should block input commands', function () {
-
-      function bad(cmd) {
-        assert.throws(function() {
-          internal.parseConsoleCommands(cmd);
-        });
-      }
-
-      bad("mapshaper foo.shp")
-      bad("-i foo");
-    })
+    // // Removed this test (now, -i command is blocked in browser by
+    // // checking the execution environment).
+    // it('should block input commands', function () {
+    //   function bad(cmd) {
+    //     assert.throws(function() {
+    //       internal.parseConsoleCommands(cmd);
+    //     });
+    //   }
+    //   bad("mapshaper foo.shp")
+    //   bad("-i foo");
+    // })
 
     it('mapshaper -filter true', function () {
       var commands = internal.parseConsoleCommands('mapshaper -filter true');