Skip to content

Instantly share code, notes, and snippets.

@rcoup

rcoup/pr-1889.patch

Created April 15, 2013 23:34
Show Gist options
  • Save rcoup/5392176 to your computer and use it in GitHub Desktop.
Save rcoup/5392176 to your computer and use it in GitHub Desktop.
Download ZIP
Squashed patch for https://github.com/mapbox/tilemill/pull/1889 (Export to mbtiles with bounds that cross the anti-meridian)
Raw
pr-1889.patch
From 209281e89e5130813186ac3cbc345b15353aea3b Mon Sep 17 00:00:00 2001
From: Ryan Lewis <[email protected]>
Date: Thu, 31 Jan 2013 17:08:34 +1300
Subject: [PATCH] Let users export mbtiles that cross the anti meridian by
writing to the file multiple times with different bounds.
* Only applies to mbtiles exports
* Doesn't allow saving projects across the anti-meridian (since loading errors out)
---
commands/export.bones | 216 +++++++++++++++------------
models/Export.bones | 3 +-
test/export.test.js | 76 ++++++++++
test/fixtures/anti-meridian-export-job.json | 13 ++
views/Metadata.bones | 9 +-
5 files changed, 221 insertions(+), 96 deletions(-)
create mode 100644 test/fixtures/anti-meridian-export-job.json
diff --git a/commands/export.bones b/commands/export.bones
index 8956581..14358ea 100644
--- a/commands/export.bones
+++ b/commands/export.bones
@@ -391,117 +391,147 @@ command.prototype.tilelive = function (project, callback) {
require('tilelive-mapnik').registerProtocols(tilelive);
- var opts = this.opts;
-
- // Try to load a job file if one was given and it exists.
- if (opts.job) {
- opts.job = path.resolve(opts.job);
- try {
- var job = fs.readFileSync(opts.job, 'utf8');
- } catch(err) {
- if (err.code !== 'EBADF') throw err;
- }
- } else {
- // Generate a job file based on the output filename.
- var slug = path.basename(opts.filepath, path.extname(opts.filepath));
- opts.job = path.join(path.dirname(opts.filepath), slug + '.export');
+ // Copy the bounds so that we can modify them if we need to split the export
+ var bboxes = [ project.mml.bounds.slice(0) ];
+ if (project.mml.bounds[2] < project.mml.bounds[0]) {
+ bboxes.push(bboxes[0].slice(0));
+ bboxes[0][2] = 180;
+ bboxes[1][0] = -180;
+
+ // If we don't set the bounds to the whole world tilestream will not be able to display the files
+ // because the center of the bounds will be outside the bounds.
+ project.mml.bounds = [ -180, project.mml.bounds[1], 180, project.mml.bounds[3] ];
}
- if (job) {
- job = JSON.parse(job);
- if (!cmd.opts.quiet) console.warn('Continuing job ' + opts.job);
- var scheme = tilelive.Scheme.unserialize(job.scheme);
- var task = new tilelive.CopyTask(job.from, job.to, scheme, opts.job);
- } else {
- if (!cmd.opts.quiet) console.warn('Creating new job ' + opts.job);
-
- var from = {
- protocol: 'mapnik:',
- slashes: true,
- xml: project.xml,
- mml: project.mml,
- pathname: path.join(opts.files, 'project', project.id, project.id + '.xml'),
- query: {
- metatile: project.mml.metatile,
- scale: project.mml.scale
+ function exportTiles(bboxIndex) {
+ if (bboxIndex >= bboxes.length) {
+ return callback();
+ }
+ var opts = $.extend({}, cmd.opts);
+
+ // Try to load a job file if one was given and it exists.
+ if (opts.job) {
+ opts.job = path.resolve(opts.job);
+ try {
+ var job = fs.readFileSync(opts.job, 'utf8');
+ } catch(err) {
+ if (err.code !== 'EBADF') throw err;
}
- };
-
- var to = {
- protocol: opts.format + ':',
- pathname: opts.filepath,
- query: { batch: 100 }
- };
+ } else {
+ // Generate a job file based on the output filename.
+ var slug = path.basename(opts.filepath, path.extname(opts.filepath));
+ opts.job = path.join(path.dirname(opts.filepath), slug + '.export');
+ }
- var scheme = tilelive.Scheme.create(opts.scheme, {
- list: opts.list,
- bbox: project.mml.bounds,
- minzoom: project.mml.minzoom,
- maxzoom: project.mml.maxzoom,
- metatile: project.mml.metatile,
- concurrency: Math.floor(
- Math.pow(project.mml.metatile, 2) * // # of tiles in each metatile
- require('os').cpus().length * // expect one metatile to occupy each core
- 4 / cmd.opts.concurrency // overcommit x4 throttle by export concurrency
- )
- });
- var task = new tilelive.CopyTask(from, to, scheme, opts.job);
- }
+ if (job) {
+ job = JSON.parse(job);
+ if (!cmd.opts.quiet) console.warn('Continuing job ' + opts.job);
+ bboxIndex = job.from.bboxIndex;
+ // If we don't reset the filepath here the next bbox exported will be exported using a filepath with a hash
+ // instead of into the same file.
+ cmd.opts.filepath = job.to.pathname;
+ var scheme = tilelive.Scheme.unserialize(job.scheme);
+ var task = new tilelive.CopyTask(job.from, job.to, scheme, opts.job);
+ } else {
+ if (!cmd.opts.quiet) console.warn('Creating new job ' + opts.job);
+
+ var from = {
+ protocol: 'mapnik:',
+ slashes: true,
+ xml: project.xml,
+ mml: project.mml,
+ pathname: path.join(opts.files, 'project', project.id, project.id + '.xml'),
+ query: {
+ metatile: project.mml.metatile,
+ scale: project.mml.scale
+ },
+ // Add a hash with the bounding box to prevent tilelive pulling data from the cache
+ // that has the previously exported bounds.
+ hash: "bbox=" + bboxes[bboxIndex].join(','),
+ bboxIndex: bboxIndex
+ };
+
+ var to = {
+ protocol: opts.format + ':',
+ pathname: opts.filepath,
+ query: { batch: 100 }
+ };
+
+ var scheme = tilelive.Scheme.create(opts.scheme, {
+ list: opts.list,
+ bbox: bboxes[bboxIndex],
+ minzoom: project.mml.minzoom,
+ maxzoom: project.mml.maxzoom,
+ metatile: project.mml.metatile,
+ concurrency: Math.floor(
+ Math.pow(project.mml.metatile, 2) * // # of tiles in each metatile
+ require('os').cpus().length * // expect one metatile to occupy each core
+ 4 / cmd.opts.concurrency // overcommit x4 throttle by export concurrency
+ )
+ });
+ var task = new tilelive.CopyTask(from, to, scheme, opts.job);
+ }
- var errorfile = path.join(path.dirname(opts.job), path.basename(opts.job) + '-failed');
- if (!cmd.opts.quiet) console.warn('Writing errors to ' + errorfile);
+ var errorfile = path.join(path.dirname(opts.job), path.basename(opts.job) + '-failed');
+ if (!cmd.opts.quiet) console.warn('Writing errors to ' + errorfile);
- fs.open(errorfile, 'a', function(err, fd) {
- if (err) throw err;
+ fs.open(errorfile, 'a', function(err, fd) {
+ if (err) throw err;
- task.on('error', function(err, tile) {
- console.warn('\r\033[K' + tile.toString() + ': ' + err.message);
- fs.write(fd, JSON.stringify(tile) + '\n');
- report(task.stats.snapshot());
- });
+ task.on('error', function(err, tile) {
+ console.warn('\r\033[K' + tile.toString() + ': ' + err.message);
+ fs.write(fd, JSON.stringify(tile) + '\n');
+ report(task.stats.snapshot());
+ });
- task.on('progress', report);
+ task.on('progress', report);
- task.on('finished', function() {
- if (!cmd.opts.quiet) console.warn('\nfinished');
- callback();
- });
+ task.on('finished', function() {
+ if (!cmd.opts.quiet) console.warn('\nfinished');
+ cmd.opts.job = false;
+ exportTiles(bboxIndex + 1);
+ });
- task.start(function(err) {
- if (err) throw err;
- task.sink.putInfo(project.mml, function(err) {
+ task.start(function(err) {
if (err) throw err;
+ task.sink.putInfo(project.mml, function(err) {
+ if (err) throw err;
+ });
});
});
- });
- function report(stats) {
- var progress = stats.processed / stats.total;
- var remaining = cmd.remaining(progress, task.started);
- cmd.put({
- status: 'processing',
- progress: progress,
- remaining: remaining,
- updated: +new Date(),
- rate: stats.speed
- });
+ function report(stats) {
+ var progress = stats.processed / stats.total;
+ var remaining = cmd.remaining(progress, task.started);
+ cmd.put({
+ status: 'processing',
+ progress: progress,
+ remaining: remaining,
+ updated: +new Date(),
+ rate: stats.speed
+ });
- if (!cmd.opts.quiet) {
- util.print(formatString('\r\033[K[%s] %s%% %s/%s @ %s/s | %s left | ✓ %s ■ %s □ %s fail %s',
- formatDuration(stats.date - task.started),
- ((progress || 0) * 100).toFixed(4),
- formatNumber(stats.processed),
- formatNumber(stats.total),
- formatNumber(stats.speed),
- formatDuration(remaining),
- formatNumber(stats.unique),
- formatNumber(stats.duplicate),
- formatNumber(stats.skipped),
- formatNumber(stats.failed)
- ));
+ if (!cmd.opts.quiet) {
+ util.print(formatString('\r\033[K[%s] Part(%s/%s) %s%% %s/%s @ %s/s | %s left | ✓ %s ■ %s □ %s fail %s',
+ formatDuration(stats.date - task.started),
+ bboxIndex + 1,
+ bboxes.length,
+ ((progress || 0) * 100).toFixed(4),
+ formatNumber(stats.processed),
+ formatNumber(stats.total),
+ formatNumber(stats.speed),
+ formatDuration(remaining),
+ formatNumber(stats.unique),
+ formatNumber(stats.duplicate),
+ formatNumber(stats.skipped),
+ formatNumber(stats.failed)
+ ));
+ }
}
}
+
+ exportTiles(0);
};
command.prototype.upload = function (callback) {
diff --git a/models/Export.bones b/models/Export.bones
index 05f4937..73da867 100644
--- a/models/Export.bones
+++ b/models/Export.bones
@@ -91,7 +91,8 @@ model.prototype.validate = function(attr) {
var error = this.validateAttributes(attr);
if (error) return error;
- if (attr.bbox && attr.bbox[0] >= attr.bbox[2])
+ var format = this.get('format') || attr.format;
+ if (format !== 'mbtiles' && attr.bbox && attr.bbox[0] >= attr.bbox[2])
return new Error('Bounds W must be less than E.');
if (attr.bbox && attr.bbox[1] >= attr.bbox[3])
return new Error('Bounds S must be less than N.');
diff --git a/test/export.test.js b/test/export.test.js
index a68e16f..59e4b45 100644
--- a/test/export.test.js
+++ b/test/export.test.js
@@ -27,8 +27,10 @@ after(function(done) {
var id = Date.now().toString();
var job = readJSON('export-job');
+var antiMeridianJob = readJSON('anti-meridian-export-job');
var token = job['bones.token'];
job.id = id;
+antiMeridianJob.id = id + 1;
it('PUT should create export job', function(done) {
assert.response(core, {
url: '/api/Export/' + id,
@@ -79,6 +81,80 @@ it('DELETE should stop export job', function(done) {
done();
});
});
+it('PUT create export job spanning the anti-meridian', function(done) {
+ assert.response(core, {
+ url: '/api/Export/' + antiMeridianJob.id,
+ method: 'PUT',
+ data: JSON.stringify(antiMeridianJob),
+ headers: {
+ cookie: "bones.token=" + token,
+ 'content-type': "application/json"
+ }
+ }, {
+ body: '{}',
+ status: 200
+ }, function (res) {
+ assert.deepEqual(JSON.parse(res.body), {});
+ done();
+ });
+});
+it("GET anti-meridian job to see if it succeeded", function(done) {
+ function ping() {
+ assert.response(core, {
+ url: '/api/Export/' + antiMeridianJob.id,
+ method: 'GET',
+ headers: {
+ cookie: "bones.token=" + token,
+ 'content-type': "application/json"
+ }
+ }, {
+ status: 200
+ }, function(res) {
+ var data = JSON.parse(res.body);
+ if (data.status === 'processing') {
+ // Don't flood the socket or the test will fail.
+ setTimeout(ping, 500);
+ } else if (data.status === 'error') {
+ throw new Error(data.error);
+ } else {
+ antiMeridianJob.status = "complete";
+ antiMeridianJob.progress = 1;
+ console.log(data);
+ assert.ok(data.pid);
+ assert.ok(data.created);
+ delete antiMeridianJob['bones.token'];
+ delete data.created;
+ delete data.pid;
+
+ delete data.remaining;
+ delete data.updated;
+ delete data.rate;
+
+ assert.deepEqual(antiMeridianJob, data);
+ done();
+ }
+ });
+ }
+ ping();
+});
+it('DELETE should remove anti-meridian export job', function(done) {
+ job['bones.token'] = token;
+ assert.response(core, {
+ url: '/api/Export/' + antiMeridianJob.id,
+ method: 'DELETE',
+ headers: {
+ cookie: "bones.token=" + token,
+ 'content-type': "application/json"
+ },
+ body: JSON.stringify(job)
+ }, {
+ body: '{}',
+ status: 200
+ }, function (res) {
+ assert.deepEqual(JSON.parse(res.body), {});
+ done();
+ });
+});
it('GET should find no export jobs', function(done) {
assert.response(core, {
url: '/api/Export'
diff --git a/test/fixtures/anti-meridian-export-job.json b/test/fixtures/anti-meridian-export-job.json
new file mode 100644
index 0000000..5a38047
--- /dev/null
+++ b/test/fixtures/anti-meridian-export-job.json
@@ -0,0 +1,13 @@
+{
+ "progress": 0,
+ "status": "waiting",
+ "format": "mbtiles",
+ "project": "demo_01",
+ "tile_format": "png",
+ "id": "",
+ "filename": "demo_anti-meridian.mbtiles",
+ "bbox": [ 175, 31.8402, -175, 71.9245 ],
+ "minzoom": 0,
+ "maxzoom": 2,
+ "bones.token": "zbx6dr0ghgvRNZOuu8PXtt2VCAIKO2qK"
+}
\ No newline at end of file
diff --git a/views/Metadata.bones b/views/Metadata.bones
index c8a807f..13065ec 100644
--- a/views/Metadata.bones
+++ b/views/Metadata.bones
@@ -86,7 +86,7 @@ view.prototype.render = function() {
tj.minzoom = 0;
tj.maxzoom = 22;
this.map = new MM.Map('meta-map', new wax.mm.connector(tj));
-
+
// Override project attributes to allow unbounded zooming.
this.map.setZoomRange(
tj.minzoom,
@@ -299,7 +299,12 @@ view.prototype.save = function() {
delete attr.filename;
delete attr.width;
delete attr.height;
- if (!this.project.set(attr, {error:error})) return false;
+ if (!this.project.set(attr, {error: function(m, e) {
+ if (e.message === "Bounds W must be less than E.") {
+ e.message = "Cannot save to project if export crosses the Anti-Meridian";
+ }
+ error(m, e);
+ }})) return false;
Bones.utils.serial([
_(function(next) {
this.model.save({}, { success:next, error:this.error });
--
1.7.10.4
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment