Commit 1c8ac48b authored by Adrien Pavie's avatar Adrien Pavie
Browse files

WIP route for data extract

parent 4c64b8b8
......@@ -14,6 +14,7 @@ Osm4odf (OpenStreetMap pour OpenDataFrance) est un outil simple d'extraction th
* wget
* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql)
* [jq](https://stedolan.github.io/jq/)
* GDAL >= 3.1 (et en particulier la commande `ogr2ogr`)
## Configuration
......
......@@ -4,7 +4,11 @@
const { Pool } = require('pg');
const fs = require('fs');
const util = require('util');
const exec = util.promisify(require('child_process').exec);
const CONFIG = require('../../config.json');
const FORMAT_TO_EXT = { "csv": "csv", "shapefile": "shp.zip", "geojson": "geojson" };
const FORMAT_TO_OGR = { "csv": "CSV", "shapefile": "ESRI Shapefile", "geojson": "GeoJSON" };
// Create pool of connections
const pool = new Pool({
......@@ -56,6 +60,49 @@ function getBoundaries() {
});
}
/**
* Extract thematic data
* @param {Object} theme The theme metadata
* @param {string} boundary The boundary ID
* @param {Object} [options] Options
* @param {string} [options.format] Output file format
* @param {number} [options.radius] Radius around boundary center for data filtering
* @return {Promise} Resolves on output file
*/
function getThemeData(theme, boundary, options) {
return whenAvailable()
.then(() => {
const resultViewName = "tmp_"+Date.now();
const format = options.format || "csv";
// Create view in DB
boundary = "'" + boundary.replace(/'/g, "") + "'";
const request = `CREATE VIEW ${resultViewName} AS ${theme.sql.replace(/<BID>/g, boundary)}`;
return pool.query(request)
.then(() => {
// Run export command according to output format
const outFile = CONFIG.WORK_DIR + "/" + resultViewName + "." + FORMAT_TO_EXT[format];
const ogrCommand = `ogr2ogr -f "${FORMAT_TO_OGR[format]}" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" ${resultViewName}`;
return exec(ogrCommand)
.then(result => {
// Remove view from DB
pool.query(`DROP VIEW IF EXISTS ${resultViewName}`)
.catch(e => console.log("View "+resultViewName+" not removed"));
// Send failure message
if(result.stderr.length > 0) {
throw new Error("Can't extract data from DB: "+stderr);
}
return outFile;
});
});
});
}
module.exports = {
getBoundaries: getBoundaries
getBoundaries: getBoundaries,
getThemeData: getThemeData
};
......@@ -7,6 +7,8 @@ const cors = require('cors');
const fs = require('fs');
const db = require('./db');
const themes = require('./themes');
const FORMAT_TO_MIME = { "csv": "text/csv", "shapefile": "application/zip", "geojson": "application/vnd.geo+json" };
const FORMAT_TO_EXT = { "csv": "csv", "shapefile": "shp.zip", "geojson": "geojson" };
// Init API
const app = express();
......@@ -61,6 +63,58 @@ app.use('/doc', express.static(__dirname+'/../doc'));
* @apiExample As CSV, using radius:
* http://localhost:3000/data/bicycle_parking/1234?format=csv&radius=5
*/
app.get("/data/:themeId/:boundaryId", (req, res) => {
// Check format
const formats = [ "csv", "geojson", "shapefile" ];
const format = req.query.format || "csv";
if(!formats.includes(format)) {
return res.status(400).send("format parameter is invalid (should be "+formats.join(", ")+")");
}
// Check radius
const radius = req.query.radius || -1;
if(isNaN(parseFloat(radius))) {
return res.status(400).send("radius should either not be defined, or be a positive number");
}
// Check theme ID
const theme = themes.full.find(t => t.id === req.params.themeId);
if(!theme) {
return res.status(400).send("Theme ID is invalid, please use one ID given on /themes URL");
}
// Retrieve data
const boundary = req.params.boundaryId;
db.getThemeData(theme, boundary, { format: format, radius: radius })
.then(filename => {
// Read file and send it
const stat = fs.statSync(filename);
res.writeHead(200, {
'Content-Type': FORMAT_TO_MIME[format],
'Content-Length': stat.size,
'Content-Disposition': `attachment; filename="extract.${format}"`
});
var readStream = fs.createReadStream(filename);
readStream.pipe(res);
// Handle errors on sending
res.on('error', (err) => {
readStream.end();
fs.unlink(filename);
});
// Delete output file
res.on('end', () => {
fs.unlink(filename);
});
})
.catch(e => {
console.log(e);
res.status(500).send("Can't extract data: "+e.message);
});
});
/**
* @api {get} /themes List of themes
......
......@@ -35,8 +35,8 @@ fs.readdirSync(folder).forEach((file) => {
// Append data
themesLight.push(Object.assign({ id: themeId }, metadata));
let completeSQL = "WITH data AS ("+geom.map(g => "SELECT * FROM planet_osm_"+g+" WHERE <GEOMCOND>").join(" UNION ")+") ";
completeSQL += sql.replace("<TABLE>", "data");
let completeSQL = "WITH data AS ("+geom.map(g => "SELECT p.*, p.way AS geom FROM planet_osm_"+g+" p, boundary b WHERE b.osm_id = <BID> AND p.way && b.way AND ST_Intersects(p.way, b.way)").join(" UNION ")+") ";
completeSQL += sql.replace(/<TABLE>/g, "data");
themes.push(Object.assign({ id: themeId, sql: completeSQL.trim() }, metadata));
}
catch(e) {
......
{
"WORK_DIR": "/tmp/osm4odf",
"PBF_URL": "http://download.geofabrik.de/europe/france/bretagne-latest.osm.pbf",
"WORK_DIR": "/home/adrien/tmp/osm4odf",
"PBF_URL": "http://download.geofabrik.de/europe/france/reunion-latest.osm.pbf",
"PARALLEL_JOBS": "6",
"PG_HOST": "localhost",
......
--METADATA={ "name:fr": "Lieux d'enseignement", "name:en": "Education" }
--GEOMETRY=point,polygon
SELECT
osm_id, name, tags->'amenity' AS type, tags->'operator:type' AS statut, geom
FROM <TABLE>
WHERE amenity IN ('school', 'kindergarten', 'college', 'university')
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment