Commit e8bfb580 authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Done handling of file formats and aspoint option

parent 1c8ac48b
......@@ -14,7 +14,8 @@ Osm4odf (OpenStreetMap pour OpenDataFrance) est un outil simple d'extraction th
* wget
* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql)
* [jq](https://stedolan.github.io/jq/)
* GDAL >= 3.1 (et en particulier la commande `ogr2ogr`)
* GDAL >= 3 (et en particulier la commande `ogr2ogr`)
* zip
## Configuration
......@@ -56,13 +57,13 @@ Elle décrit le type de géométrie à interroger pour ce thème. Les types poss
### Le reste du fichier
Les lignes suivantes du fichiers contiennent la requête SQL permettant d'extraire les données OpenStreetMap souhaitées. La liste des colonnes et leur ordre sera utilisé pour définir les données attributaires du fichier de sortie. La requête est à renseigner une seule fois même si l'on souhaite récupérer différents types de géométries. La table à utiliser dans le `FROM` doit être `<TABLE>` (le motif sera remplacé par l'API par le nom correct). La structure de la table à interroger correspond à ce que vous avez configuré dans le fichier `db/import.style` (voir ci-après).
Les lignes suivantes du fichiers contiennent la requête SQL permettant d'extraire les données OpenStreetMap souhaitées. La liste des colonnes et leur ordre sera utilisé pour définir les données attributaires du fichier de sortie. La requête est à renseigner une seule fois même si l'on souhaite récupérer différents types de géométries. La table à utiliser dans le `FROM` doit être `<TABLE>` (le motif sera remplacé par l'API par le nom correct). La structure de la table à interroger correspond à ce que vous avez configuré dans le fichier `db/import.style` (voir ci-après). Enfin, la géométrie de l'objet est à représenter avec la balise `<GEOM>`. Selon les paramètres passés par l'utilisateur, `<GEOM>` contiendra soit la géométrie initiale, soit le centroïde de celle-ci.
```sql
SELECT
osm_id, name, access, tags->'operator' AS operator,
tags->'opening_hours' AS opening_hours,
tags->'defibrillator:location' AS location, geom
tags->'defibrillator:location' AS location, <GEOM>
FROM <TABLE>
WHERE emergency = 'defibrillator'
```
......
......@@ -183,6 +183,35 @@
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
"dev": true
},
"compressible": {
"version": "2.0.17",
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.17.tgz",
"integrity": "sha512-BGHeLCK1GV7j1bSmQQAi26X+GgWcTjLr/0tzSvMCl3LH1w1IJ4PFSPoV5316b30cneTziC+B1a+3OjoSUcQYmw==",
"requires": {
"mime-db": "1.42.0"
}
},
"compression": {
"version": "1.7.4",
"resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz",
"integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==",
"requires": {
"accepts": "1.3.7",
"bytes": "3.0.0",
"compressible": "2.0.17",
"debug": "2.6.9",
"on-headers": "1.0.2",
"safe-buffer": "5.1.2",
"vary": "1.1.2"
},
"dependencies": {
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
}
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
......@@ -623,6 +652,11 @@
"ee-first": "1.1.1"
}
},
"on-headers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz",
"integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA=="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
......
......@@ -13,6 +13,7 @@
"author": "Adrien Pavie",
"license": "AGPL-3.0",
"dependencies": {
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
"pg": "^7.14.0"
......
......@@ -8,7 +8,7 @@ const util = require('util');
const exec = util.promisify(require('child_process').exec);
const CONFIG = require('../../config.json');
const FORMAT_TO_EXT = { "csv": "csv", "shapefile": "shp.zip", "geojson": "geojson" };
const FORMAT_TO_OGR = { "csv": "CSV", "shapefile": "ESRI Shapefile", "geojson": "GeoJSON" };
const GEOM_TO_COND = { "point": "ST_GeometryType(geom) = 'ST_Point'", "line": "ST_GeometryType(geom) = 'ST_LineString'", "polygon": "ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon')" };
// Create pool of connections
const pool = new Pool({
......@@ -67,6 +67,7 @@ function getBoundaries() {
* @param {Object} [options] Options
* @param {string} [options.format] Output file format
* @param {number} [options.radius] Radius around boundary center for data filtering
* @param {boolean} [options.aspoint] Geometry will be converted as point, instead full geometry is used
* @return {Promise} Resolves on output file
*/
function getThemeData(theme, boundary, options) {
......@@ -77,23 +78,55 @@ function getThemeData(theme, boundary, options) {
// Create view in DB
boundary = "'" + boundary.replace(/'/g, "") + "'";
const request = `CREATE VIEW ${resultViewName} AS ${theme.sql.replace(/<BID>/g, boundary)}`;
let request =
`CREATE VIEW ${resultViewName} AS
${theme.sql
.replace(/<BID>/g, boundary)
.replace(/<GEOM>/g, options.aspoint ? "ST_Centroid(geom) AS geom" : "geom")
}`;
return pool.query(request)
.then(() => {
// Run export command according to output format
const outFile = CONFIG.WORK_DIR + "/" + resultViewName + "." + FORMAT_TO_EXT[format];
const ogrCommand = `ogr2ogr -f "${FORMAT_TO_OGR[format]}" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" ${resultViewName}`;
/*
* Export commands
* Depends of the format to use
*/
let outFile = CONFIG.WORK_DIR + "/" + resultViewName + "." + FORMAT_TO_EXT[format];
const bashCommands = [];
return exec(ogrCommand)
// GeoJSON
if(format === "geojson") {
bashCommands.push(`ogr2ogr -f "GeoJSON" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" ${resultViewName} -t_srs EPSG:4326`);
}
// CSV
else if(format === "csv") {
bashCommands.push(`ogr2ogr -f "CSV" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" ${resultViewName} -t_srs EPSG:4326 -lco GEOMETRY=${options.aspoint ? "AS_XY" : "AS_WKT"}`);
}
// Shapefile
else if(format === "shapefile") {
const folder = CONFIG.WORK_DIR + "/" + resultViewName;
bashCommands.push(`mkdir -p ${folder}`);
// Process geometry type by type
(options.aspoint ? ["point"] : theme.geomtypes).forEach(type => {
const out = folder + "/" + (options.aspoint ? "data" : type) + ".shp";
bashCommands.push(`ogr2ogr -f "ESRI Shapefile" ${out} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" ${resultViewName} -where "${GEOM_TO_COND[type]}"`);
});
// Zip the whole output folder
bashCommands.push(`zip -j -r ${outFile} ${folder} && rm -rf ${folder}`);
}
// Execute command on system
return exec(bashCommands.join(" && "))
.then(result => {
// Remove view from DB
pool.query(`DROP VIEW IF EXISTS ${resultViewName}`)
.catch(e => console.log("View "+resultViewName+" not removed"));
// Send failure message
// Send failure message if any
if(result.stderr.length > 0) {
throw new Error("Can't extract data from DB: "+stderr);
throw new Error("Can't extract data from DB: "+result.stderr);
}
return outFile;
......
......@@ -4,6 +4,7 @@
const express = require('express');
const cors = require('cors');
const compression = require("compression");
const fs = require('fs');
const db = require('./db');
const themes = require('./themes');
......@@ -15,6 +16,7 @@ const app = express();
const port = process.env.PORT || 3000;
app.use(cors());
app.options('*', cors());
app.use(compression());
// Log queries
app.use((req, res, next) => {
......@@ -57,6 +59,7 @@ app.use('/doc', express.static(__dirname+'/../doc'));
* @apiParam {String} boundaryId Boundary of area where data must be extracted (one in /boundaries list)
* @apiParam {String} [format=csv] Output file format (csv, geojson, shapefile)
* @apiParam {Number} [radius] Filter by radius around boundary centroid (in kilometers) instead of boundary polygon
* @apiParam {Boolean} [aspoint=false] Geometry will always be point if set to true (center of complete geometry)
*
* @apiExample As GeoJSON:
* http://localhost:3000/data/bicycle_parking/1234?format=geojson
......@@ -83,9 +86,12 @@ app.get("/data/:themeId/:boundaryId", (req, res) => {
return res.status(400).send("Theme ID is invalid, please use one ID given on /themes URL");
}
// Check aspoint
const aspoint = req.query.aspoint === "true";
// Retrieve data
const boundary = req.params.boundaryId;
db.getThemeData(theme, boundary, { format: format, radius: radius })
db.getThemeData(theme, boundary, { format: format, radius: radius, aspoint: aspoint })
.then(filename => {
// Read file and send it
const stat = fs.statSync(filename);
......@@ -93,21 +99,24 @@ app.get("/data/:themeId/:boundaryId", (req, res) => {
res.writeHead(200, {
'Content-Type': FORMAT_TO_MIME[format],
'Content-Length': stat.size,
'Content-Disposition': `attachment; filename="extract.${format}"`
'Content-Disposition': `attachment; filename="extract.${FORMAT_TO_EXT[format]}"`
});
var readStream = fs.createReadStream(filename);
readStream.pipe(res);
// Handle errors on sending
res.on('error', (err) => {
readStream.on('error', (err) => {
readStream.end();
fs.unlink(filename);
});
// Delete output file
res.on('end', () => {
fs.unlink(filename);
readStream.on('end', () => {
fs.unlink(filename, (err) => {
if(err) {
console.log("Can't delete tmp file: ", err);
}
});
});
})
.catch(e => {
......
......@@ -37,7 +37,7 @@ fs.readdirSync(folder).forEach((file) => {
let completeSQL = "WITH data AS ("+geom.map(g => "SELECT p.*, p.way AS geom FROM planet_osm_"+g+" p, boundary b WHERE b.osm_id = <BID> AND p.way && b.way AND ST_Intersects(p.way, b.way)").join(" UNION ")+") ";
completeSQL += sql.replace(/<TABLE>/g, "data");
themes.push(Object.assign({ id: themeId, sql: completeSQL.trim() }, metadata));
themes.push(Object.assign({ id: themeId, sql: completeSQL.trim(), geomtypes: geom }, metadata));
}
catch(e) {
throw new Error("Invalid theme file: "+file+" "+e.message);
......
......@@ -4,6 +4,6 @@
SELECT
osm_id, name, access, tags->'operator' AS operator,
tags->'opening_hours' AS opening_hours,
tags->'defibrillator:location' AS location, geom
tags->'defibrillator:location' AS location, <GEOM>
FROM <TABLE>
WHERE emergency = 'defibrillator'
......@@ -4,6 +4,6 @@
SELECT
osm_id, name, access, tags->'operator' AS operator,
tags->'covered' AS covered, tags->'capacity' AS capacity,
tags->'bicycle_parking' AS type, geom
tags->'bicycle_parking' AS type, <GEOM>
FROM <TABLE>
WHERE amenity = 'bicycle_parking'
......@@ -2,6 +2,6 @@
--GEOMETRY=point,polygon
SELECT
osm_id, name, tags->'amenity' AS type, tags->'operator:type' AS statut, geom
osm_id, name, tags->'amenity' AS type, tags->'operator:type' AS statut, <GEOM>
FROM <TABLE>
WHERE amenity IN ('school', 'kindergarten', 'college', 'university')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment