Commit 901ed16f authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Metadata file available as API option, by default in UI

parent 5e9f7f03
......@@ -18,7 +18,7 @@ Les dépendances sont les suivantes :
* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql)
* [jq](https://stedolan.github.io/jq/)
* GDAL >= 2.1 (et en particulier la commande `ogr2ogr`)
* zip
* zip et zipnote
* [osmium-tool](https://github.com/osmcode/osmium-tool) >= 1.11
Installation des dépendances sous Debian 9 :
......@@ -115,9 +115,17 @@ Les thèmes de données sont à configurer dans le dossier `themes/`. Chaque th
Elle décrit les métadonnées du thème, au format JSON avec un préfixe. On y trouve en particulier le nom en français et anglais.
```sql
--METADATA={ "name:fr": "Le nom en français", "name:en": "Le nom en anglais" }
--METADATA={ "name:fr": "Le nom en français", "name:en": "Le nom en anglais", "theme:fr": "Le thème", "keywords:fr": [ "mot clé 1", "mot clé 2" ], "description:fr": "Blabla" }
```
Les propriétés à renseigner sont :
* `name:fr` : nom du thème en français
* `name:en` : nom du thème en anglais
* `theme:fr` : thématique du jeu de données, sur la base de la nomenclature Eurovoc ou de la [nomenclature OpenDataFrance](https://docs.google.com/document/d/1oDJsHw3bmABfto6HPgCPG1ztrV3CihuHjcfU8tQpvPc/edit#heading=h.5aw35qsix45f)
* `keywords:fr` : liste de mots clés, sous la forme d'un tableau
* `description:fr` : un bref résumé du jeu de données, ainsi que des clés OSM utilisées pour l'extraction
### 2e ligne du fichier
Elle décrit le type de géométrie à interroger pour ce thème. Les types possibles sont `point`, `line`, `polygon`. Il est possible d'en renseigner plusieurs, séparés par une virgule.
......
......@@ -22,6 +22,13 @@ function createPool() {
});
}
function isoToday() {
return (new Date()).toISOString().split("T")[0];
}
function toSlug(text) {
return text.toLowerCase().normalize("NFD").replace(/[\u0300-\u036f]/g, "").replace(/[ '"_]/g, "-");
}
/**
* Checks if database is ready to receive requests (running and not being updated)
......@@ -104,14 +111,49 @@ function findBoundariesByName(name) {
}
}
/**
* Generate metadata text
*/
function getMetadataText(theme, boundary, format, url) {
let text = "COLL_NOM;COLL_SIRET;ID;TITRE;DESCRIPTION;THEME;PRODUCTEUR_NOM;PRODUCTEUR_SIRET;COUV_SPAT_MAILLE;COUV_SPAT_NOM;COUV_TEMP_DEBUT;COUV_TEMP_FIN;DATE_PUBL;FREQ_MAJ;DATE_MAJ;MOTS_CLES;LICENCE;NOMBRE_RESSOURCES;FORMAT_RESSOURCES;URL";
const params = [
boundary.name,
boundary.ref,
`osm-${theme.id}-${boundary.ref || boundary.id}`,
`${theme["name:fr"]} de ${boundary.name} (OpenStreetMap)`,
`"${theme["description:fr"].replace(/"/g, '"""')}"`,
theme["theme:fr"],
"OpenStreetMap",
"",
"Infracommunale",
boundary.name,
"",
isoToday(),
isoToday(),
"Ponctuelle",
isoToday(),
`"scdl;${toSlug(theme["theme:fr"])}${theme["keywords:fr"] ? ";" + theme["keywords:fr"].map(t => toSlug(t)).join(";") : ""}"`,
"Open Database License-ODBL",
"1",
format,
url
];
text += "\n" + params.join(";");
return text;
}
/**
* Extract thematic data
* @param {Object} theme The theme metadata
* @param {string} boundary The boundary ID
* @param {Object} boundary The boundary metadata
* @param {Object} [options] Options
* @param {string} [options.format] Output file format
* @param {number} [options.radius] Radius around boundary center for data filtering
* @param {boolean} [options.aspoint] Geometry will be converted as point, instead full geometry is used
* @param {boolean} [options.metadata] Add a metadata file (SCDL format)
* @param {string} [options.url] Original URL used for getting this dataset
* @return {Promise} Resolves on output file
*/
function getThemeData(theme, boundary, options) {
......@@ -119,13 +161,15 @@ function getThemeData(theme, boundary, options) {
.then(() => {
const start = Date.now();
const format = options.format || "csv";
const resultViewName = theme.id + boundary + Date.now();
const metadata = options.metadata || false;
const resultViewName = theme.id + boundary.id.toString() + Date.now();
const baseFilePath = CONFIG.WORK_DIR + "/" + resultViewName;
// Create view in DB
boundary = "'" + boundary.replace(/'/g, "") + "'";
// Create SQL request
const boundaryId = "'" + boundary.id.toString().replace(/'/g, "") + "'";
const dist = options.radius > 0 ? options.radius * 1000 : null;
const hasAdm8 = theme.sql.includes("<ADM8>");
const geomCond = `b.osm_id = ${boundary} AND ${dist ? `ST_DWithin(t.way, b.center, ${dist})` : "t.way && b.way AND ST_Intersects(t.way, b.way)"} ${hasAdm8 ? " AND b2.type = 'admin_8' AND ST_Contains(b2.way, ST_Centroid(t.way))" : ""}`;
const geomCond = `b.osm_id = ${boundaryId} AND ${dist ? `ST_DWithin(t.way, b.center, ${dist})` : "t.way && b.way AND ST_Intersects(t.way, b.way)"} ${hasAdm8 ? " AND b2.type = 'admin_8' AND ST_Contains(b2.way, ST_Centroid(t.way))" : ""}`;
let request = theme.geomtypes.map(g => (
theme.sql
.replace(/<TABLE>/g, `boundary b, planet_osm_${g} t${hasAdm8 ? ", boundary b2" : ""}`)
......@@ -133,11 +177,18 @@ function getThemeData(theme, boundary, options) {
.replace(/<ADM8>/g, "b2.name AS adm8_name, b2.ref AS adm8_ref")
)).join(" UNION ");
// Generate metadata file content
const metadataFile = baseFilePath + "_metadata.csv";
if(metadata) {
const metadataTxt = getMetadataText(theme, boundary, format, options.url);
fs.writeFileSync(metadataFile, metadataTxt);
}
/*
* Export commands
* Depends of the format to use
*/
let outFile = CONFIG.WORK_DIR + "/" + resultViewName + "." + FORMAT_TO_EXT[format];
let outFile = baseFilePath + "." + FORMAT_TO_EXT[format];
const bashCommands = [];
const sql2bash = txt => txt.replace(/"/g, '\\"').replace(/\n/g, " ");
......@@ -149,29 +200,38 @@ function getThemeData(theme, boundary, options) {
// CSV
else if(format === "csv") {
request = request.replace(/<GEOMCOND>/g, geomCond);
bashCommands.push(`ogr2ogr -f "CSV" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" -sql "${sql2bash(request)}" -t_srs EPSG:4326 -lco GEOMETRY=${options.aspoint ? "AS_XY" : "AS_WKT"}`);
bashCommands.push(`ogr2ogr -f "CSV" ${outFile} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" -sql "${sql2bash(request)}" -t_srs EPSG:4326 -lco GEOMETRY=${options.aspoint ? "AS_XY" : "AS_WKT"} -lco SEPARATOR=SEMICOLON`);
}
// Shapefile
else if(format === "shapefile") {
const folder = CONFIG.WORK_DIR + "/" + resultViewName;
bashCommands.push(`mkdir -p ${folder}`);
bashCommands.push(`mkdir -p ${baseFilePath}`);
// Process geometry type by type
if(options.aspoint) {
const out = folder + "/data.shp";
const out = baseFilePath + "/data.shp";
request = request.replace(/<GEOMCOND>/g, geomCond);
bashCommands.push(`ogr2ogr -f "ESRI Shapefile" ${out} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" -sql "${sql2bash(request)}" -lco ENCODING=UTF-8`);
}
else {
theme.geomtypes.forEach(type => {
const out = folder + "/" + type + ".shp";
const out = baseFilePath + "/" + type + ".shp";
const typeReq = request.replace(/<GEOMCOND>/g, GEOM_TO_COND[type] + " AND " + geomCond);
bashCommands.push(`ogr2ogr -f "ESRI Shapefile" ${out} PG:"host=${CONFIG.PG_HOST} user=${CONFIG.PG_USER} dbname=${CONFIG.PG_DB_CLEAN} port=${CONFIG.PG_PORT}" -sql "${sql2bash(typeReq)}" -lco ENCODING=UTF-8`);
});
}
// Zip the whole output folder
bashCommands.push(`zip -j -r ${outFile} ${folder} && rm -rf ${folder}`);
bashCommands.push(`zip -j -r ${outFile} ${baseFilePath} && rm -rf ${baseFilePath}`);
}
// Add metadata along dataset
if(metadata) {
const zipFile = baseFilePath + ".zip";
bashCommands.push(`zip -j ${zipFile} ${outFile} ${metadataFile}`);
bashCommands.push(`printf "@ ${outFile.replace(CONFIG.WORK_DIR + "/", "")}\n@=data.${FORMAT_TO_EXT[format]}\n" | zipnote -w ${zipFile}`);
bashCommands.push(`printf "@ ${metadataFile.replace(CONFIG.WORK_DIR + "/", "")}\n@=metadata.csv\n" | zipnote -w ${zipFile}`);
bashCommands.push(`rm ${outFile} ${metadataFile}`);
outFile = zipFile;
}
// console.log(request);
......@@ -187,7 +247,7 @@ function getThemeData(theme, boundary, options) {
}
}
console.log(`Execution time (${theme.id} on ${boundary}): ${Date.now()-start}ms`);
console.log(`Execution time (${theme.id} on ${boundary.name}): ${Date.now()-start}ms`);
return outFile;
});
});
......
......@@ -8,8 +8,8 @@ const compression = require("compression");
const fs = require('fs');
const db = require('./db');
const themes = require('./themes');
const FORMAT_TO_MIME = { "csv": "text/csv", "shapefile": "application/zip", "geojson": "application/vnd.geo+json" };
const FORMAT_TO_EXT = { "csv": "csv", "shapefile": "shp.zip", "geojson": "geojson" };
const FORMAT_TO_MIME = { "csv": "text/csv", "shapefile": "application/zip", "geojson": "application/vnd.geo+json", "zip": "application/zip" };
const FORMAT_TO_EXT = { "csv": "csv", "shapefile": "shp.zip", "geojson": "geojson", "zip": "zip" };
// Init API
const app = express();
......@@ -68,6 +68,7 @@ app.use('/doc', express.static(__dirname+'/../doc'));
* @apiParam {String} [format=csv] Output file format (csv, geojson, shapefile)
* @apiParam {Number} [radius] Filter by radius around boundary centroid (in kilometers) instead of boundary polygon (max 50)
* @apiParam {Boolean} [aspoint=false] Geometry will always be point if set to true (center of complete geometry)
* @apiParam {Boolean} [metadata=false] Add a metadata file for the generated dataset, in <a href="https://scdl.opendatafrance.net/docs/schemas/catalogue.html">SCDL format</a>
*
* @apiExample As GeoJSON:
* http://localhost:3000/data/bicycle_parking/1234?format=geojson
......@@ -100,21 +101,29 @@ app.get("/data/:themeId/:boundaryId", (req, res) => {
// Check aspoint
const aspoint = req.query.aspoint === "true";
// Check metadata
const metadata = req.query.metadata === "true";
// Save used URL (for metadata)
const myPort = req.app.settings.port || port;
const url = req.protocol + '://' + req.hostname + (myPort == 80 || myPort == 443 ? '' : ':' + myPort) + req.originalUrl;;
// Retrieve data
const boundaryId = req.params.boundaryId;
db.getBoundary(boundaryId)
.then(boundary => {
return db.getThemeData(theme, boundaryId, { format: format, radius: radius, aspoint: aspoint })
return db.getThemeData(theme, boundary, { format: format, radius: radius, aspoint: aspoint, metadata: metadata, url: url })
.then(filename => {
// Read file and send it
const stat = fs.statSync(filename);
const day = (new Date()).toISOString().split("T")[0];
const boundaryName = boundary.name.toLowerCase().replace(/ /g, "_");
const finalFormat = metadata ? "zip" : format;
res.writeHead(200, {
'Content-Type': FORMAT_TO_MIME[format],
'Content-Type': FORMAT_TO_MIME[finalFormat],
'Content-Length': stat.size,
'Content-Disposition': `attachment; filename="${boundaryName}_${theme.id}_${day}.${FORMAT_TO_EXT[format]}"`
'Content-Disposition': `attachment; filename="${boundaryName}_${theme.id}_${day}.${FORMAT_TO_EXT[finalFormat]}"`
});
var readStream = fs.createReadStream(filename);
......
......@@ -70,13 +70,20 @@
</select>
</div>
<div class="form-check mb-3">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="aspoint" checked />
<label class="form-check-label" for="aspoint">
Tout simplifier sous forme de points
</label>
</div>
<div class="form-check mb-3">
<input class="form-check-input" type="checkbox" id="metadata" checked />
<label class="form-check-label" for="metadata">
Inclure une fiche de métadonnées
</label>
</div>
<div class="form-group">
<label for="radius">Rayon de recherche</label>
<input type="range" class="custom-range" id="radius" min="0" max="50" value="0" />
......
......@@ -132,6 +132,7 @@ function createForm() {
// Follow options updates
document.getElementById("format").addEventListener("change", () => setDownloadable(true));
document.getElementById("aspoint").addEventListener("change", () => setDownloadable(true));
document.getElementById("metadata").addEventListener("change", () => setDownloadable(true));
}
......@@ -150,6 +151,7 @@ function setDownloadable(shouldI) {
const opts = {
format: document.getElementById("format").value,
aspoint: document.getElementById("aspoint").checked ? "true" : "",
metadata: document.getElementById("metadata").checked ? "true" : "",
radius: parseInt(iptRadius.value) === 0 ? null : iptRadius.value
};
......
{
"WORK_DIR": "/home/adrien/tmp/osm4odf",
"PBF_URL": "http://download.openstreetmap.fr/extracts/europe/france/bretagne-latest.osm.pbf",
"PBF_URL": "http://download.openstreetmap.fr/extracts/europe/france/bretagne/ille_et_vilaine-latest.osm.pbf",
"PARALLEL_JOBS": "6",
"PG_HOST": "localhost",
"PG_USER": "postgres",
"PG_PORT": "5432",
"PG_DB_CLEAN": "osm4odf",
"PG_DB_TMP": "osm4odf_next",
"PG_DB_OLD": "osm4odf_prev",
"PG_DB_CLEAN": "geodatamine",
"PG_DB_TMP": "geodatamine_next",
"PG_DB_OLD": "geodatamine_prev",
"CACHE_MEM": 10000,
"FLAT_NODES": false
......
......@@ -38,6 +38,7 @@ if ! [ -f "${last_ts_file}" -a -f "${last_ts_file}" -a `if [ -f "${last_ts_file}
# Success -> Write TS file
date +%s > "${last_ts_file}"
rm -f "${pbf_light_file}"
echo "Success"
else
# Failure -> Stop script
echo "Can't download file"
......@@ -62,6 +63,7 @@ if ! [ -f "${pbf_light_file}" ]; then
-f pbf,add_metadata=version+timestamp \
--no-progress \
--overwrite -o ${pbf_light_file}
echo "Success"
else
echo "Reuse existing light PBF file"
fi
......@@ -109,7 +111,8 @@ SELECT
array_to_json(array_agg(json_build_object(
'id', b.osm_id,
'name', b.name,
'ref', b.ref
'ref', b.ref,
'type', b.type
))) AS data
FROM boundary b;
"
......@@ -127,8 +130,10 @@ psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -t -A -c
# Rename databases to make next goes live
echo "======= Switch next with current DB ======="
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname='${PG_DB_CLEAN}'"
set +e
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "ALTER DATABASE ${PG_DB_CLEAN} RENAME TO ${PG_DB_OLD}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_OLD}" -c "ALTER DATABASE ${PG_DB_TMP} RENAME TO ${PG_DB_CLEAN}"
set -e
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "ALTER DATABASE ${PG_DB_TMP} RENAME TO ${PG_DB_CLEAN}"
# Unlock API
rm "${lock_file}"
......
--METADATA={ "name:fr": "Défibrillateur", "name:en": "Automated external defibrillator" }
--METADATA={ "name:fr": "Défibrillateur", "name:en": "Automated external defibrillator", "theme:fr": "Santé", "keywords:fr": [ "défibrilateur", "aed", "dae" ], "description:fr": "Défibrillateurs extraits d'OpenStreetMap (emergency=defibrillator)" }
--GEOMETRY=point
SELECT
......
--METADATA={ "name:fr": "Jardins familliaux", "name:en": "Allotments" }
--METADATA={ "name:fr": "Jardins familliaux", "name:en": "Allotments", "theme:fr": "Agriculture", "keywords:fr": [ "jardins familliaux" ], "description:fr": "Jardins familliaux issus d'OpenStreetMap (landuse=allotments)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Banque et DAB", "name:en": "Bank and ATM" }
--METADATA={ "name:fr": "Banque et DAB", "name:en": "Bank and ATM", "theme:fr": "Économie", "keywords:fr": [ "banque", "dab", "distributeur de billets" ], "description:fr": "Banques et distributeurs de billets issus d'OpenStreetMap (amenity=bank|atm)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Parking à vélos", "name:en": "Bicycle parking" }
--METADATA={ "name:fr": "Parking à vélos", "name:en": "Bicycle parking", "theme:fr": "Transports", "keywords:fr": [ "vélo", "parking à vélos" ], "description:fr": "Parkings à vélos issus d'OpenStreetMap (amenity=bicycle_parking)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Cimetière", "name:en": "Cemetery" }
--METADATA={ "name:fr": "Cimetière", "name:en": "Cemetery", "theme:fr": "Infrastructures", "keywords:fr": [ "cimetière" ], "description:fr": "Cimetières issus d'OpenStreetMap (landuse=cemetery)" }
--GEOMETRY=polygon
SELECT
......
--METADATA={ "name:fr": "Piste/bande cyclable", "name:en": "Cycleway" }
--METADATA={ "name:fr": "Piste/bande cyclable", "name:en": "Cycleway", "theme:fr": "Transports", "keywords:fr": [ "piste cyclable", "bande cyclable" ], "description:fr": "Pistes et bandes cyclables issues d'OpenStreetMap (cycleway=* ou cycleway:right=* ou cycleway:left=* ou cycleway:both=* ou highway=cycleway)" }
--GEOMETRY=line
SELECT
......
--METADATA={ "name:fr": "Lieu d'enseignement", "name:en": "Education" }
--METADATA={ "name:fr": "Lieu d'enseignement", "name:en": "Education", "theme:fr": "Éducation et communication", "keywords:fr": [ "école", "université", "collège", "lycée" ], "description:fr": "Lieu d'enseignement issus d'OpenStreetMap (amenity=school|kindergarten|college|university)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Station-service", "name:en": "Fuel station" }
--METADATA={ "name:fr": "Station-service", "name:en": "Fuel station", "theme:fr": "Transports", "keywords:fr": [ "station-service", "station essence", "carburant" ], "description:fr": "Stations-service issues d'OpenStreetMap (amenity=fuel)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Santé (hôpital, clinique, pharmacie)", "name:en": "Healthcare equipment" }
--METADATA={ "name:fr": "Santé (hôpital, clinique, pharmacie)", "name:en": "Healthcare equipment", "theme:fr": "Santé", "keywords:fr": [ "hôpital", "clinique", "médecin", "pharmacie" ], "description:fr": "Établissements liés à la santé issus d'OpenStreetMap (amenity=hospital|clinic|dentist|doctors|nursing_home|pharmacy)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Patrimoine historique", "name:en": "Historic features" }
--METADATA={ "name:fr": "Patrimoine historique", "name:en": "Historic features", "theme:fr": "Culture", "keywords:fr": [ "monument historique", "petit patrimoine", "église", "fontaine", "lavoir" ], "description:fr": "Patrimoine historique issu d'OpenStreetMap (historic=* ou amenity=fountain|place_of_worship|lavoir)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Hébergement touristique", "name:en": "Hosting for tourists" }
--METADATA={ "name:fr": "Hébergement touristique", "name:en": "Hosting for tourists", "theme:fr": "Tourisme", "keywords:fr": [ "hôtel", "chalet", "camping", "camping-car", "chambre d'hôtes" ], "description:fr": "Hébergements touristiques issus d'OpenStreetMap (tourism=hotel|chalet|guest_house|apartment|hostel|motel|wilderness_hut|alpine_hut|camp_site|caravan_site)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Point d'accès Internet", "name:en": "Internet access point" }
--METADATA={ "name:fr": "Point d'accès Internet", "name:en": "Internet access point", "theme:fr": "Infrastructures", "keywords:fr": [ "point d'accès internet", "wifi", "internet" ], "description:fr": "Établissements disposant d'un accès Internet (internet_access=*)" }
--GEOMETRY=point,polygon
SELECT
......
--METADATA={ "name:fr": "Bibliothèque", "name:en": "Library" }
--METADATA={ "name:fr": "Bibliothèque", "name:en": "Library", "theme:fr": "Culture", "keywords:fr": [ "bibliothèque" ], "description:fr": "Bibliothèques et micro-bibliothèques issues d'OpenStreetMap (amenity=library|public_bookcase)" }
--GEOMETRY=point,polygon
SELECT
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment