Commit c0402197 authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Data dump for every theme

parent 440989a0
......@@ -2,3 +2,4 @@
/api/node_modules
/api/*.log
/api/build
/api/dump
......@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
## Next version (to be released)
### Added
- Raw exports of whole database in CSV and GeoJSON in /dump directory
## 0.2.1 - 2020-05-06
......
......@@ -106,6 +106,7 @@ Les paramètres généraux de configuration du service sont à renseigner dans l
* `CACHE_MEM` : mémoire à allouer lors de l'import de données, en MB, idéalement 75% de la mémoire vive disponible (exemple: `10000`)
* `FLAT_NODES` : activer l'option `--flat-nodes` de l'outil osm2pgsql, utile pour les imports de grosses régions ou de pays (`true` pour activer, `false` pour désactiver)
* `API_BASE_URL` : URL à utiliser dans les fichiers de métadonnées pour accéder à l'API (utile si API derrière un proxy), si absent l'adresse directe de l'API est utilisée (exemple: `https://geodatamine.fr`)
* `BOUNDARY` : métadonnées de la zone chargée en base (exemple: `{ "name": "Ille-et-Vilaine", "ref": "35", "id": "35"}`)
### Thèmes
......@@ -127,6 +128,7 @@ Les propriétés à renseigner sont :
* `keywords:fr` : liste de mots clés, sous la forme d'un tableau
* `description:fr` : un bref résumé du jeu de données, ainsi que des clés OSM utilisées pour l'extraction
* `skipGeomCSV` : mettre à `true` pour ne pas ajouter la géométrie lors d'un export CSV (si des colonnes lat/long sont déjà prévues)
* `skipDump` : mettre à `true` si le thème doit être ignoré lors des exports complets de la base
### 2e ligne du fichier
......
......@@ -545,6 +545,11 @@
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
"dev": true
},
"batch": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz",
"integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY="
},
"binary-extensions": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
......@@ -4069,6 +4074,38 @@
}
}
},
"serve-index": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz",
"integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=",
"requires": {
"accepts": "1.3.7",
"batch": "0.6.1",
"debug": "2.6.9",
"escape-html": "1.0.3",
"http-errors": "1.6.3",
"mime-types": "2.1.25",
"parseurl": "1.3.3"
},
"dependencies": {
"http-errors": {
"version": "1.6.3",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
"integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=",
"requires": {
"depd": "1.1.2",
"inherits": "2.0.3",
"setprototypeof": "1.1.0",
"statuses": "1.5.0"
}
},
"setprototypeof": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
"integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
}
}
},
"serve-static": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
......
......@@ -12,7 +12,8 @@
"test": "echo \"Error: no test specified\" && exit 1",
"doc": "apidoc -i src/ -o doc/",
"clean": "npm run clean:log && rm -rf ./doc && rm -rf ./build",
"clean:log": "rm -f api.log"
"clean:log": "rm -f api.log",
"dump": "node src/dump.js"
},
"author": "Adrien Pavie",
"license": "AGPL-3.0",
......@@ -23,7 +24,8 @@
"express": "^4.17.1",
"jquery-slim": "^3.0.0",
"pg": "^7.14.0",
"popper.js": "^1.16.0"
"popper.js": "^1.16.0",
"serve-index": "^1.9.1"
},
"devDependencies": {
"apidoc": "^0.18.0",
......
......@@ -66,7 +66,7 @@ function whenAvailable(t) {
/**
* Retrieve list of available boundaries
*/
function getBoundaries() {
exports.getBoundaries = () => {
return whenAvailable()
.then(() => {
return new Promise((resolve, reject) => {
......@@ -76,12 +76,12 @@ function getBoundaries() {
});
});
});
}
};
/*
* Retrieve info of a specific boundary
*/
function getBoundary(id) {
exports.getBoundary = (id) => {
return whenAvailable()
.then(() => {
return pool.query(`SELECT osm_id AS id, name, replace(simple_name, ' ', '_') AS simple_name, ref, type FROM boundary b WHERE osm_id = $1`, [ id ])
......@@ -94,9 +94,9 @@ function getBoundary(id) {
}
});
});
}
};
function findBoundariesByName(name) {
exports.findBoundariesByName = (name) => {
if(pool === null) {
createPool();
}
......@@ -113,12 +113,13 @@ function findBoundariesByName(name) {
return pool.query(`SELECT osm_id AS id, name, ref, type FROM boundary b ORDER BY lower(unaccent($1)) <<<-> b.simple_name LIMIT 10`, [ name ])
.then(res => res.rows);
}
}
};
/**
* Generate metadata text
*/
function getMetadataText(theme, boundary, format, url) {
boundary = boundary || CONFIG.BOUNDARY;
let text = "COLL_NOM;COLL_SIRET;ID;TITRE;DESCRIPTION;THEME;PRODUCTEUR_NOM;PRODUCTEUR_SIRET;COUV_SPAT_MAILLE;COUV_SPAT_NOM;COUV_TEMP_DEBUT;COUV_TEMP_FIN;DATE_PUBL;FREQ_MAJ;DATE_MAJ;MOTS_CLES;LICENCE;NOMBRE_RESSOURCES;FORMAT_RESSOURCES;URL";
const params = [
......@@ -151,7 +152,7 @@ function getMetadataText(theme, boundary, format, url) {
/**
* Extract thematic data
* @param {Object} theme The theme metadata
* @param {Object} boundary The boundary metadata
* @param {Object} boundary The boundary metadata, or null for whole database export
* @param {Object} [options] Options
* @param {string} [options.format] Output file format
* @param {number} [options.radius] Radius around boundary center for data filtering
......@@ -160,26 +161,30 @@ function getMetadataText(theme, boundary, format, url) {
* @param {string} [options.url] Original URL used for getting this dataset
* @return {Promise} Resolves on output file
*/
function getThemeData(theme, boundary, options) {
exports.getThemeData = (theme, boundary, options) => {
return whenAvailable()
.then(() => {
const start = Date.now();
const format = options.format || "csv";
const metadata = options.metadata || false;
const resultViewName = theme.id + boundary.id.toString() + Date.now();
const resultViewName = `${theme.id}${boundary ? boundary.id.toString() : "dump"}${Date.now()}`;
const baseFilePath = CONFIG.WORK_DIR + "/" + resultViewName;
const boundaryId = "'" + boundary.id.toString().replace(/'/g, "") + "'";
const boundaryId = boundary ? "'" + boundary.id.toString().replace(/'/g, "") + "'" : null;
// Create condition for filtering
const geomConds = [ { sort: 0, cond: `b.osm_id = ${boundaryId}` } ];
const geomConds = [];
const dist = options.radius > 0 ? options.radius * 1000 : null;
if(dist) {
geomConds.push({ sort: 2, cond: `ST_DWithin(t.centroid, b.center, ${dist})` });
}
else {
geomConds.push({ sort: 1, cond: "b.way && t.centroid" });
geomConds.push({ sort: 2, cond: "ST_Intersects(t.centroid, b.way)" });
if(boundaryId) {
geomConds.push({ sort: 0, cond: `b.osm_id = ${boundaryId}` });
const dist = options.radius > 0 ? options.radius * 1000 : null;
if(dist) {
geomConds.push({ sort: 2, cond: `ST_DWithin(t.centroid, b.center, ${dist})` });
}
else {
geomConds.push({ sort: 1, cond: "b.way && t.centroid" });
geomConds.push({ sort: 2, cond: "ST_Intersects(t.centroid, b.way)" });
}
}
const hasAdm8 = theme.sql.includes("<ADM8");
......@@ -205,7 +210,7 @@ function getThemeData(theme, boundary, options) {
// Create SQL request
let request = theme.geomtypes.map(g => (
theme.sql
.replace(/<TABLE>/g, `boundary b, planet_osm_${g} t${hasAdm8 ? ", boundary b2" : ""}`)
.replace(/<TABLE>/g, `${boundaryId ? 'boundary b, ' : ''}planet_osm_${g} t${hasAdm8 ? ", boundary b2" : ""}`)
.replace(/<GEOM>/g, g === "point" ? geom.replace(/t\.centroid/g, "t.way") : geom)
.replace(/<GEOMEMBED>/g, "t.way")
.replace(/<ADM8NAME>/g, "b2.name")
......@@ -296,15 +301,8 @@ function getThemeData(theme, boundary, options) {
}
}
console.log(`Request ${theme.id} on ${boundary.name} (${new Date().toISOString().split("T")[0]}): ${Math.round((Date.now()-start)/1000)}s`);
console.log(`Request ${theme.id} on ${boundary ? boundary.name : "dump"} (${new Date().toISOString().split("T")[0]}): ${Math.round((Date.now()-start)/1000)}s`);
return outFile;
});
});
}
module.exports = {
getBoundaries: getBoundaries,
getBoundary: getBoundary,
getThemeData: getThemeData,
findBoundariesByName: findBoundariesByName
};
/*
* Code for database-wide theme dumps
*/
const CONFIG = require('../../config.json');
const db = require('./db');
const themes = require('./themes');
const fs = require('fs');
// Create dump directory
const DUMP_DIR = __dirname + '/../dump';
if(!fs.existsSync(DUMP_DIR)) {
fs.mkdirSync(DUMP_DIR, { recursive: true });
}
// Dump a single theme + format
const dumpTheme = (theme, format) => {
return db.getThemeData(theme, null, {
format: format,
url: (CONFIG.API_BASE_URL || "http://localhost:3000") + "/dump/" + theme.id + ".geojson",
metadata: true,
aspoint: format === "csv"
})
.then(outFile => {
// Copy + remove to support cross-device mv
fs.copyFileSync(outFile, `${DUMP_DIR}/${theme.id}_${format}.zip`);
fs.unlinkSync(outFile);
return true;
});
};
// Create dump for each theme
const nextTheme = () => {
const theme = themes.full.shift();
if(!theme) {
console.log("Dumping all themes done");
return;
}
if(theme.skipDump) {
return nextTheme();
}
dumpTheme(theme, "geojson")
.then(() => {
dumpTheme(theme, "csv")
.then(() => nextTheme());
});
};
console.log("Dumping all themes");
nextTheme();
......@@ -6,6 +6,7 @@ const CONFIG = require('../../config.json');
const express = require('express');
const cors = require('cors');
const compression = require("compression");
const serveIndex = require('serve-index');
const fs = require('fs');
const db = require('./db');
const themes = require('./themes');
......@@ -50,6 +51,14 @@ app.use((req, res, next) => {
*/
app.use('/', express.static(__dirname+'/../build'));
/**
* @api {get} /dump Raw data downloads
* @apiDescription Get files for each theme as a raw data download
* @apiName GetDump
* @apiGroup Default
*/
app.use('/dump', express.static(__dirname+'/../dump'), serveIndex(__dirname+'/../dump', {'icons': true, 'view': 'details'}));
/**
* @api {get} /doc API documentation
* @apiDescription Get the user documentation of the API (this page)
......@@ -108,7 +117,7 @@ app.get("/data/:themeId/:boundaryId", (req, res) => {
// Save used URL (for metadata)
const myPort = req.app.settings.port || port;
const baseUrl = CONFIG.API_BASE_URL ? CONFIG.API_BASE_URL : req.protocol + '://' + req.hostname + (myPort == 80 || myPort == 443 ? '' : ':' + myPort);
const url = baseUrl + req.originalUrl;
const url = baseUrl + req.originalUrl;
// Retrieve data
const boundaryId = req.params.boundaryId;
......
......@@ -24,6 +24,9 @@
<li class="nav-item">
<a class="nav-link" href="/doc">API</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/dump">Exports bruts</a>
</li>
<li class="nav-item">
<a class="nav-link" href="https://framagit.org/PanierAvide/GeoDataMine">Code source</a>
</li>
......
{
"WORK_DIR": "/tmp/geodatamine",
"WORK_DIR": "/home/adrien/tmp/geodatamine",
"PBF_URL": "http://download.openstreetmap.fr/extracts/europe/france/bretagne/ille_et_vilaine-latest.osm.pbf",
"PARALLEL_JOBS": "6",
"BOUNDARY": { "name": "Ille-et-Vilaine", "ref": "35", "id": "35"},
"PG_HOST": "localhost",
"PG_USER": "postgres",
......
......@@ -151,6 +151,11 @@ rm "${lock_file}"
# Cleanup
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "DROP DATABASE IF EXISTS ${PG_DB_OLD}"
# Export dumps
cd ../api/
npm run dump
cd ../db/
end=`date +%s`
date
......
--METADATA={ "name:fr": "Base adresses (schéma Étalab Base adresse locale 1.1.4)", "name:en": "Addresses", "theme:fr": "Infrastructures", "keywords:fr": [ "adresses", "bal", "adresse postale", "numéro de rue" ], "description:fr": "Adresses issues d'OpenStreetMap (addr:housenumber=*)", "skipGeomCSV": true }
--METADATA={ "name:fr": "Base adresses (schéma Étalab Base adresse locale 1.1.4)", "name:en": "Addresses", "theme:fr": "Infrastructures", "keywords:fr": [ "adresses", "bal", "adresse postale", "numéro de rue" ], "description:fr": "Adresses issues d'OpenStreetMap (addr:housenumber=*)", "skipGeomCSV": true, "skipDump": true }
--GEOMETRY=point,polygon
SELECT
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment