Commit f215808b authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Frontend for data extract

parent e8bfb580
/api/doc
/api/node_modules
/api/*.log
/api/build
This diff is collapsed.
......@@ -4,10 +4,13 @@
"description": "API to extract ready-to-use OpenStreetMap data on various themes",
"main": "src/index.js",
"scripts": {
"start": "npm run clean:log && npm run doc && node src/index.js",
"start": "npm run clean:log && npm run doc && npm run build && node src/index.js",
"build": "npm run build:assets && npm run build:js",
"build:assets": "mkdir -p build && cp src/website/*.html src/website/*.css build/",
"build:js": "simplifyify src/website/index.js --outfile build/index.min.js --minify",
"test": "echo \"Error: no test specified\" && exit 1",
"doc": "apidoc -i src/ -o doc/",
"clean": "npm run clean:log && rm -rf ./doc",
"clean": "npm run clean:log && rm -rf ./doc && rm -rf ./build",
"clean:log": "rm -f api.log"
},
"author": "Adrien Pavie",
......@@ -16,10 +19,13 @@
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
"pg": "^7.14.0"
"pg": "^7.14.0",
"request": "^2.88.0",
"request-promise-native": "^1.0.8"
},
"devDependencies": {
"apidoc": "^0.18.0"
"apidoc": "^0.18.0",
"simplifyify": "^7.0.6"
},
"apidoc": {
"title": "OpenStreetMap for OpenDataFrance"
......
......@@ -25,7 +25,7 @@ const pool = new Pool({
function whenAvailable(t) {
t = t || 0;
if(t === 10) { return Promise.reject(); }
if(t === 10) { return Promise.reject(new Error("Database is not available")); }
return new Promise((resolve, reject) => {
if(fs.existsSync(CONFIG.WORK_DIR+"/db.lock")) {
......@@ -73,6 +73,7 @@ function getBoundaries() {
function getThemeData(theme, boundary, options) {
return whenAvailable()
.then(() => {
const start = Date.now();
const resultViewName = "tmp_"+Date.now();
const format = options.format || "csv";
......@@ -129,6 +130,7 @@ ${theme.sql
throw new Error("Can't extract data from DB: "+result.stderr);
}
console.log(`Execution time (${theme.id} on ${boundary}): ${Date.now()-start}ms`);
return outFile;
});
});
......
......@@ -41,6 +41,14 @@ app.use((req, res, next) => {
* List of routes
*/
/**
* @api {get} / Simple website
* @apiDescription Get the website for extracting data from API
* @apiName GetSite
* @apiGroup Default
*/
app.use('/', express.static(__dirname+'/../build'));
/**
* @api {get} /doc API documentation
* @apiDescription Get the user documentation of the API (this page)
......
......@@ -35,7 +35,9 @@ fs.readdirSync(folder).forEach((file) => {
// Append data
themesLight.push(Object.assign({ id: themeId }, metadata));
let completeSQL = "WITH data AS ("+geom.map(g => "SELECT p.*, p.way AS geom FROM planet_osm_"+g+" p, boundary b WHERE b.osm_id = <BID> AND p.way && b.way AND ST_Intersects(p.way, b.way)").join(" UNION ")+") ";
let completeSQL = "WITH data AS ("+geom.map(g => (
"SELECT p.*, p.way AS geom FROM planet_osm_"+g+" p, boundary b WHERE b.osm_id = <BID> AND p.way && b.way AND ST_Intersects(p.way, b.way)"
)).join(" UNION ")+") ";
completeSQL += sql.replace(/<TABLE>/g, "data");
themes.push(Object.assign({ id: themeId, sql: completeSQL.trim(), geomtypes: geom }, metadata));
}
......
<!DOCTYPE html5>
<html>
<head>
<title>OpenStreetMap via OpenDataFrance</title>
<meta charset="utf-8" />
<meta name="description" content="Extraction simplifiée de données OpenStreetMap pour votre territoire" />
<meta name="keywords" content="openstreetmap, osm, data, données, extraction, extract" />
<meta name="author" content="Adrien PAVIE (PanierAvide)" />
<meta name="copyright" content="&copy; Adrien PAVIE 2019, under AGPL v3" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<link rel="stylesheet" type="text/css" href="style.css" />
</head>
<body>
<h1>Extracteur OpenStreetMap thématique</h1>
<h2>Extraction de données</h2>
<div id="form-load">
Chargement en cours...
</div>
<div id="form-extract" class="hidden">
<p>Données actualisées quotidiennement</p>
<h3>Thématique</h3>
<select id="theme"></select>
<h3>Territoire</h3>
<select id="boundary-type"></select>
<input list="boundaries" id="boundary-search" placeholder="Saisissez le nom du territoire..." />
<datalist id="boundaries"></datalist>
<h3>Options</h3>
<p>Format</p>
<select id="format">
<option value="csv" selected>Tableur CSV</option>
<option value="geojson">GeoJSON</option>
<option value="shapefile">ESRI Shapefile</option>
</select>
<input type="checkbox" id="aspoint" /><label for="aspoint">Tout simplifier sous forme de points</label>
<p>Récupérer les données</p>
<select id="radius">
<option value="" selected>Dans l'emprise de la commune</option>
<option value="1">1 km autour du centre géométrique</option>
<option value="5">5 km autour du centre géométrique</option>
<option value="10">10 km autour du centre géométrique</option>
<option value="20">20 km autour du centre géométrique</option>
</select>
<h3>Télécharger</h3>
<p id="permalink-bloc" class="hidden"><a id="launch" href="" style="font-size: 20px">Télécharger les données</a><br />Lien permanent : <a href="" id="permalink"></a></p>
<p id="permalink-missing">Merci de compléter les champs ci-dessus</p>
</div>
<h2>Mentions légales</h2>
<p>Données sous licence ODbL <a href="https://www.openstreetmap.org/copyright">&copy; contributeurs OpenStreetMap</a><br />Site web réalisé par <a href="https://pavie.info">Adrien Pavie</a> pour OpenDataFrance</p>
<p>Aussi disponible sous forme d'API, voir <a href="/doc">la documentation</a></p>
<script src="index.min.js"></script>
</body>
</html>
const request = require("request-promise-native");
const data = {};
const url = window.location.href;
let selectedBoundary = null;
setDownloadable(false);
/*
* Data download from API
*/
request(url+"themes")
.then(themes => data.themes = JSON.parse(themes).themes)
.catch(e => {
data.themes = "error";
console.error(e);
});
request(url+"boundaries")
.then(boundaries => data.boundaries = JSON.parse(boundaries).categories)
.catch(e => {
data.boundaries = "error";
console.error(e);
});
/*
* Populate form using received data
*/
function createForm() {
// Prepare form
const formExtract = document.getElementById("form-extract");
// Show theme list
const selectTheme = document.getElementById("theme");
selectTheme.innerHTML = "";
data.themes.forEach(theme => {
const tdom = document.createElement("option");
tdom.value = theme.id;
tdom.innerHTML = theme["name:fr"];
selectTheme.appendChild(tdom);
});
// Show boundary search
const selectBoundaryType = document.getElementById("boundary-type");
data.boundaries.forEach(category => {
const cdom = document.createElement("option");
cdom.value = category.id;
cdom.innerHTML = category["name:fr"];
if(category.id === "admin_8") {
cdom.selected = "true";
initSearchBar(category.id);
}
selectBoundaryType.appendChild(cdom);
});
selectBoundaryType.addEventListener("change", () => {
initSearchBar(selectBoundaryType.value);
});
const searchTextField = document.getElementById("boundary-search");
searchTextField.addEventListener("input", e => {
// Find associated option
const result = [...document.getElementById("boundaries").children].find(c => c.innerHTML.toLowerCase() === searchTextField.value.toLowerCase());
if(result) {
selectedBoundary = result["data-value"];
setDownloadable(true);
}
else {
selectedBoundary = null;
setDownloadable(false);
}
});
// Hide loader
const formLoad = document.getElementById("form-load");
formLoad.classList.add("hidden");
formExtract.classList.remove("hidden");
// Follow options updates
document.getElementById("format").addEventListener("change", () => setDownloadable(true));
document.getElementById("aspoint").addEventListener("change", () => setDownloadable(true));
document.getElementById("radius").addEventListener("change", () => setDownloadable(true));
}
/*
* Associate search engine to input fields
*/
function initSearchBar(categoryId) {
selectedBoundary = null;
setDownloadable(false);
const category = data.boundaries.find(b => b.id === categoryId);
if(category) {
const searchTextField = document.getElementById("boundary-search");
searchTextField.value = "";
const datalist = document.getElementById("boundaries");
datalist.innerHTML = "";
category.list.forEach(v => {
const bdom = document.createElement("option");
bdom["data-value"] = v.id;
bdom.innerHTML = v.name;
datalist.appendChild(bdom);
});
}
}
/*
* Update nodes to allow download start or not
*/
function setDownloadable(shouldI) {
const btnDl = document.getElementById("launch");
const permalink = document.getElementById("permalink");
const permabloc = document.getElementById("permalink-bloc");
const permamiss = document.getElementById("permalink-missing");
const theme = document.getElementById("theme").value;
if(shouldI && selectedBoundary && theme && theme.length > 0) {
const opts = {
format: document.getElementById("format").value,
aspoint: document.getElementById("aspoint").checked ? "true" : "",
radius: document.getElementById("radius").value
};
let dlUrl = url+"data/"+theme+"/"+selectedBoundary+"?";
dlUrl += Object.entries(opts).filter(o => o[1] && o[1] !== null && o[1].length > 0).map(o => o.join("=")).join("&");
permabloc.classList.remove("hidden");
permamiss.classList.add("hidden");
launch.disabled = false;
permalink.href = dlUrl;
permalink.innerHTML = dlUrl;
btnDl.href = dlUrl;
}
else {
permabloc.classList.add("hidden");
permamiss.classList.remove("hidden");
launch.disabled = true;
permalink.href = "";
permalink.innerHTML = "";
btnDl.dlUrl = "";
}
}
/*
* Wait for data retrieval
*/
const itv = setInterval(() => {
if(data && data.themes && data.boundaries) {
clearInterval(itv);
if(data.themes === "error" || data.boundaries === "error") {
alert("Une erreur s'est produite lors du chargement des métadonnées, merci de réessayer");
}
else {
createForm();
}
}
}, 100);
.hidden {
display: none;
}
{
"WORK_DIR": "/home/adrien/tmp/osm4odf",
"PBF_URL": "http://download.geofabrik.de/europe/france/reunion-latest.osm.pbf",
"PBF_URL": "http://download.geofabrik.de/europe/france/bretagne-latest.osm.pbf",
"PARALLEL_JOBS": "6",
"PG_HOST": "localhost",
......
......@@ -60,7 +60,7 @@ osm2pgsql --create \
# Add boundaries for API requests
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE TABLE boundary AS SELECT osm_id, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, name, way FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way);"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE TABLE boundary AS SELECT osm_id, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, name, ST_SimplifyPreserveTopology(way, 10) AS way FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way);"
# JSON boundary version for API
bounds_json="
......@@ -100,7 +100,7 @@ FROM (
echo "Stop API queries to update database"
lock_file="${WORK_DIR}/db.lock"
echo "lock" > "${lock_file}"
sleep 5
sleep 30
# Change bounds
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -t -A -c "${bounds_json}" > "${WORK_DIR}/bounds.json"
......@@ -111,6 +111,7 @@ set +e
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='${PG_DB_CLEAN}'; ALTER DATABASE ${PG_DB_CLEAN} RENAME TO ${PG_DB_OLD}"
set -e
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "ALTER DATABASE ${PG_DB_TMP} RENAME TO ${PG_DB_CLEAN}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_CLEAN}" -c "VACUUM FULL"
# Unlock API
rm "${lock_file}"
......
{
"lockfileVersion": 1
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment