Commit 0aed2323 authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Move boundary search to API-side

parent a142a022
......@@ -13,7 +13,7 @@ L'outil est disponible à cette adresse : [geodatamine.fr](https://geodatamine.f
Les dépendances sont les suivantes :
* NodeJS >= 9
* PostgreSQL >= 9.6 et ses extensions PostGIS et HStore
* PostgreSQL >= 9.6 et ses extensions PostGIS, HStore, PG_TRGM, Unaccent
* wget
* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql)
* [jq](https://stedolan.github.io/jq/)
......
{
"name": "osm4odf-api",
"name": "geodatamine-api",
"version": "0.0.0",
"lockfileVersion": 1,
"requires": true,
......@@ -2462,11 +2462,6 @@
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
"dev": true
},
"fuse.js": {
"version": "3.4.6",
"resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-3.4.6.tgz",
"integrity": "sha512-H6aJY4UpLFwxj1+5nAvufom5b2BT2v45P1MkPvdGIK8fWjQx/7o6tTT1+ALV0yawQvbmvCF0ufl2et8eJ7v7Cg=="
},
"get-assigned-identifiers": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/get-assigned-identifiers/-/get-assigned-identifiers-1.2.0.tgz",
......
......@@ -21,7 +21,6 @@
"compression": "^1.7.4",
"cors": "^2.8.5",
"express": "^4.17.1",
"fuse.js": "^3.4.6",
"jquery-slim": "^3.0.0",
"pg": "^7.14.0",
"popper.js": "^1.16.0"
......
......@@ -12,12 +12,16 @@ const GEOM_TO_COND = { "point": "ST_GeometryType(geom) = 'ST_Point'", "line": "S
let viewId = 0;
// Create pool of connections
const pool = new Pool({
user: CONFIG.PG_USER,
host: CONFIG.PG_HOST,
database: CONFIG.PG_DB_CLEAN,
port: CONFIG.PG_PORT
});
let pool = null;
function createPool() {
pool = new Pool({
user: CONFIG.PG_USER,
host: CONFIG.PG_HOST,
database: CONFIG.PG_DB_CLEAN,
port: CONFIG.PG_PORT
});
}
/**
......@@ -26,7 +30,10 @@ const pool = new Pool({
function whenAvailable(t) {
t = t || 0;
if(t === 10) { return Promise.reject(new Error("Database is not available")); }
if(t === 10) {
pool = null;
return Promise.reject(new Error("Database is not available"));
}
return new Promise((resolve, reject) => {
if(fs.existsSync(CONFIG.WORK_DIR+"/db.lock")) {
......@@ -35,6 +42,10 @@ function whenAvailable(t) {
}, 1000);
}
else {
if(pool === null) {
createPool();
}
pool.query("SELECT 1+1")
.then(() => resolve())
.catch(() => {
......@@ -75,6 +86,25 @@ function getBoundary(id) {
});
}
function findBoundariesByName(name) {
if(pool === null) {
createPool();
}
name = name.trim();
// Search by ref
if(name.match(/^[0-9]+$/)) {
return pool.query(`SELECT osm_id AS id, name, ref, type FROM boundary b WHERE ref LIKE '${name}%' ORDER BY ref LIMIT 10`)
.then(res => res.rows);
}
// Search by name
else {
return pool.query(`SELECT osm_id AS id, name, ref, type FROM boundary b ORDER BY lower(unaccent($1)) <<<-> b.simple_name LIMIT 10`, [ name ])
.then(res => res.rows);
}
}
/**
* Extract thematic data
* @param {Object} theme The theme metadata
......@@ -160,5 +190,6 @@ function getThemeData(theme, boundary, options) {
module.exports = {
getBoundaries: getBoundaries,
getBoundary: getBoundary,
getThemeData: getThemeData
getThemeData: getThemeData,
findBoundariesByName: findBoundariesByName
};
......@@ -176,9 +176,7 @@ app.get("/themes", (req, res) => {
* {
* "id": "1234",
* "name": "Rennes",
* "type": "admin_8",
* "type:fr": "Commune",
* "type:en": "City"
* "type": "admin_8"
* },
* ...
* ]
......@@ -192,6 +190,38 @@ app.get("/boundaries", (req, res) => {
});
});
/**
* @api {get} /boundaries/search Search administrative boundaries
* @apiDescription Get administrative boundaries matching search pattern
* @apiName GetBoundariesSearch
* @apiGroup Default
* @apiParam {String} text Search pattern (can be name, ref)
*
* @apiSuccessExample {json} Success-Response:
* HTTP/1.1 200 OK
* [
* {
* "id": "1234",
* "name": "Rennes",
* "type": "admin_8"
* },
* ...
* ]
*/
app.get("/boundaries/search", (req, res) => {
const text = req.query.text.trim() || "";
if(text.length === 0) {
return res.send([]);
}
db.findBoundariesByName(text)
.then(b => res.send(b))
.catch(e => {
console.log(e);
res.status(500).send("Can't search in boundaries: " + e.message);
});
});
// 404
app.use((req, res) => {
res.status(404).send(req.originalUrl + ' not found')
......
const Fuse = require("fuse.js");
const data = {};
const url = window.location.origin + window.location.pathname.substring(0, window.location.pathname.lastIndexOf('/')) + "/";
const BOUNDARY_TYPES = {"admin_6": "Département", "admin_7": "Pays (loi Voynet)", "admin_8": "Commune", "political": "Canton", "local_authority": "EPCI"};
let selectedBoundary = null;
let timerSearchInput = null;
let hasPrevTextSearch = false;
setDownloadable(false);
......@@ -30,13 +31,6 @@ request(url+"themes")
console.error(e);
});
request(url+"boundaries")
.then(boundaries => data.boundaries = JSON.parse(boundaries))
.catch(e => {
data.boundaries = "error";
console.error(e);
});
/*
* Populate form using received data
......@@ -58,51 +52,61 @@ function createForm() {
});
// Set-up boundary search
const fuse = new Fuse(data.boundaries, {
shouldSort: true,
threshold: 0.4,
location: 0,
distance: 100,
maxPatternLength: 32,
minMatchCharLength: 1,
keys: [
"name",
"ref",
"type:fr"
]
});
const searchTextField = document.getElementById("boundary-search");
const searchSuggestions = document.getElementById("boundary-suggestions");
searchTextField.value = "";
searchTextField.addEventListener("input", e => {
if(timerSearchInput) {
clearTimeout(timerSearchInput);
}
$('#boundary-search').dropdown('show');
selectedBoundary = null;
setDownloadable(false);
const text = e.target.value;
if(text && text.trim().length === 0) {
if(!text || text.trim().length === 0) {
searchSuggestions.innerHTML = '<a class="dropdown-item disabled">Tapez un nom pour lancer la recherche</a>';
hasPrevTextSearch = false;
}
else {
searchSuggestions.innerHTML = '';
const results = fuse.search(text);
if(results.length > 0) {
results.slice(0, 10).forEach(r => {
const entry = document.createElement("a");
entry.classList.add("dropdown-item");
entry.innerHTML = r.name + ' <small>(' + r["type:fr"] + ' - ' + r.ref + ')</small>';
searchSuggestions.appendChild(entry);
entry.addEventListener("click", () => {
searchTextField.value = r.name + ' (' + r["type:fr"] + ' - ' + r.ref + ')';
selectedBoundary = r.id;
setDownloadable(true);
});
});
}
else {
searchSuggestions.innerHTML = '<a class="dropdown-item disabled">Aucun résultat</a>';
const waitText = '<a class="dropdown-item disabled"><span class="spinner-border text-primary" role="status"></span></a>';
if(!hasPrevTextSearch && searchSuggestions.innerHTML !== waitText) {
searchSuggestions.innerHTML = waitText;
}
timerSearchInput = setTimeout(() => {
hasPrevTextSearch = true;
request(url + "boundaries/search?text="+encodeURIComponent(text))
.then(txt => JSON.parse(txt))
.then(results => {
searchSuggestions.innerHTML = '';
if(results.length > 0) {
results.forEach(r => {
const type = BOUNDARY_TYPES[r.type];
const ref = r.ref || "ID inconnu";
const entry = document.createElement("a");
entry.classList.add("dropdown-item");
entry.innerHTML = r.name + ' <small>(' + type + ' - ' + ref + ')</small>';
searchSuggestions.appendChild(entry);
entry.addEventListener("click", () => {
searchTextField.value = r.name + ' (' + type + ' - ' + ref + ')';
selectedBoundary = r.id;
setDownloadable(true);
});
});
}
else {
searchSuggestions.innerHTML = '<a class="dropdown-item disabled">Aucun résultat</a>';
}
})
.catch(e => {
console.log(e);
searchSuggestions.innerHTML = '<a class="dropdown-item disabled">Erreur</a>';
});
}, 250);
}
});
......@@ -128,31 +132,6 @@ function createForm() {
}
/*
* Associate search engine to input fields
*/
function initSearchBar(categoryId) {
selectedBoundary = null;
setDownloadable(false);
const category = data.boundaries.find(b => b.id === categoryId);
if(category) {
const searchTextField = document.getElementById("boundary-search");
searchTextField.value = "";
const datalist = document.getElementById("boundaries");
datalist.innerHTML = "";
category.list.forEach(v => {
const bdom = document.createElement("option");
bdom["data-value"] = v.id;
bdom.innerHTML = v.name;
datalist.appendChild(bdom);
});
}
}
/*
* Update nodes to allow download start or not
*/
......@@ -193,10 +172,10 @@ function setDownloadable(shouldI) {
* Wait for data retrieval
*/
const itv = setInterval(() => {
if(data && data.themes && data.boundaries) {
if(data && data.themes) {
clearInterval(itv);
if(data.themes === "error" || data.boundaries === "error") {
if(data.themes === "error") {
document.getElementById("form-failed").classList.remove("d-none");
document.getElementById("form-load").classList.add("d-none");
}
......
......@@ -94,29 +94,17 @@ for col in ${used_tags}; do
done
# Add boundaries for API requests
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE TABLE boundary AS SELECT osm_id, COALESCE(tags->'ref:INSEE', tags->'ref:NUTS', tags->'ref') AS ref, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, tags->'name' AS name, ST_SimplifyPreserveTopology(way, 10) AS way, ST_Centroid(way) AS center FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way); CREATE INDEX boundary_center_idx ON boundary USING GIST(center);"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE EXTENSION pg_trgm; CREATE EXTENSION unaccent; CREATE TABLE boundary AS SELECT osm_id, COALESCE(tags->'ref:INSEE', tags->'ref:FR:SIREN', tags->'ref:NUTS', tags->'ref') AS ref, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, tags->'name' AS name, lower(unaccent(tags->'name')) AS simple_name, ST_SimplifyPreserveTopology(way, 10) AS way, ST_Centroid(way) AS center FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way); CREATE INDEX boundary_center_idx ON boundary USING GIST(center); CREATE INDEX boundary_ref_trgm_idx ON boundary USING GIST(ref gist_trgm_ops); CREATE INDEX boundary_simple_name_trgm_idx ON boundary USING GIST(name gist_trgm_ops);"
# JSON boundary version for API
bounds_json="
WITH types(id, en, fr)
AS (VALUES
('admin_6', 'Department', 'Département'),
('admin_7', 'Pays', 'Pays (loi Voynet)'),
('admin_8', 'City', 'Commune'),
('political', 'Canton', 'Canton'),
('local_authority', 'Local authority', 'EPCI')
)
SELECT
array_to_json(array_agg(json_build_object(
'id', b.osm_id,
'name', b.name,
'ref', b.ref,
'type', b.type,
'type:en', t.en,
'type:fr', t.fr
'ref', b.ref
))) AS data
FROM boundary b
LEFT JOIN types t on t.id = b.type;
FROM boundary b;
"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment