Commit 82204beb authored by Adrien Pavie's avatar Adrien Pavie
Browse files

Flat nodes for DB, Osmium for pre-filter data

parent 4a73bfe3
......@@ -19,6 +19,7 @@ Les dépendances sont les suivantes :
* [jq](https://stedolan.github.io/jq/)
* GDAL >= 2.1 (et en particulier la commande `ogr2ogr`)
* zip
* [osmium-tool](https://github.com/osmcode/osmium-tool) >= 1.11
Installation des dépendances sous Debian 9 :
......@@ -43,6 +44,35 @@ sudo make install
cd ../../
rm -rf osm2pgsql
# Osmium-tool
sudo apt-get install libboost-program-options-dev libbz2-dev zlib1g-dev libexpat1-dev
mkdir work
cd work
git clone https://github.com/mapbox/protozero
cd protozero
mkdir build
cd build
cmake ..
make
sudo make install
cd ../../
git clone https://github.com/osmcode/libosmium
cd libosmium
mkdir build
cd build
cmake ..
make
sudo make install
cd ../../
git clone https://github.com/osmcode/osmium-tool
cd osmium-tool
mkdir build
cd build
cmake ..
make
sudo make install
cd ../../
# PostgreSQL 11
sudo echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
......@@ -65,7 +95,7 @@ cd GeoDataMine
Les paramètres généraux de configuration du service sont à renseigner dans le fichier `config.json`. Les variables à renseigner sont les suivantes :
* `WORK_DIR` : dossier de travail, dans lequel les fichiers temporaires sont stockés (exemple: `/tmp/osm4odf`)
* `PBF_URL` : URL de téléchargement de l'extrait OpenStreetMap au format PBF (exemple: `http://download.geofabrik.de/europe/france-latest.osm.pbf`)
* `PBF_URL` : URL de téléchargement de l'extrait OpenStreetMap au format PBF (exemple: `http://download.openstreetmap.fr/extracts/europe/france-latest.osm.pbf`)
* `PG_HOST` : nom d'hôte pour accéder à la base de données PostgreSQL (exemple: `localhost`)
* `PG_USER` : nom d'utilisateur pour accéder à la base de données PostgreSQL (exemple: `postgres`)
* `PG_USER` : numéro de port pour accéder à la base de données PostgreSQL (exemple: `5432`)
......
{
"WORK_DIR": "/home/adrien/tmp/osm4odf",
"PBF_URL": "http://download.geofabrik.de/europe/france/reunion-latest.osm.pbf",
"PBF_URL": "http://download.openstreetmap.fr/extracts/europe/france/bretagne-latest.osm.pbf",
"PARALLEL_JOBS": "6",
"PG_HOST": "localhost",
......
......@@ -86,7 +86,6 @@
node,way boundary text polygon
node,way admin_level text polygon
node,way name text linear
node,way area text polygon
......@@ -97,7 +96,6 @@ node,way area text polygon
node,way leisure text polygon
node,way amenity text polygon
node,way building text polygon
node,way tourism text polygon
node,way landuse text polygon
node,way shop text polygon
......
......@@ -6,7 +6,11 @@
# #
##############################################
# Start date (for logs)
date
# Read configuration from JSON file
echo "======= Preparing ======="
for s in $(cat "../config.json" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
export $s
done
......@@ -24,17 +28,20 @@ fi
last_ts_file="${WORK_DIR}/last_ts.txt"
yesterday_ts=`date +%s -d 'yesterday'`
pbf_file="${WORK_DIR}/extract.osm.pbf"
pbf_light_file="${WORK_DIR}/extract_light.osm.pbf"
# We check how old is the latest extract to avoid over-downloading
if ! [ -f "${last_ts_file}" -a -r "${last_ts_file}" -a `cat "${last_ts_file}"` -gt "${yesterday_ts}" ]; then
echo "======= OSM data downloading ======="
if ! [ -f "${last_ts_file}" -a -f "${last_ts_file}" -a `if [ -f "${last_ts_file}" ]; then cat "${last_ts_file}"; else echo "0"; fi` -gt "${yesterday_ts}" ]; then
# Start download
if wget "${PBF_URL}" -O "${pbf_file}"; then
if wget "${PBF_URL}" --quiet -O "${pbf_file}"; then
# Success -> Write TS file
date +%s > "${last_ts_file}"
rm -f "${pbf_light_file}"
else
# Failure -> Stop script
echo "Can't download file"
rm "${pbf_file}" "${last_ts_file}"
rm -f "${pbf_file}" "${pbf_light_file}" "${last_ts_file}"
exit 1
fi
else
......@@ -43,6 +50,22 @@ fi
set -e
# Pre-filter PBF file to only import what's necessary
echo "======= Filtering input PBF data (osmium) ======="
used_tags=`cat ./import.style | grep -v '^#' | grep -v '^$' | sed -r 's/ +/#/g' | cut -d '#' -f 2`
used_tags=${used_tags/area/}
if ! [ -f "${pbf_light_file}" ]; then
osmium tags-filter \
${pbf_file} \
${used_tags} \
-f pbf,add_metadata=false \
--no-progress \
--overwrite -o ${pbf_light_file}
else
echo "Reuse existing light PBF file"
fi
# Setup next database
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "DROP DATABASE IF EXISTS ${PG_DB_TMP}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "CREATE DATABASE ${PG_DB_TMP}"
......@@ -50,26 +73,27 @@ psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREAT
# Start import
echo "======= Import OSM data into DB ======="
flat_nodes="${WORK_DIR}/flat.nodes"
osm2pgsql --create \
-H "${PG_HOST}" -U "${PG_USER}" -P "${PG_PORT}" -d "${PG_DB_TMP}" \
--cache "${CACHE_MEM}" \
--slim --drop \
--flat-nodes "${flat_nodes}" \
--number-processes "${PARALLEL_JOBS}" \
--style "import.style" \
--hstore --multi-geometry --keep-coastlines --extra-attributes \
"${pbf_file}"
#rm "${flat_nodes}"
"${pbf_light_file}"
# Create indexes for all tables
for col in `cat ./import.style | grep -v '^#' | grep -v '^$' | sed -r 's/ +/#/g' | cut -d '#' -f 2`; do
echo "======= Post-processing ======="
for col in ${used_tags}; do
echo "Create index for $col"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE INDEX planet_osm_point_${col//:/_}_idx ON planet_osm_point(\"$col\"); CREATE INDEX planet_osm_line_${col//:/_}_idx ON planet_osm_line(\"$col\"); CREATE INDEX planet_osm_polygon_${col//:/_}_idx ON planet_osm_polygon(\"$col\");"
done
# Add boundaries for API requests
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE TABLE boundary AS SELECT osm_id, COALESCE(tags->'ref:INSEE', tags->'ref:NUTS', tags->'ref') AS ref, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, name, ST_SimplifyPreserveTopology(way, 10) AS way, ST_Centroid(way) AS center FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way); CREATE INDEX boundary_center_idx ON boundary USING GIST(center);"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "CREATE TABLE boundary AS SELECT osm_id, COALESCE(tags->'ref:INSEE', tags->'ref:NUTS', tags->'ref') AS ref, CASE WHEN boundary = 'administrative' THEN 'admin_'||admin_level ELSE boundary END AS type, tags->'name' AS name, ST_SimplifyPreserveTopology(way, 10) AS way, ST_Centroid(way) AS center FROM planet_osm_polygon WHERE boundary IN ('administrative', 'local_authority', 'political') AND (admin_level IS NULL OR (admin_level::int >= 6 AND admin_level::int <= 8)); CREATE INDEX boundary_osm_id_idx ON boundary(osm_id); CREATE INDEX boundary_way_idx ON boundary USING GIST(way); CREATE INDEX boundary_center_idx ON boundary USING GIST(center);"
# JSON boundary version for API
bounds_json="
......@@ -105,16 +129,20 @@ sleep 30
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -t -A -c "${bounds_json}" > "${WORK_DIR}/bounds.json"
# Rename databases to make next goes live
echo "======= Switch next with current DB ======="
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "DROP DATABASE IF EXISTS ${PG_DB_OLD}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname='${PG_DB_CLEAN}'"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_TMP}" -c "ALTER DATABASE ${PG_DB_CLEAN} RENAME TO ${PG_DB_OLD}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_OLD}" -c "ALTER DATABASE ${PG_DB_TMP} RENAME TO ${PG_DB_CLEAN}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "DROP DATABASE IF EXISTS ${PG_DB_OLD}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_CLEAN}" -c "VACUUM FULL"
# Unlock API
rm "${lock_file}"
# Cleanup
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -c "DROP DATABASE IF EXISTS ${PG_DB_OLD}"
psql -h "${PG_HOST}" -U "${PG_USER}" -p "${PG_PORT}" -d "${PG_DB_CLEAN}" -c "VACUUM FULL"
end=`date +%s`
date
echo "Done in $((end-start)) seconds"
......@@ -2,7 +2,7 @@
--GEOMETRY=point
SELECT
t.osm_id, t.name, t.access, t.tags->'operator' AS operator,
t.osm_id, t.tags->'name' AS name, t.access, t.tags->'operator' AS operator,
t.tags->'opening_hours' AS opening_hours,
t.tags->'defibrillator:location' AS location, <GEOM>
FROM <TABLE>
......
......@@ -2,7 +2,7 @@
--GEOMETRY=point,polygon
SELECT
t.osm_id, t.name, t.access, t.tags->'operator' AS operator,
t.osm_id, t.tags->'name' AS name, t.access, t.tags->'operator' AS operator,
t.tags->'covered' AS covered, t.tags->'capacity' AS capacity,
t.tags->'bicycle_parking' AS type, <GEOM>
FROM <TABLE>
......
......@@ -2,6 +2,6 @@
--GEOMETRY=point,polygon
SELECT
t.osm_id, t.name, t.tags->'amenity' AS type, t.tags->'operator:type' AS statut, <GEOM>
t.osm_id, t.tags->'name' AS name, t.tags->'amenity' AS type, t.tags->'operator:type' AS statut, <GEOM>
FROM <TABLE>
WHERE <GEOMCOND> AND t.amenity IN ('school', 'kindergarten', 'college', 'university')
......@@ -5,7 +5,7 @@ SELECT
t.osm_id, t.internet_access AS connexion_type, t.tags->'internet_access:fee' AS fee,
COALESCE(t.tags->'internet_access:description', t.tags->'description') AS description,
t.tags->'internet_access:operator' AS internet_operator, t.tags->'internet_access:ssid' AS ssid,
t.name AS place_name, t.access AS place_access, t.tags->'opening_hours' AS place_opening_hours,
t.tags->'name' AS place_name, t.access AS place_access, t.tags->'opening_hours' AS place_opening_hours,
COALESCE(t.amenity, t.shop, t.tourism, t.office, 'unknown') AS place_type, <GEOM>
FROM <TABLE>
WHERE <GEOMCOND> AND internet_access IS NOT NULL AND internet_access != 'no'
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment