works.py 29 KB
Newer Older
1
2
3
4
#!/usr/bin/env python
"""
Import French opendata about works on roads.
"""
5
import datetime
6
import json
7
8
9
10
11
12
13
14
15
16
17
18
19
import logging
import os
import sys

SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.abspath(os.path.join(SCRIPT_DIRECTORY, '..', '..')))

import arrow
import pyproj
import requests

from functools import partial

20
21
from shapely.geometry import (LineString, MultiPolygon, MultiLineString,
                              MultiPoint, Point)
22
from shapely.geometry import mapping, shape
23
24
25
from shapely.ops import transform

from server.models import db, Report
26
from server.tools import UTC_now
27
28

level = logging.WARN
29
RAISE_ON_EXCEPT = False
30
if 'DEBUG' in os.environ:
31
32
33
    level = logging.INFO
if 'RAISE_ON_EXCEPT' in os.environ:
    RAISE_ON_EXCEPT = True
34
35
logging.basicConfig(level=level)

36
37
38
# Same as in src/constants.js
REPORT_DOWNVOTES_THRESHOLD = 1

39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def preprocess_hauts_de_seine(data):
    out = []
    for item in data:
        try:
            fields = item['fields']
            new_item = {
                'fields': fields,
                'geometry': {
                    'type': 'Point',
                    # Use lng, lat
                    'coordinates': fields['geo_point_2d'][::-1]
                },
                'recordid': item['recordid'],
                'source': 'opendata-hauts-de-seine'
            }
            new_fields = new_item['fields']
            # Guess start date / end date
            # If the item is in the data, it should be valid for the current
            # trimester.
            now = datetime.datetime.now()
            new_fields['date_debut'] = arrow.get(
                datetime.datetime(
                    now.year,
63
                    max(1, min((now.month // 3) * 3, 12)),
64
65
66
67
68
69
                    1
                )
            ).isoformat()
            new_fields['date_fin'] = arrow.get(
                datetime.datetime(
                    now.year,
70
                    max(1, min((now.month // 3 + 1) * 3, 12)),
71
72
73
74
75
76
                    1
                )
            ).isoformat()
            out.append(new_item)
        except KeyError as exc:
            logging.warning(
77
                'Invalid item %s in Hauts-de-Seine data: %s.',
78
79
80
81
82
83
84
                item.get('recordid', '?'),
                exc
            )
            continue
    return out


85
def preprocess_lille(data):
86
    out = []
87
88
89
    for item in data:
        try:
            fields = item['fields']
90
91
92
93
94
95
96
97
98
99
100
101
102
            new_item = {
                'fields': fields,
                'geometry': {
                    'type': 'Point',
                    # Use lng, lat
                    'coordinates': fields['geo_point_2d'][::-1]
                },
                'recordid': item['recordid'],
                'source': 'opendata-lille'
            }
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = new_item['geometry']
103
            # Homogeneize start date spelling
104
105
            new_fields['date_debut'] = new_fields['date_demarrage']
            out.append(new_item)
106
        except KeyError as exc:
107
108
109
110
111
            logging.warning(
                'Invalid item %s in Lille data: %s.',
                item.get('recordid', '?'),
                exc
            )
112
            continue
113
    return out
114
115


116
117
def preprocess_loiret(data):
    out = []
118
119
120
121
    if not 'features' in data:
        logging.warning('Invalid data for Loiret.')
        return out

122
123
124
125
126
127
128
129
130
131
    for item in data['features']:
        try:
            if 'paths' in item['geometry']:
                paths = item['geometry']['paths']
            else:
                paths = [
                    [
                        [item['geometry']['x'], item['geometry']['y']]
                    ]
                ]
132
            # In Loiret, multiple paths are for multiple LineStrings
133
134
            for path in paths:
                if len(path) == 1:
135
                    geo_shape = {
136
137
138
139
                        'type': 'Point',
                        'coordinates': path[0]
                    }
                else:
140
                    geo_shape = {
141
142
143
144
145
                        'type': 'LineString',
                        'coordinates': path
                    }
                new_item = {
                    'fields': item['attributes'],
146
147
148
                    'geometry': shape(geo_shape).centroid,
                    'recordid': item['attributes']['OBJECTID'],
                    'source': 'opendata-loiret'
149
                }
150
151
152
                new_fields = new_item['fields']
                # Homogeneize geo_shape
                new_fields['geo_shape'] = geo_shape
153
                # Homogeneize start and end date spelling
154
155
                new_fields['date_debut'] = arrow.get(
                    float(new_fields['STARTDATE']) / 1000
156
                ).isoformat()
157
158
                new_fields['date_fin'] = arrow.get(
                    float(new_fields['ENDDATE']) / 1000
159
160
161
                ).isoformat()
                out.append(new_item)
        except KeyError as exc:
162
163
164
165
166
            logging.warning(
                'Invalid item %s in Loiret data: %s.',
                item.get('attributes', {}).get('OBJECTID', '?'),
                exc
            )
167
168
169
170
171
172
            if RAISE_ON_EXCEPT:
                raise
            continue
    return out


173
174
def preprocess_lyon(data):
    out = []
175
176
177
178
    if not 'features' in data:
        logging.warning('Invalid data for Lyon.')
        return out

179
180
181
182
    for item in data['features']:
        try:
            new_item = {
                'fields': item['properties'],
183
184
185
                'geometry': shape(item['geometry']).centroid,
                'recordid': item['properties']['identifiant'],
                'source': 'opendata-lyon'
186
            }
187
188
189
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = item['geometry']
190
            # Homogeneize start date and end date spelling
191
192
            new_fields['date_debut'] = new_fields['debutchantier']
            new_fields['date_fin'] = new_fields['finchantier']
193
194
            out.append(new_item)
        except KeyError as exc:
195
196
197
198
199
            logging.warning(
                'Invalid item %s in Lyon data: %s.',
                item.get('properties', {}).get('identifiant', '?'),
                exc
            )
200
201
            if RAISE_ON_EXCEPT:
                raise
202
203
204
205
206
207
            continue
    return out


def preprocess_montpellier(data):
    out = []
208
209
210
211
    if not 'features' in data:
        logging.warning('Invalid data for Montpellier.')
        return out

212
213
214
215
    for item in data['features']:
        try:
            new_item = {
                'fields': item['properties'],
216
217
218
                'geometry': shape(item['geometry']).centroid,
                'recordid': item['properties']['numero'],
                'source': 'opendata-montpellier'
219
            }
220
221
222
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = item['geometry']
223
            # Homogeneize start date and end date spelling
224
225
            new_fields['date_debut'] = new_fields['datedebut']
            new_fields['date_fin'] = new_fields['datefin']
226
227
            out.append(new_item)
        except KeyError as exc:
228
229
230
231
232
            logging.warning(
                'Invalid item %s in Montpellier data: %s.',
                item.get('properties', {}).get('numero', '?'),
                exc
            )
233
234
            if RAISE_ON_EXCEPT:
                raise
235
236
237
238
239
240
            continue
    return out


def preprocess_nancy(data):
    out = []
241
242
243
244
    if not 'features' in data:
        logging.warning('Invalid data for Nancy.')
        return out

245
246
247
248
249
250
251
252
253
254
255
256
    for item in data['features']:
        try:
            geometry = {
                'type': 'Point',
                'coordinates': [
                    item['geometry']['x'],
                    item['geometry']['y']
                ]
            }
            new_item = {
                'fields': item['attributes'],
                'geometry': geometry,
257
258
                'recordid': item['attributes']['OBJECTID'],
                'source': 'opendata-nancy'
259
            }
260
261
262
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = geometry
263
            # Homogeneize start and end date spelling
264
265
266
            if not new_fields['DATE_DEBUT'] or not new_fields['DATE_FIN']:
                # Invalid start / end date
                continue
267
268
            new_fields['date_debut'] = arrow.get(
                float(new_fields['DATE_DEBUT']) / 1000
269
            ).isoformat()
270
271
            new_fields['date_fin'] = arrow.get(
                float(new_fields['DATE_FIN']) / 1000
272
273
274
            ).isoformat()
            out.append(new_item)
        except KeyError as exc:
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
            logging.warning(
                'Invalid item %s in Nancy data: %s.',
                item.get('attributes', {}).get('OBJECTID', '?'),
                exc
            )
            if RAISE_ON_EXCEPT:
                raise
            continue
    return out


def preprocess_paris(data):
    out = []
    for item in data:
        try:
            new_item = {
                'fields': item['fields'],
                'geometry': item['geometry'],
                'recordid': item['recordid'],
                'source': 'opendata-paris',
            }
            out.append(new_item)
        except KeyError as exc:
            logging.warning(
                'Invalid item %s in Paris data: %s.',
                item.get('recordid', '?'),
                exc
            )
303
304
            if RAISE_ON_EXCEPT:
                raise
305
306
307
308
309
310
            continue
    return out


def preprocess_rennes(data):
    out = []
311
312
313
314
    if not 'features' in data:
        logging.warning('Invalid data for Rennes.')
        return out

315
316
317
318
    for item in data['features']:
        try:
            new_item = {
                'fields': item['properties'],
319
320
321
                'geometry': shape(item['geometry']),
                'recordid': item['properties']['id'],
                'source': 'opendata-rennes'
322
            }
323
324
325
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = item['geometry']
326
            # Homogeneize start date spelling
327
            new_fields['date_debut'] = new_fields['date_deb']
328
329
            out.append(new_item)
        except KeyError as exc:
330
331
332
333
334
            logging.warning(
                'Invalid item %s in Rennes data: %s.',
                item.get('properties', {}).get('id', '?'),
                exc
            )
335
336
            if RAISE_ON_EXCEPT:
                raise
337
338
339
340
341
342
            continue
    return out


def preprocess_seine_saint_denis(data):
    out = []
343
344
345
346
    if not 'features' in data:
        logging.warning('Invalid data for Seine-Saint-Denis.')
        return out

347
348
349
350
    for item in data['features']:
        try:
            new_item = {
                'fields': item['properties'],
351
352
353
                'geometry': shape(item['geometry']).centroid,
                'recordid': item['properties']['id'],
                'source': 'opendata-seine_saint_denis'
354
            }
355
356
            # Homogeneize geo_shape
            new_item['fields']['geo_shape'] = item['geometry']
357
358
            out.append(new_item)
        except KeyError as exc:
359
360
361
362
363
            logging.warning(
                'Invalid item %s in Seine-Saint-Denis data: %s.',
                item.get('properties', {}).get('id', '?'),
                exc
            )
364
365
            if RAISE_ON_EXCEPT:
                raise
366
367
368
369
370
            continue
    return out


def preprocess_sicoval(data):
371
    out = []
Phyks (Lucas Verney)'s avatar
Phyks (Lucas Verney) committed
372
373
374
375
376

    if 'error' in data:
        logging.warning('Invalid data for Sicoval.')
        return out

377
378
    for item in data:
        try:
379
380
381
382
383
384
385
386
387
            new_item = {
                'fields': item['fields'],
                'geometry': item['geometry'],
                'recordid': item['recordid'],
                'source': 'opendata-sicoval'
            }
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = new_fields['geoshape2']
388
            # Homogeneize start date and end date spelling
389
390
391
            new_fields['date_debut'] = new_fields['startdate']
            new_fields['date_fin'] = new_fields['enddate']
            out.append(new_item)
392
        except KeyError as exc:
393
394
395
396
397
            logging.warning(
                'Invalid item %s in Sicoval data: %s.',
                item.get('recordid', '?'),
                exc
            )
398
399
            if RAISE_ON_EXCEPT:
                raise
400
            continue
401
    return out
402
403
404


def preprocess_toulouse(data):
405
    out = []
406
407
    for item in data:
        try:
408
409
410
411
412
413
414
415
416
            new_item = {
                'fields': item['fields'],
                'geometry': item['geometry'],
                'recordid': item['recordid'],
                'source': 'opendata-toulouse'
            }
            new_fields = new_item['fields']
            # Homogeneize geo_shape
            new_fields['geo_shape'] = item['geometry']
417
            # Homogeneize start date and end date spelling
418
419
420
            new_fields['date_debut'] = new_fields['datedebut']
            new_fields['date_fin'] = new_fields['datefin']
            out.append(new_item)
421
        except KeyError as exc:
422
423
424
425
426
            logging.warning(
                'Invalid item %s in Toulouse data: %s.',
                item.get('recordid', '?'),
                exc
            )
427
428
            if RAISE_ON_EXCEPT:
                raise
429
            continue
430
    return out
431
432
433
434


def preprocess_versailles(data):
    out = []
435
436
437
438
    if not 'features' in data:
        logging.warning('Invalid data for Versailles.')
        return out

439
440
    for item in data['features']:
        try:
441
442
443
444
445
446
447
448
            if 'paths' in item['geometry']:
                paths = item['geometry']['paths']
            else:
                paths = [
                    [
                        [item['geometry']['x'], item['geometry']['y']]
                    ]
                ]
449
            # In Versailles, multiple paths are for multiple LineStrings
450
451
452
453
454
455
456
457
458
459
460
461
462
            for path in paths:
                if len(path) == 1:
                    geometry = {
                        'type': 'Point',
                        'coordinates': path[0]
                    }
                else:
                    geometry = {
                        'type': 'LineString',
                        'coordinates': path
                    }
                new_item = {
                    'fields': item['attributes'],
463
464
465
                    'geometry': shape(geometry).centroid,
                    'recordid': item['attributes']['OBJECTID'],
                    'source': 'opendata-versailles'
466
                }
467
468
469
                new_fields = new_item['fields']
                # Homogeneize geo_shape
                new_fields['geo_shape'] = geometry
470
                # Homogeneize start and end date spelling
471
472
                new_fields['date_debut'] = arrow.get(
                    float(new_fields['STARTDATE']) / 1000
473
                ).isoformat()
474
475
                new_fields['date_fin'] = arrow.get(
                    float(new_fields['ENDDATE']) / 1000
476
477
                ).isoformat()
                out.append(new_item)
478
        except KeyError as exc:
479
480
481
482
483
            logging.warning(
                'Invalid item %s in Versailles data: %s.',
                item.get('attributes', {}).get('OBJECTID', '?'),
                exc
            )
484
485
            if RAISE_ON_EXCEPT:
                raise
486
487
488
489
490
491
            continue
    return out


MIN_DISTANCE_REPORT_DETAILS = 40  # Distance in meters, same as in constants.js
OPENDATA_URLS = {
492
493
494
495
496
497
498
    # Work in Hauts de Seine
    # https://opendata.hauts-de-seine.fr/explore/dataset/travaux-sur-la-voirie-departementale-lignes/information/
    # Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
    "hauts-de-seine": {
        "preprocess": preprocess_hauts_de_seine,
        "url": "https://opendata.hauts-de-seine.fr/explore/dataset/travaux-sur-la-voirie-departementale-lignes/download/?format=json&timezone=Europe/Berlin",
    },
499
    # Work in Lille
500
501
    # https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/
    # Licence Ouverte (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2014/05/Licence_Ouverte.pdf
502
503
504
505
    "lille": {
        "preprocess": preprocess_lille,
        "url": "https://opendata.lillemetropole.fr/explore/dataset/troncons-de-voirie-impactes-par-des-travaux-en-temps-reel/download/?format=json&timezone=Europe/Berlin"
    },
506
507
508
509
510
511
512
    # Work in Loiret
    # https://open-loiret.opendata.arcgis.com/datasets/74c95548589d4ddeb3fcf094f7d61a67_1?geometry=0.609%2C47.694%2C3.245%2C48.016&orderBy=BLOCKNM
    # Custom license
    "loiret": {
        "preprocess": preprocess_loiret,
        "url": "https://services2.arcgis.com/IEzPuQhCEVCtkVvT/arcgis/rest/services/Travaux_routiers/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson",
    },
513
    # Work in Lyon
514
515
    # https://data.grandlyon.com/equipements/chantiers-perturbants-de-la-mftropole-de-lyon/
    # Licence Ouverte : https://download.data.grandlyon.com/files/grandlyon/LicenceOuverte.pdf
516
517
518
519
520
    "lyon": {
        "preprocess": preprocess_lyon,
        "url": "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=2.0.0&outputformat=GEOJSON&maxfeatures=30&request=GetFeature&typename=pvo_patrimoine_voirie.pvochantierperturbant&SRSNAME=urn:ogc:def:crs:EPSG::4171",
    },
    # Work in Montpellier
521
522
    # http://data.montpellier3m.fr/dataset/chantiers-genants-de-montpellier
    # Licence ODbL : http://opendefinition.org/licenses/odc-odbl/
523
524
525
526
527
    "montpellier": {
        "preprocess": preprocess_montpellier,
        "url": "http://data.montpellier3m.fr/sites/default/files/ressources/MMM_MMM_Chantiers.json"
    },
    # Work in Nancy
528
529
    # http://opendata.grandnancy.eu/jeux-de-donnees/detail-dune-fiche-de-donnees/?tx_icsoddatastore_pi1%5Bkeywords%5D=travaux&tx_icsoddatastore_pi1%5Buid%5D=63&tx_icsoddatastore_pi1%5BreturnID%5D=447
    # Licence libre / Licence Ouverte (Etalab)
530
531
532
533
534
    "nancy": {
        "preprocess": preprocess_nancy,
        "url": "https://geoservices.grand-nancy.org/arcgis/rest/services/public/VOIRIE_Info_Travaux_Niveau/MapServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
    },
    # Work in Paris
535
536
    # https://opendata.paris.fr/explore/dataset/chantiers-perturbants/
    # Licence ODbL : http://opendatacommons.org/licenses/odbl/
537
    "paris": {
538
        "preprocess": preprocess_paris,
539
540
541
        "url": "https://opendata.paris.fr/explore/dataset/chantiers-perturbants/download/?format=json&timezone=Europe/Berlin",
    },
    # Work in Rennes
542
543
    # http://travaux.data.rennesmetropole.fr/
    # Usage libre / Licence ODbL : http://opendatacommons.org/licenses/odbl/
544
545
546
547
548
    "rennes": {
        "preprocess": preprocess_rennes,
        "url": "http://travaux.data.rennesmetropole.fr/api/roadworks?epsg=4326"
    },
    # Work in Seine-Saint-Denis
549
550
    # https://geo.data.gouv.fr/fr/datasets/12504debb9bb73e717ad710a746541ebf817d98c
    # Licence Ouverte : https://www.etalab.gouv.fr/licence-ouverte-open-licence
551
552
    "seine-saint-denis": {
        "preprocess": preprocess_seine_saint_denis,
553
        "url": "https://geoportail93.fr/SERV/DATA/?TYPENAME=1570&FORMAT=GEOJSON&COL=id;titre&FORMAT=GEOJSON&COL=ALL&MODE=2"
554
555
    },
    # Work in Sicoval (South of Toulouse)
556
    # https://data.sicoval.fr/explore/dataset/travauxincidents/
557
    # Licence Ouverte v2.0 (Etalab) : https://www.etalab.gouv.fr/wp-content/uploads/2017/04/ETALAB-Licence-Ouverte-v2.0.pdf
558
559
    "sicoval": {
        "preprocess": preprocess_sicoval,
560
        "url": "https://data.sicoval.fr/explore/dataset/travauxincidents/download/?format=json&timezone=Europe/Berlin"
561
562
    },
    # Work in Toulouse
563
564
    # https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/
    # Licence ODbL : http://opendatacommons.org/licenses/odbl/
565
566
567
568
569
    "toulouse": {
        "preprocess": preprocess_toulouse,
        "url": "https://data.toulouse-metropole.fr/explore/dataset/chantiers-en-cours/download/?format=json&timezone=Europe/Berlin",
    },
    # Work in Versailles
570
571
572
573
574
575
576
577
578
579
    # Licence Ouverte (Etalab)
    "versailles-blocks": {  # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_0
        "preprocess": preprocess_versailles,
        "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/0/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
    },
    "versailles-closures": {  # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_1
        "preprocess": preprocess_versailles,
        "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
    },
    "versailles-detours": {  # http://www-cavgp.opendata.arcgis.com/datasets/f58091424f38424ba04a2d3933dc979e_2
580
        "preprocess": preprocess_versailles,
581
        "url": "https://services2.arcgis.com/YECJCCLQCtaylXWh/arcgis/rest/services/Waze/FeatureServer/1/query?where=1%3D1&text=&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=*&returnGeometry=true&maxAllowableOffset=&geometryPrecision=&outSR=4326&returnIdsOnly=false&returnCountOnly=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&returnZ=false&returnM=false&gdbVersion=&returnDistinctValues=false&f=pjson"
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
    },
}
REPORT_TYPE = 'interrupt'

# Projection from WGS84 (GPS) to Lambert93 (for easy buffering of polygons)
project = partial(
    pyproj.transform,
    pyproj.Proj(init='epsg:4326'),  # source coordinates system
    pyproj.Proj(init='epsg:2154')  # destination coordinates system
)


def process_opendata(name, data, report_type=REPORT_TYPE):
    # Coordinates of current reports in db, as shapely Points in Lambert93
    current_reports_points = []
597
598
    active_reports_from_db = Report.select().where(
        # Load reports from db of the current type
599
        (Report.type == report_type) &
600
601
602
        (
            # Either with an expiration_datetime in the future
            (
603
                (Report.expiration_datetime is not None) &
604
605
606
607
608
                (Report.expiration_datetime > UTC_now())
            ) |
            # Or without expiration_datetime but which are still active (shown
            # on the map)
            (
609
                (Report.expiration_datetime is None) &
610
611
612
613
614
                (Report.downvotes < REPORT_DOWNVOTES_THRESHOLD)
            )
        )
    )
    for report in active_reports_from_db:
615
616
617
618
619
620
621
622
623
624
        current_reports_points.append(
            transform(project, Point(report.lng, report.lat))
        )

    for item in data:
        try:
            fields = item['fields']

            # Check that the work is currently being done
            now = arrow.now('Europe/Paris')
625
            if fields['date_debut']:
626
                start_date = arrow.get(fields['date_debut'].replace('/', '-'))
627
628
629
630
            else:
                # Defaults to now if start date is unknown
                start_date = arrow.get(now)
            if fields['date_fin']:
631
                end_date = arrow.get(fields['date_fin'].replace('/', '-'))
632
633
634
            else:
                # Defaults to in a week if start date is unknown
                end_date = arrow.get(now).shift(days=+7)
635
636
637
638
639
640
641
            if not (start_date < now < end_date):
                logging.info(
                    'Ignoring record %s, work not currently in progress.',
                    item['recordid']
                )
                continue

642
            # Report geographical shape
643
            if 'geo_shape' in fields:
644
                maybe_multi_geo_shape = shape(fields['geo_shape'])
645
            else:
646
647
648
649
650
651
652
653
654
                maybe_multi_geo_shape = shape(item['geometry'])

            geo_shapes = []
            if (
                isinstance(maybe_multi_geo_shape, MultiPolygon)
                or isinstance(maybe_multi_geo_shape, MultiPoint)
            ):
                # Split MultiPolygon into multiple Polygon
                # Same for MultiPoint
655
656
                positions = [
                    p.centroid
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
                    for p in maybe_multi_geo_shape
                ]
                geo_shapes = [
                    p
                    for p in maybe_multi_geo_shape
                ]
            elif isinstance(maybe_multi_geo_shape, MultiLineString):
                # Split MultiLineString into multiple LineString
                positions = [
                    p.interpolate(0.5, normalized=True)
                    for p in maybe_multi_geo_shape
                ]
                geo_shapes = [
                    p
                    for p in maybe_multi_geo_shape
                ]
            elif isinstance(maybe_multi_geo_shape, LineString):
                # LineString, interpolate midpoint
                positions = [
                    maybe_multi_geo_shape.interpolate(0.5, normalized=True)
677
                ]
678
                geo_shapes = [maybe_multi_geo_shape]
679
            else:
680
                # Polygon or Point
681
                positions = [
682
                    maybe_multi_geo_shape.centroid
683
                ]
684
                geo_shapes = [maybe_multi_geo_shape]
685

686
            for (geo_shape, position) in zip(geo_shapes, positions):
687
688
                # Check if this precise position is already in the database
                if transform(project, position) in current_reports_points:
689
690
                    logging.info(
                        ('Ignoring record %s, a similar report is already in '
691
                            'the database.'),
692
693
                        item['recordid']
                    )
694
                    continue
695
696
697
                # Check no similar reports is within the area of the report, up
                # to the report distance.
                overlap_area = transform(project, geo_shape).buffer(
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
                    MIN_DISTANCE_REPORT_DETAILS
                )
                is_already_inserted = False
                for report_point in current_reports_points:
                    if report_point.within(overlap_area):
                        # A similar report is already there
                        is_already_inserted = True
                        logging.info(
                            ('Ignoring record %s, a similar report is already '
                             'in the database.'),
                            item['recordid']
                        )
                        break
                if is_already_inserted:
                    # Skip this report if a similar one is nearby
                    continue

                # Get the position of the center of the item
                lng, lat = position.x, position.y
                # Compute expiration datetime
                expiration_datetime = end_date.replace(microsecond=0).naive

                # Add the report to the db
                logging.info('Adding record %s to the database.',
                             item['recordid'])
                Report.create(
                    type=report_type,
                    expiration_datetime=expiration_datetime,
                    lat=lat,
                    lng=lng,
                    source=item['source'],
                    shape_geojson=json.dumps(mapping(geo_shape))
                )
731
732
733
734
735
736
737
738
        except KeyError as exc:
            logging.warning(
                'Invalid record %s in %s, missing key: %s',
                item.get('recordid', '?'),
                name,
                exc
            )

739

740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
if __name__ == '__main__':
    db.connect()
    for name, item in OPENDATA_URLS.items():
        logging.info('Processing opendata from %s', name)
        try:
            r = requests.get(item['url'])
            data = r.json()
        except (requests.RequestException, ValueError) as exc:
            logging.warning('Error while fetching data for %s: %s.',
                            name, exc)
            continue

        if item['preprocess']:
            data = item['preprocess'](data)

        process_opendata(name, data)