Commit 582947b1 authored by JonathanMM's avatar JonathanMM
Browse files

Utilisation de l'opendata

parent a1510599
import hashlib
import time
import os
from config import getURLVelib
import urllib.request
def fichierEnBinaire(file):
with file:
return file.read()
urlVelib = getURLVelib()
tmpFileName = 'testMd5.json'
nbEssai = 0
while nbEssai < 15:
urllib.request.urlretrieve(urlVelib, tmpFileName)
dataBrut = open(tmpFileName)
# On regarde si ce n'est pas le md5 corrompu !
valeurMD5 = hashlib.md5(fichierEnBinaire(
open(tmpFileName, 'rb'))).hexdigest()
print(valeurMD5)
os.remove(tmpFileName)
time.sleep(15)
import pymysql
def getMysqlConnection():
return pymysql.connect(host="", user="",passwd="",db="")
return pymysql.connect(host="", user="", passwd="", db="")
def getURLVelib():
return ''
def getURLDump():
return 'https://velib.nocle.fr/dump/'
\ No newline at end of file
return 'https://velib.nocle.fr/dump/'
import urllib.request, json, os, pymysql
import urllib.request
import json
import os
import pymysql
from datetime import datetime
from config import getMysqlConnection, getURLVelib
from config import getMysqlConnection
from adresseLib import getAdresse, getInsee
def getStationIdByLatLong(latitude, longitude):
tmpFileName = 'gbfsStation.json'
urllib.request.urlretrieve("https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json", tmpFileName)
urllib.request.urlretrieve(
"https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json", tmpFileName)
data = json.load(open(tmpFileName))
os.remove(tmpFileName)
for stat in data['data']['stations']:
if abs(stat['lat'] - latitude) < 0.00002 and abs(stat['lon'] - longitude) < 0.00002:
return stat['station_id']
def getGBFSData():
tmpFileName = 'gbfsStatus.json'
urllib.request.urlretrieve("https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json", tmpFileName)
urllib.request.urlretrieve(
"https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json", tmpFileName)
data = json.load(open(tmpFileName))
os.remove(tmpFileName)
return data['data']['stations']
\ No newline at end of file
return data['data']['stations']
from adresseLib import getAdresse
import pymysql, urllib, hashlib, os, time, json
from config import getMysqlConnection, getURLVelib
import pymysql
import urllib
import hashlib
import os
import time
import json
from config import getMysqlConnection
def val(valeur):
if valeur is None:
return 'NULL'
return '"'+str(valeur)+'"'
def fichierEnBinaire(file):
with file:
return file.read()
def resetCoordonnees():
mysql = getMysqlConnection()
requete = mysql.cursor()
urlVelib = getURLVelib()
urlVelib = ""
tmpFileName = 'detailsStations.json'
nbEssai = 0
fichierOk = False
......@@ -23,15 +31,16 @@ def resetCoordonnees():
urllib.request.urlretrieve(urlVelib, tmpFileName)
dataBrut = open(tmpFileName)
#On regarde si ce n'est pas le md5 corrompu !
valeurMD5 = hashlib.md5(fichierEnBinaire(open(tmpFileName, 'rb'))).hexdigest()
# On regarde si ce n'est pas le md5 corrompu !
valeurMD5 = hashlib.md5(fichierEnBinaire(
open(tmpFileName, 'rb'))).hexdigest()
fichierOk = valeurMD5 != 'c746fec461216aa3a67602b879dafa6c'
nbEssai += 1
if not fichierOk:
os.remove(tmpFileName)
time.sleep(10)
if not fichierOk: #On n'a trouvé de valeurs correctes :(
if not fichierOk: # On n'a trouvé de valeurs correctes :(
return False
data = json.load(dataBrut)
......@@ -48,8 +57,10 @@ def resetCoordonnees():
latitude = infoStation['gps']['latitude']
adresse = getAdresse(latitude, longitude)
requete = mysql.cursor()
requete.execute('UPDATE stations SET longitude = '+val(longitude)+', latitude = '+val(latitude)+', adresse = '+val(adresse)+' WHERE code = '+val(codeStation))
requete.execute('UPDATE stations SET longitude = '+val(longitude)+', latitude = '+val(
latitude)+', adresse = '+val(adresse)+' WHERE code = '+val(codeStation))
mysql.close()
resetCoordonnees()
\ No newline at end of file
resetCoordonnees()
......@@ -3,16 +3,22 @@ import json
import os
import pymysql
from datetime import datetime
from config import getMysqlConnection, getURLVelib
from config import getMysqlConnection
from adresseLib import getAdresse, getInsee
from gbfsLib import getStationIdByLatLong
import hashlib
import time
import ssl
import requests
def val(valeur):
if valeur is None:
return 'NULL'
if valeur is True:
return '1'
if valeur is False:
return '0'
return '"'+str(valeur)+'"'
......@@ -21,7 +27,57 @@ def fichierEnBinaire(file):
return file.read()
def downloadData():
urlVelib = "https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json"
tmpFileName = 'detailsStations.json'
nbEssai = 0
fichierOk = False
dataBrut = None
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while nbEssai < 3 and not fichierOk:
with urllib.request.urlopen(urlVelib, context=ctx) as u, \
open(tmpFileName, 'wb') as f:
f.write(u.read())
dataBrut = open(tmpFileName)
# On regarde si ce n'est pas le md5 corrompu !
dataBinaire = open(tmpFileName, 'rb')
valeurMD5 = hashlib.md5(fichierEnBinaire(dataBinaire)).hexdigest()
dataBinaire.close()
bannedMD5 = ['c746fec461216aa3a67602b879dafa6c',
'6a6b5a9972d13bf04d9100585a7a1449']
fichierOk = not valeurMD5 in bannedMD5
nbEssai += 1
if not fichierOk:
os.remove(tmpFileName)
time.sleep(10)
else:
dataBrut.close()
return tmpFileName
return None
def getContentDataStation():
url = "https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_information.json"
results = requests.get(url,
headers={'Content-Type': 'application/json'})
return results.json()
def getAllStation():
fileName = downloadData()
# if fileName is None: # On n'a trouvé de valeurs correctes :(
# print("Impossible de récupérer les données")
# return False
mysql = getMysqlConnection()
# On récupère la liste des stations déjà en base
......@@ -54,96 +110,88 @@ def getAllStation():
nbStationsOuvertes = 0
nbStationsOuvertesDetectees = 0
urlVelib = getURLVelib()
tmpFileName = 'detailsStations.json'
nbEssai = 0
fichierOk = False
dataBrut = None
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while nbEssai < 3 and not fichierOk:
with urllib.request.urlopen(urlVelib, context=ctx) as u, \
open(tmpFileName, 'wb') as f:
f.write(u.read())
dataBrut = open(tmpFileName)
# On regarde si ce n'est pas le md5 corrompu !
valeurMD5 = hashlib.md5(fichierEnBinaire(
open(tmpFileName, 'rb'))).hexdigest()
bannedMD5 = ['c746fec461216aa3a67602b879dafa6c',
'6a6b5a9972d13bf04d9100585a7a1449']
fichierOk = not valeurMD5 in bannedMD5
nbEssai += 1
if not fichierOk:
os.remove(tmpFileName)
time.sleep(10)
if not fichierOk: # On n'a trouvé de valeurs correctes :(
return False
data = json.load(dataBrut)
for etatStation in data:
infoStation = etatStation['station']
try:
codeStation = int(infoStation['code'])
except Exception:
codeStation = 0
if codeStation > 100:
if codeStation not in stations:
longitude = infoStation['gps']['longitude']
latitude = infoStation['gps']['latitude']
if infoStation['state'] != 'Operative':
strDateOuverture = 'NULL'
else:
dataStationFromFile = getContentDataStation()
with open(fileName) as dataBrut:
data = json.load(dataBrut)
for etatStation in data['data']['stations']:
try:
codeStation = int(etatStation['stationCode'])
except Exception:
codeStation = 0
print("Station : " + str(codeStation))
if codeStation > 100:
stationCherche = None
for stationData in dataStationFromFile['data']['stations']:
codeStationFile = 0
try:
codeStationFile = int(stationData['stationCode'])
except Exception:
continue
if codeStationFile > 0 and codeStationFile == codeStation:
stationCherche = stationData
break
if codeStation not in stations and stationCherche != None:
longitude = stationCherche['lon']
latitude = stationCherche['lat']
strDateOuverture = 'CURDATE()'
# On récupère également le stationId des données GTFS
#stationId = getStationIdByLatLong(latitude, longitude)
requete = mysql.cursor()
requete.execute('INSERT INTO stations (code, name, longitude, latitude, type, dateOuverture, adresse, insee, stationId) VALUES \
('+val(codeStation)+', '+val(infoStation['name'])+', '+val(longitude)+', '+val(latitude)+', '+val(infoStation['type'])+', '+strDateOuverture+', '+val(getAdresse(latitude, longitude))+', '+val(getInsee(codeStation))+', '+val(None)+')')
mysql.commit()
requete.close()
stations.append(codeStation)
nbBike = int(etatStation['nbBike'])
nbEbike = int(etatStation['nbEbike'])
nbFreeEDock = int(etatStation['nbFreeDock']) + \
int(etatStation['nbFreeEDock'])
nbEDock = int(etatStation['nbDock'])+int(etatStation['nbEDock'])
nbBikeOverflow = int(etatStation['nbBikeOverflow'])
nbEBikeOverflow = int(etatStation['nbEBikeOverflow'])
requete = mysql.cursor()
requete.execute('INSERT INTO status (code, idConso, state, nbBike, nbEBike, nbFreeEDock, nbEDock, nbBikeOverflow, nbEBikeOverflow, maxBikeOverflow, overflow, overflowActivation) VALUES \
('+val(codeStation)+', '+val(strIdConso)+', '+val(infoStation['state'])+', '+val(nbBike)+', '+val(nbEbike)+', '+val(nbFreeEDock)+', '+val(nbEDock)+', '+val(nbBikeOverflow)+', '+val(nbEBikeOverflow)+', '+val(etatStation['maxBikeOverflow'])+', '+val(etatStation['overflow'])+', '+val(etatStation['overflowActivation'])+')')
mysql.commit()
requete.close()
# On met à jour la station au besoin
if codeStation in stationsFutur and nbEDock > 0:
# On récupère également le stationId des données GTBS
stationId = stationCherche['station_id']
requete = mysql.cursor()
requete.execute('INSERT INTO stations (code, name, longitude, latitude, type, dateOuverture, adresse, insee, stationId) VALUES \
('+val(codeStation)+', '+val(stationCherche['name'])+', '+val(longitude)+', '+val(latitude)+', '+val("Unknown")+', '+strDateOuverture+', '+val(getAdresse(latitude, longitude))+', '+val(getInsee(codeStation))+', '+val(stationId)+')')
mysql.commit()
requete.close()
stations.append(codeStation)
bikesAvailable = etatStation['num_bikes_available_types']
nbBike = 0
nbEbike = 0
for etatBikesAvailable in bikesAvailable:
if 'mechanical' in etatBikesAvailable:
nbBike = int(etatBikesAvailable['mechanical'])
if 'ebike' in etatBikesAvailable:
nbEbike = int(etatBikesAvailable['ebike'])
nbFreeEDock = int(etatStation['num_docks_available'])
# nbEDock = int(etatStation['numDocksAvailable'])
if stationCherche != None:
nbEDock = int(stationCherche['capacity'])
else:
nbEDock = nbFreeEDock
nbBikeOverflow = 0
nbEBikeOverflow = 0
state = "Operative"
requete = mysql.cursor()
requete.execute('UPDATE stations \
SET dateOuverture = CURDATE() WHERE code = '+val(codeStation))
requete.execute('INSERT INTO status (code, idConso, state, nbBike, nbEBike, nbFreeEDock, nbEDock, nbBikeOverflow, nbEBikeOverflow, maxBikeOverflow, overflow, overflowActivation) VALUES \
('+val(codeStation)+', '+val(strIdConso)+', '+val(state)+', '+val(nbBike)+', '+val(nbEbike)+', '+val(nbFreeEDock)+', '+val(nbEDock)+', '+val(nbBikeOverflow)+', '+val(nbEBikeOverflow)+', '+val(0)+', '+val(0)+', '+val(0)+')')
mysql.commit()
requete.close()
# On ajoute a la conso
nbTotalBike += nbBike
nbTotalEBike += nbEbike
nbTotalFreeEDock += nbFreeEDock
nbTotalEDock += nbEDock
nbTotalBikeOverflow += nbBikeOverflow
nbTotalEBikeOverflow += nbEBikeOverflow
if infoStation['state'] == "Operative":
nbStationsOuvertes += 1
if nbEDock > 0 and nbBike + nbEbike + nbFreeEDock > 0:
nbStationsOuvertesDetectees += 1
os.remove(tmpFileName)
# On met à jour la station au besoin
if codeStation in stationsFutur and nbEDock > 0:
requete = mysql.cursor()
requete.execute('UPDATE stations \
SET dateOuverture = CURDATE() WHERE code = '+val(codeStation))
mysql.commit()
requete.close()
# On ajoute a la conso
nbTotalBike += nbBike
nbTotalEBike += nbEbike
nbTotalFreeEDock += nbFreeEDock
nbTotalEDock += nbEDock
nbTotalBikeOverflow += nbBikeOverflow
nbTotalEBikeOverflow += nbEBikeOverflow
if state == "Operative":
nbStationsOuvertes += 1
if nbEDock > 0 and nbBike + nbEbike + nbFreeEDock > 0:
nbStationsOuvertesDetectees += 1
os.remove(fileName)
# On insert tout dans le statut
requete = mysql.cursor()
......
......@@ -6,54 +6,45 @@
-- Généré le : Ven 01 Mai 2020 à 16:44
-- Version du serveur : 10.1.44-MariaDB-0ubuntu0.18.04.1
-- Version de PHP : 7.0.22-0ubuntu0.17.04.1
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */
;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */
;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */
;
/*!40101 SET NAMES utf8mb4 */
;
--
-- Base de données : `c6velib`
--
-- --------------------------------------------------------
--
-- Structure de la table `comptage_resumeConso`
--
CREATE TABLE `comptage_resumeConso` (
`id` int(11) NOT NULL,
`date` datetime NOT NULL,
`duree` int(4) NOT NULL,
`nbStation` int(5) NOT NULL,
`nombre` int(6) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
) ENGINE = MyISAM DEFAULT CHARSET = utf8;
-- --------------------------------------------------------
--
-- Structure de la table `comptage_resumeStatus`
--
CREATE TABLE `comptage_resumeStatus` (
`id` int(11) NOT NULL,
`code` int(11) NOT NULL,
`date` datetime NOT NULL,
`duree` int(4) NOT NULL,
`nombre` int(6) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
) ENGINE = MyISAM DEFAULT CHARSET = utf8 ROW_FORMAT = COMPACT;
-- --------------------------------------------------------
--
-- Structure de la table `comptage_stations`
--
CREATE TABLE `comptage_stations` (
`code` int(11) NOT NULL,
`nom` varchar(128) NOT NULL,
......@@ -62,59 +53,49 @@ CREATE TABLE `comptage_stations` (
`dateOuverture` date DEFAULT NULL,
`type` tinyint(4) NOT NULL DEFAULT '1',
`parent` int(11) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
-- --------------------------------------------------------
--
-- Structure de la table `comptage_status`
--
CREATE TABLE `comptage_status` (
`id` int(11) NOT NULL,
`code` int(11) NOT NULL,
`idConso` int(11) NOT NULL,
`nombre` int(11) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
-- --------------------------------------------------------
--
-- Structure de la table `comptage_statusConso`
--
CREATE TABLE `comptage_statusConso` (
`id` int(11) NOT NULL,
`date` datetime NOT NULL,
`nbStation` int(11) NOT NULL DEFAULT '0',
`nombre` int(11) NOT NULL DEFAULT '0'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
-- --------------------------------------------------------
--
-- Structure de la table `config`
--
CREATE TABLE `config` (
`id` int(11) NOT NULL,
`cle` varchar(64) NOT NULL,
`valeur` varchar(256) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
--
-- Contenu de la table `config`
--
INSERT INTO `config` (`id`, `cle`, `valeur`) VALUES
(1, 'comptage-derniere-maj', '2020-05-01T05:53:39+00:00');
INSERT INTO `config` (`id`, `cle`, `valeur`)
VALUES (
1,
'comptage-derniere-maj',
'2020-05-01T05:53:39+00:00'
);
-- --------------------------------------------------------
--
-- Structure de la table `resumeConso`
--
CREATE TABLE `resumeConso` (
`id` int(11) NOT NULL,
`date` datetime NOT NULL,
......@@ -123,28 +104,25 @@ CREATE TABLE `resumeConso` (
`nbStationDetecte` int(5) DEFAULT NULL,
`nbBikeMin` int(6) NOT NULL,
`nbBikeMax` int(6) NOT NULL,
`nbBikeMoyenne` decimal(8,2) NOT NULL,
`nbBikeMoyenne` decimal(8, 2) NOT NULL,
`nbEBikeMin` int(6) NOT NULL,
`nbEBikeMax` int(6) NOT NULL,
`nbEBikeMoyenne` decimal(8,2) NOT NULL,
`nbEBikeMoyenne` decimal(8, 2) NOT NULL,
`nbBikeOverflowMin` int(6) DEFAULT NULL,
`nbBikeOverflowMax` int(6) DEFAULT NULL,
`nbBikeOverflowMoyenne` decimal(8,2) DEFAULT NULL,
`nbBikeOverflowMoyenne` decimal(8, 2) DEFAULT NULL,
`nbEbikeOverflowMin` int(6) DEFAULT NULL,
`nbEbikeOverflowMax` int(6) DEFAULT NULL,
`nbEbikeOverflowMoyenne` decimal(8,2) DEFAULT NULL,
`nbEbikeOverflowMoyenne` decimal(8, 2) DEFAULT NULL,
`nbFreeEDockMin` int(6) NOT NULL,
`nbFreeEDockMax` int(6) NOT NULL,
`nbFreeEDockMoyenne` decimal(8,2) NOT NULL,
`nbFreeEDockMoyenne` decimal(8, 2) NOT NULL,
`nbEDock` int(6) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
) ENGINE = MyISAM DEFAULT CHARSET = utf8;
-- --------------------------------------------------------
--
-- Structure de la table `resumeStatus`
--
CREATE TABLE `resumeStatus` (
`id` int(11) NOT NULL,
`code` int(11) NOT NULL,
......@@ -152,68 +130,59 @@ CREATE TABLE `resumeStatus` (
`duree` int(4) NOT NULL,
`nbBikeMin` int(3) NOT NULL,
`nbBikeMax` int(3) NOT NULL,
`nbBikeMoyenne` decimal(5,2) NOT NULL,
`nbBikeMoyenne` decimal(5, 2) NOT NULL,
`nbBikePris` int(3) NOT NULL,
`nbBikeRendu` int(3) NOT NULL,
`nbEBikeMin` int(3) NOT NULL,
`nbEBikeMax` int(3) NOT NULL,
`nbEBikeMoyenne` decimal(5,2) NOT NULL,
`nbEBikeMoyenne` decimal(5, 2) NOT NULL,
`nbEBikePris` int(3) NOT NULL,
`nbEBikeRendu` int(3) NOT NULL,
`nbFreeEDockMin` int(3) NOT NULL,
`nbFreeEDockMax` int(3) NOT NULL,
`nbFreeEDockMoyenne` decimal(5,2) NOT NULL,
`nbFreeEDockMoyenne` decimal(5, 2) NOT NULL,
`nbEDock` int(3) NOT NULL,
`nbBikeOverflowMin` int(3) NOT NULL,
`nbBikeOverflowMax` int(3) NOT NULL,
`nbBikeOverflowMoyenne` decimal(5,2) NOT NULL,
`nbBikeOverflowMoyenne` decimal(5, 2) NOT NULL,
`nbEBikeOverflowMin` int(3) NOT NULL,
`nbEBikeOverflowMax` int(3) NOT NULL,
`nbEBikeOverflowMoyenne` decimal(5,2) NOT NULL,
`nbEBikeOverflowMoyenne` decimal(5, 2) NOT NULL,
`maxBikeOverflow` int(3) NOT NULL,
`nbEDockPerdusMin` int(3) DEFAULT NULL,
`nbEDockPerdusMax` int(3) DEFAULT NULL,
`nbEDockPerdusMoyenne` decimal(5,2) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;
`nbEDockPerdusMoyenne` decimal(5, 2) DEFAULT NULL
) ENGINE = MyISAM DEFAULT CHARSET = utf8 ROW_FORMAT = COMPACT;
-- --------------------------------------------------------
--
-- Structure de la table `signalement`
--
CREATE TABLE `signalement` (
`id` int(11) NOT NULL,
`code` int(11) NOT NULL,
`dateSignalement` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`estFonctionnel` tinyint(1) NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
) ENGINE = MyISAM DEFAULT CHARSET = utf8;
-- --------------------------------------------------------