...
 
Commits (2)
......@@ -20,14 +20,16 @@
# limitations under the License.
"""Attend sur Eliasse les actualités sur les amendements et met à jour les fichiers JSON correspondants"""
"""Récupère sur Eliasse les actualités sur les amendements et met à jour les fichiers JSON correspondants"""
import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import threading
import time
......@@ -38,18 +40,7 @@ app_name = os.path.splitext(os.path.basename(__file__))[0]
context_by_organe = {}
legislature = 15
log = logging.getLogger(app_name)
organes = [
'AN', # Séance, Hémicycle,
'CION_AFETR', # Commission Affaires étrangères
'CION-CEDU', # Commission Affaires culturelles et éducation
'CION_DEF', # Commission Défense
'CION-DVP', # Commission Développement durable
'CION-ECO', # Commission Affaires économiques
'CION_FIN', # Commission Finances
'CION_LOIS', # Commission Lois
'CION-SOC', # Commission Affaires sociales
'CSCONF', # Commission spéciale
]
organes = None
target_dir = None
......@@ -66,7 +57,7 @@ def add_task(context, **kwargs):
tasks.append(kwargs)
def get_amendements(context, bibard = None, bibard_suffixe = None, full = False, numeros = None):
def get_amendements(context, bibard=None, bibard_suffixe=None, full=False, numeros=None):
organe = context['organe']
log.info('get_amendements(bibard={}, bibard_suffixe={}, organe={}, full={})'.format(
bibard, bibard_suffixe, organe, full))
......@@ -76,7 +67,8 @@ def get_amendements(context, bibard = None, bibard_suffixe = None, full = False,
assert numeros is not None
assert organe in organes
remaining_numeros = numeros
json_dir = os.path.join(target_dir, 'assemblee', str(legislature), organe, bibard + bibard_suffixe)
json_dir = os.path.join(target_dir, 'assemblee', str(
legislature), organe, bibard + bibard_suffixe)
os.makedirs(json_dir, exist_ok=True)
existing_filenames = set(
filename
......@@ -99,7 +91,7 @@ def get_amendements(context, bibard = None, bibard_suffixe = None, full = False,
)
data = response.json()
assert sorted(data.keys()) == ['amendements'], sorted(data.keys())
amendements = data['amendements']
amendements = data['amendements']
for amendement in amendements:
numero = amendement['numeroReference']
json_filename = 'amendement-{}.json'.format(numero)
......@@ -115,9 +107,10 @@ def get_amendements(context, bibard = None, bibard_suffixe = None, full = False,
os.remove(json_file_path)
def get_discussion(context, bibard = None, bibard_suffixe = None):
def get_discussion(context, bibard=None, bibard_suffixe=None):
organe = context['organe']
log.info('get_discussion(bibard={}, bibard_suffixe={}, organe={})'.format(bibard, bibard_suffixe, organe))
log.info('get_discussion(bibard={}, bibard_suffixe={}, organe={})'.format(
bibard, bibard_suffixe, organe))
assert bibard
assert bibard_suffixe is not None
bibard_suffixed = bibard + bibard_suffixe
......@@ -131,7 +124,8 @@ def get_discussion(context, bibard = None, bibard_suffixe = None):
),
)
data = response.json()
assert sorted(data.keys()) == ['amdtsParOrdreDeDiscussion'], sorted(data.keys())
assert sorted(data.keys()) == [
'amdtsParOrdreDeDiscussion'], sorted(data.keys())
discussion = data['amdtsParOrdreDeDiscussion']
write_json(discussion, os.path.join(
target_dir, 'assemblee', str(legislature), organe, bibard + bibard_suffixe, 'discussion.json'))
......@@ -181,7 +175,8 @@ def get_prochain_a_discuter(context):
# }
assert sorted(data.keys()) == ['prochainADiscuter']
data = data['prochainADiscuter']
write_json(data, os.path.join(target_dir, 'assemblee', str(legislature), organe, 'prochain-a-discuter.json'))
write_json(data, os.path.join(target_dir, 'assemblee', str(
legislature), organe, 'prochain-a-discuter.json'))
if int(data['legislature']) == legislature:
bibard_suffixed = data['bibard'] + data['bibardSuffixe']
......@@ -201,13 +196,41 @@ def get_prochain_a_discuter(context):
bibard=data['bibard'],
bibard_suffixe=data['bibardSuffixe'],
full=False,
numeros=numeros_without_sort + numeros[index:index + 2],
numeros=numeros_without_sort +
numeros[index:index + 2],
)
timer = threading.Timer(2, set_need, (context, 'need_prochain_a_discuter'))
timer.start()
def get_references_organes():
log.info('get_references_organes()')
response = requests.get(
'http://eliasse.assemblee-nationale.fr/eliasse/getListeReferenceDesOrganes.do',
)
data = response.json()
# [
# {
# "text":"Séance",
# "value":"AN"
# },{
# "text":"Toutes commissions",
# "value":"CION_TOUTE"
# },
# ...
# ]
write_json(data, os.path.join(
target_dir, 'assemblee', str(legislature), 'organes.json'))
global organes
organes = [
reference['value']
for reference in data
]
def get_textes_ordre_du_jour(context):
organe = context['organe']
log.info('get_textes_ordre_du_jour(organe={})'.format(organe))
......@@ -231,7 +254,8 @@ def get_textes_ordre_du_jour(context):
# }
# ]
write_json(data, os.path.join(target_dir, 'assemblee', str(legislature), organe, 'ordre-du-jour.json'))
write_json(data, os.path.join(target_dir, 'assemblee',
str(legislature), organe, 'ordre-du-jour.json'))
# Remove obsolete discussions.
bibard_suffixed_set = set(
......@@ -253,7 +277,8 @@ def get_textes_ordre_du_jour(context):
bibard_suffixe=discussion['textBibardSuffixe'],
)
timer = threading.Timer(3600, set_need, (context, 'need_textes_ordre_du_jour'))
timer = threading.Timer(
3600, set_need, (context, 'need_textes_ordre_du_jour'))
timer.start()
......@@ -293,17 +318,21 @@ def harvest_organe(organe):
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='increase output verbosity')
parser.add_argument('-v', '--verbose', action='store_true',
default=False, help='increase output verbosity')
parser.add_argument(
'target_dir',
help='path of target directory containing JSON files',
)
)
args = parser.parse_args()
global target_dir
target_dir = args.target_dir
logging.basicConfig(level = logging.INFO if args.verbose else logging.WARNING, stream = sys.stdout)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING, stream=sys.stdout)
get_references_organes()
threads = [
threading.Thread(target=harvest_organe, args=(organe,))
......@@ -331,8 +360,11 @@ def write_json(data, file_path):
else:
old_text = None
if new_text != old_text:
with open(file_path, 'w', encoding='utf-8') as json_file:
_, temp_file_path = tempfile.mkstemp(
suffix='.json', prefix='eliasse-', text=True)
with open(temp_file_path, 'w', encoding='utf-8') as json_file:
json_file.write(new_text)
shutil.move(temp_file_path, file_path)
return True
return False
......