Si vous constatez la perte de tickets, de commentaires ou de projets, veuillez consulter la page https://status.framasoft.org/incident/626.

Commit cecaeeda authored by Christophe Benz's avatar Christophe Benz

Refactor and improve transpiler (especially dependencies between variables)

parent 30192e83
......@@ -2,7 +2,6 @@
import importlib
import itertools
from .formulas_helpers import * # noqa
......@@ -10,13 +9,6 @@ from .formulas_helpers import * # noqa
saisies = None # Global variable used by formulas, filled by `evaluate_formulas`
# Functional helpers
def mapcat(function, sequence):
return itertools.chain.from_iterable(map(function, sequence))
# Core functions
......
......@@ -19,7 +19,8 @@ import pprint
import sys
import textwrap
from calculateur_impots.core import mapcat
from toolz.curried import concat, map, mapcat, pipe, valmap
from calculateur_impots.transpiler import to_dependencies_visitors, to_python_source_visitors
from calculateur_impots.transpiler.base import sanitized_variable_name
from calculateur_impots.transpiler.dependencies_helpers import get_ordered_formulas_names
......@@ -36,21 +37,9 @@ script_dir_path = os.path.dirname(os.path.abspath(__file__))
generated_dir_path = os.path.abspath(os.path.join(script_dir_path, '..', 'generated'))
# Python helpers
list_ = list # To use in ipdb since "list" is reserved to display the current source code.
# Source code helper functions
def get_formula_source(formula_source_by_name, formula_name):
return formula_source_by_name.get(
formula_name,
'{} = 0'.format(formula_name),
)
def lines_to_python_source(sequence):
return ''.join(itertools.chain.from_iterable(zip(sequence, itertools.repeat('\n'))))
......@@ -85,7 +74,7 @@ def iter_json_file_names(*pathnames):
pathnames,
)):
json_file_name = os.path.basename(json_file_path)
if args.json is None or args.json == json_file_name:
if args.json is None or json_file_name in args.json:
file_name_head = os.path.splitext(json_file_name)[0]
yield json_file_name
......@@ -97,9 +86,9 @@ def load_regles_file(json_file_name):
lambda node: 'batch' in node['applications'],
regles_nodes,
))
# dependencies = list(map(to_dependencies_visitors.visit_node, batch_application_regles_nodes))
formula_name_and_source_list = mapcat(to_python_source_visitors.visit_node, batch_application_regles_nodes)
return formula_name_and_source_list
formula_name_and_source_list = list(mapcat(to_python_source_visitors.visit_node, batch_application_regles_nodes))
formula_name_and_dependencies_list = list(mapcat(to_dependencies_visitors.visit_node, batch_application_regles_nodes))
return formula_name_and_source_list, formula_name_and_dependencies_list
def load_tgvH_file():
......@@ -125,7 +114,7 @@ def load_tgvH_file():
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-d', '--debug', action='store_true', default=False, help='Display debug messages')
parser.add_argument('--json', help='Parse only this JSON file and exit')
parser.add_argument('--json', nargs='+', help='Parse only this JSON file and exit')
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Increase output verbosity')
parser.add_argument('json_dir', help='Directory containing the JSON AST files')
global args
......@@ -139,9 +128,10 @@ def main():
os.mkdir(generated_dir_path)
if args.json is not None:
json_file_path = os.path.join(args.json_dir, args.json)
if not os.path.exists(json_file_path):
parser.error('JSON file "{}" does not exist.'.format(json_file_path))
for json_file_name in args.json:
json_file_path = os.path.join(args.json_dir, json_file_name)
if not os.path.exists(json_file_path):
parser.error('JSON file "{}" does not exist.'.format(json_file_path))
# Transpile variables definitions (before regles)
variable_definition_by_name = load_tgvH_file()
......@@ -151,12 +141,22 @@ def main():
)
# Load regles
formula_source_by_name = dict(mapcat(
vals = list(map(
load_regles_file,
iter_json_file_names('chap-*.json', 'res-ser*.json'),
))
# import ipdb; ipdb.set_trace()
# formula_source_by_name = dict(mapcat())
formula_source_by_name, formula_dependencies_by_name = pipe(
zip(*vals),
map(concat),
map(dict),
)
formula_dependencies_by_name = valmap(list, formula_dependencies_by_name)
dependencies_file_path = os.path.join(generated_dir_path, 'dependencies.json')
with open(dependencies_file_path, 'w') as dependencies_file:
dependencies_file.write(json.dumps(formula_dependencies_by_name, indent=2))
log.info('Output file "{}" written with success'.format(dependencies_file_path))
# Load verifs
# for json_file_name in iter_json_file_names('coc*.json', 'coi*.json'):
......@@ -180,7 +180,10 @@ def main():
'from .constants import *',
'\n',
),
map(get_formula_source, ordered_formulas_names),
map(
lambda formula_name: formula_source_by_name.get(formula_name, '{} = 0'.format(formula_name)),
ordered_formulas_names,
),
)),
)
......
# -*- coding: utf-8 -*-
import inspect
import sys
def get_visitor(node, module_name):
visitor_name = 'visit_' + node['type']
module = sys.modules[module_name]
module_visitors = filter(lambda member: member[0].startswith('visit_'), inspect.getmembers(module))
visitor = dict(module_visitors).get(visitor_name)
return visitor
def sanitized_variable_name(value):
# Python variables must not begin with a digit.
return '_' + value if value[0].isdigit() else value
......@@ -8,7 +8,9 @@ Dependencies helpers are functions dealing with enumerating formulas in the righ
import logging
from ..core import get_variable_type, mapcat
from toolz import mapcat
from ..core import get_variable_type
log = logging.getLogger(__name__)
......
# -*- coding: utf-8 -*-
import pprint
import logging
from toolz.curried import concat, filter, mapcat, pipe
from .base import get_visitor, sanitized_variable_name
from .unloop_helpers import iter_unlooped_nodes
log = logging.getLogger(__name__)
# Main visitor
def visit_node(node, parenthesised=False):
"""
Main transpilation function which calls the specific transpilation functions below.
They are subfunctions to ensure they are never called directly.
"""
transpilation_function_name = 'visit_' + node['type']
if transpilation_function_name not in globals():
error_message = '"def {}(node):" is not defined, node = {}'.format(
transpilation_function_name,
pprint.pformat(node, width=120),
)
raise NotImplementedError(error_message)
transpilation_function = globals()[transpilation_function_name]
result = transpilation_function(node)
deep_level = 0
def visit_node(node):
"""Main visitor which calls the specific visitors below."""
global deep_level
visitor = get_visitor(module_name=__name__, node=node)
log.debug('{}:{}:{}'.format(deep_level, visitor.__name__, node))
deep_level += 1
result = list(visitor(node))
deep_level -= 1
log.debug('{}:{}: => {}'.format(deep_level, visitor.__name__, result))
assert deep_level >= 0, deep_level
return result
# Specific visitors
# def visit_regle(node):
# import ipdb; ipdb.set_trace()
def visit_boolean_expression(node):
return mapcat(visit_node, node['operands'])
def visit_comparaison(node):
return mapcat(visit_node, (node['left_operand'], node['right_operand']))
def visit_dans(node):
return visit_node(node['expression'])
# def visit_enumeration_values(node):
def visit_float(node):
return []
def visit_formula(node):
sanitized_formula_name = sanitized_variable_name(node['name'])
dependencies = set(visit_node(node['expression']))
sanitized_dependencies = set(map(sanitized_variable_name, dependencies))
return (sanitized_formula_name, sanitized_dependencies)
def visit_function_call(node):
return mapcat(visit_node, node['arguments'])
def visit_integer(node):
return []
def visit_interval(node):
return []
def visit_loop_expression(node):
return mapcat(visit_node, iter_unlooped_nodes(
loop_variables_nodes=node['loop_variables'],
node=node['expression'],
))
def visit_pour_formula(node):
return map(visit_node, iter_unlooped_nodes(
loop_variables_nodes=node['loop_variables'],
node=node['formula'],
unloop_keys=['name'],
))
def visit_product_expression(node):
return mapcat(visit_node, node['operands'])
def visit_regle(node):
return mapcat(
lambda node: visit_node(node) if node['type'] == 'pour_formula' else [visit_node(node)],
node['formulas'],
)
def visit_sum_expression(node):
return mapcat(visit_node, node['operands'])
def visit_symbol(node):
return [node['value']]
def visit_ternary_operator(node):
return pipe([
visit_node(node['value_if_true']),
visit_node(node['condition']),
visit_node(node['value_if_false']) if 'value_if_false' in node else None,
],
filter(None),
concat,
)
......@@ -8,15 +8,16 @@ These functions are called by transpiler.core.
"""
# from functools import reduce
import itertools
import json
import logging
import pprint
import textwrap
from ..core import get_variable_type, mapcat
from .base import sanitized_variable_name
from toolz import mapcat
from ..core import get_variable_type
from .base import get_visitor, sanitized_variable_name
from .unloop_helpers import iter_unlooped_nodes
......@@ -27,7 +28,7 @@ log = logging.getLogger(__name__)
def visit_infix_expression(node, operators={}):
def merge(*iterables):
def interleave(*iterables):
for values in itertools.zip_longest(*iterables, fillvalue=UnboundLocalError):
for index, value in enumerate(values):
if value != UnboundLocalError:
......@@ -37,26 +38,11 @@ def visit_infix_expression(node, operators={}):
visit_node(operand_or_operator)
if index == 0
else operators.get(operand_or_operator, operand_or_operator)
for index, operand_or_operator in merge(node['operands'], node['operators'])
for index, operand_or_operator in interleave(node['operands'], node['operators'])
)
return ' '.join(map(str, tokens))
def iter_find_nodes(node, type, skip_type=None):
"""
Iterates over all nodes matching `type` recursively in `node`.
"""
if isinstance(node, dict):
if node['type'] == type:
yield node
elif skip_type is None or node['type'] != skip_type:
yield from iter_find_nodes(node=list(node.values()), skip_type=skip_type, type=type)
elif isinstance(node, list):
for child_node in node:
if isinstance(child_node, (list, dict)):
yield from iter_find_nodes(node=child_node, skip_type=skip_type, type=type)
# Main visitor
......@@ -64,35 +50,32 @@ deep_level = 0
def visit_node(node, parenthesised=False):
"""
Main transpilation function which calls the specific transpilation functions below.
They are subfunctions to ensure they are never called directly.
"""
"""Main visitor which calls the specific visitors below."""
global deep_level
transpilation_function_name = 'visit_' + node['type']
if transpilation_function_name not in globals():
visitor = get_visitor(module_name=__name__, node=node)
visitor_name = 'visit_' + node['type']
if visitor is None:
error_message = '"def {}(node):" is not defined, node = {}'.format(
transpilation_function_name,
visitor_name,
pprint.pformat(node, width=120),
)
raise NotImplementedError(error_message)
nb_prefix_chars = len('DEBUG:' + __name__) + 1
transpilation_function_short_name = transpilation_function_name[len('visit_'):]
visitor_short_name = visitor_name[len('visit_'):]
prefix = '> {}{}'.format(
transpilation_function_short_name + ' ' * (nb_prefix_chars - len(transpilation_function_short_name) - 3) + ':',
visitor_short_name + ' ' * (nb_prefix_chars - len(visitor_short_name) - 3) + ':',
' ' * deep_level * 4,
)
node_str = textwrap.indent(json.dumps(node, indent=4), prefix=prefix)[nb_prefix_chars:].lstrip()
log.debug(
'{}{}({})'.format(
' ' * deep_level * 4 + '={}=> '.format(deep_level),
transpilation_function_name,
visitor_name,
node_str,
)
)
transpilation_function = globals()[transpilation_function_name]
deep_level += 1
result = transpilation_function(node)
result = visitor(node)
assert result is not None
deep_level -= 1
unparenthesised_node_types = ('float', 'function_call', 'integer', 'string', 'symbol')
......@@ -126,10 +109,6 @@ def visit_comparaison(node):
)
def visit_variable_const(node):
return '{} = {}'.format(node['name'], node['value'])
def visit_dans(node):
return '{} {} {}'.format(
visit_node(node['expression'], parenthesised=True),
......@@ -147,21 +126,9 @@ def visit_float(node):
def visit_formula(node):
# state['current_formula'] = []
expression_source = visit_node(node['expression'])
# dependencies = set(map(
# lambda node: sanitized_variable_name(node['value']),
# iter_find_nodes(
# node=node,
# skip_type='loop_expression', # visit_loop_expression will handle its dependencies.
# type='symbol',
# ),
# ))
formula_name = sanitized_variable_name(node['name'])
# state['formulas_dependencies'][formula_name] = dependencies
source = '{} = {}'.format(formula_name, expression_source)
# state['formulas_sources'][formula_name] = source
# state['current_formula'] = None
return (formula_name, source)
......@@ -199,19 +166,6 @@ def visit_interval(node):
def visit_loop_expression(node):
# def iter_unlooped_loop_expressions_nodes_and_dependencies():
# for unlooped_node in iter_unlooped_nodes(
# loop_variables_nodes=node['loop_variables'],
# node=node['expression'],
# ):
# dependencies = set(map(
# lambda node: sanitized_variable_name(node['value']),
# iter_find_nodes(node=unlooped_node, type='symbol'),
# ))
# yield unlooped_node, dependencies
# unlooped_loop_expressions_nodes, dependencies = zip(*iter_unlooped_loop_expressions_nodes_and_dependencies())
# dependencies = reduce(or_, dependencies)
unlooped_loop_expressions_nodes = iter_unlooped_nodes(
loop_variables_nodes=node['loop_variables'],
node=node['expression'],
......@@ -219,7 +173,6 @@ def visit_loop_expression(node):
source = '[{}]'.format(
', '.join(map(visit_node, unlooped_loop_expressions_nodes)),
)
# state['current_formula'].append(dependencies)
return source
......@@ -235,10 +188,10 @@ def visit_loop_variable(node):
def visit_pour_formula(node):
return map(visit_node, iter_unlooped_nodes(
loop_variables_nodes=node['loop_variables'],
node=node['formula'],
unloop_keys=['name'],
))
loop_variables_nodes=node['loop_variables'],
node=node['formula'],
unloop_keys=['name'],
))
def visit_product_expression(node):
......@@ -246,7 +199,7 @@ def visit_product_expression(node):
def visit_regle(node):
return map(
return mapcat(
lambda node1: visit_node(node1) if node1['type'] == 'pour_formula' else [visit_node(node1)],
node['formulas'],
)
......@@ -270,6 +223,5 @@ def visit_ternary_operator(node):
)
def visit_verif(node):
pass
# import ipdb; ipdb.set_trace()
def visit_variable_const(node):
return '{} = {}'.format(node['name'], node['value'])
......@@ -10,7 +10,7 @@ They take a node containing a loop and return as many "unlooped" nodes as the nu
import copy
import itertools
from ..core import mapcat
from toolz import mapcat
def enumeration_node_to_sequence(enumeration_node):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment