Commit 50875fb2 authored by François Fortin's avatar François Fortin
Browse files

speed optimisation in metro_xml,

removed all call to "exec" found inside loop
now the dynamically generated code by "exec" is generated only once.

removed "exec" at couple of other places
parent 9b841979
......@@ -84,6 +84,34 @@ class Metro_dom2metro(Metro_module):
self.domObservation_ref = None
self.domStation = pStation.get_input_information()
#
# construct dictionnary of read handler
#
import toolbox.metro_dom2metro_handler
# Retrieve the informations about the data type
dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
dData_type = metro_util.join_dictionaries(dStandard_data_type,
dExtended_data_type)
dReadHandlers = {}
# construct dictionnary of read handlers (only done once so not expensive)
sMessage = _("-------------------------- Read handlers available --------------------------\n")
for dType in dData_type:
if dData_type[dType]['READ'] != "":
sCode = "handler = " + dData_type[dType]['READ']
exec sCode
dReadHandlers[dType] = handler
sMessage += _("TYPE: %s , HANDLER: %s\n") % (dType.ljust(15),dData_type[dType]['READ'])
sMessage += "-----------------------------------------------------------------------------"
metro_logger.print_message(metro_logger.LOGGER_MSG_DEBUG,
sMessage)
#
# Forecast extraction
#
......@@ -122,6 +150,7 @@ class Metro_dom2metro(Metro_module):
sHeader_xpath,
lStandard_forecast,
lExtended_forecast,
dReadHandlers,
sData_xpath)
except IOERROR:
sXmlError = _("XML error in file '%s'.") % (sFilename)
......@@ -174,6 +203,7 @@ class Metro_dom2metro(Metro_module):
sHeader_xpath,
lStandard_observation,
lExtended_observation,
dReadHandlers,
sData_xpath)
except :
sXmlError = _("XML error in file '%s'.\n") % (sFilename) +\
......@@ -234,6 +264,7 @@ class Metro_dom2metro(Metro_module):
sHeader_xpath,
lStandard_observation,
lExtended_observation,
dReadHandlers,
sData_xpath)
except:
sXmlError = _("XML error in file '%s'.") % (sFilename)
......@@ -289,6 +320,7 @@ class Metro_dom2metro(Metro_module):
sHeader_xpath,
lStandard_roadlayer,
lExtended_roadlayer,
dReadHandlers,
sData_xpath)
except:
......@@ -320,6 +352,7 @@ class Metro_dom2metro(Metro_module):
def __extract_data_from_dom(self, metro_data, domDom,
ldHeader_keys, sHeader_xpath,
lStdData_keys, lExtData_keys,
dReadHandlers,
sData_xpath):
"""
Name: __extract_data_from_dom
......@@ -332,6 +365,9 @@ class Metro_dom2metro(Metro_module):
[I] sHeader_xpath: xpath of the header
[I] lStdData_keys: like ldHeader_keys but for standard data
[I] lExtData_keys: like ldHeader_keys but for extended data
[I] dReadHandlers: dictionnay of read handler.
key = data type name (ex: INTEGER),
value = handler (ex: toolbox.metro_dom2metro_handler.read_integer)
[I] sData_xpath : xpath for the data.
Output: metro_data : object containing all the data extracted.
......@@ -343,7 +379,7 @@ class Metro_dom2metro(Metro_module):
if ldHeader_keys != None and sHeader_xpath != None:
# extraction of data included in nodes
lHeader_data = metro_xml.extract_xpath(ldHeader_keys,\
lHeader_data = metro_xml.extract_xpath(ldHeader_keys,dReadHandlers,\
domDom, sHeader_xpath)
# Add all the elements in the header's dictionnary
......@@ -359,7 +395,7 @@ class Metro_dom2metro(Metro_module):
if lData_keys != None and sData_xpath != None:
# extraction of all the nodes included in the DOM
lData_matrix = metro_xml.extract_xpath(lData_keys, domDom,\
lData_matrix = metro_xml.extract_xpath(lData_keys, dReadHandlers, domDom,\
sData_xpath, True)
# matrix transfer in object Metro_data
......
......@@ -68,11 +68,39 @@ class Metro_metro2dom( Metro_module ):
forecast_data = pForecast.get_data_collection()
pRoadcast = self.get_infdata_reference('ROADCAST')
roadcast_data = pRoadcast.get_data_collection()
#
# construct dictionnary of write handler
#
import toolbox.metro_metro2dom_handler
# Retrieve the informations about the data type
dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
dData_type = metro_util.join_dictionaries(dStandard_data_type,
dExtended_data_type)
dWriteHandlers = {}
# construct dictionnary of write handlers (only done once so not expensive)
sMessage = _("-------------------------- Write handlers available --------------------------\n")
for dType in dData_type:
if dData_type[dType]['WRITE'] != "":
sCode = "handler = " + dData_type[dType]['WRITE']
exec sCode
dWriteHandlers[dType] = handler
sMessage += _("TYPE: %s , HANDLER: %s\n") % (dType.ljust(15),dData_type[dType]['WRITE'])
sMessage += "-----------------------------------------------------------------------------"
metro_logger.print_message(metro_logger.LOGGER_MSG_DEBUG,
sMessage)
# Create forecast
if forecast_data != None:
domForecast = \
self.__create_forecast(forecast_data.get_original_data())
self.__create_forecast(forecast_data.get_original_data(),dWriteHandlers)
else:
metro_logger.print_message(metro_logger.LOGGER_MSG_WARNING,
_("No forecast, can't create DOM forecast"))
......@@ -81,7 +109,7 @@ class Metro_metro2dom( Metro_module ):
# Create roadcast
if roadcast_data != None:
domRoadcast = \
self.__create_roadcast(roadcast_data.get_subsampled_data())
self.__create_roadcast(roadcast_data.get_subsampled_data(),dWriteHandlers)
else:
metro_logger.print_message(metro_logger.LOGGER_MSG_WARNING,
......@@ -100,7 +128,7 @@ class Metro_metro2dom( Metro_module ):
def get_send_type( self ):
return Metro_module.DATATYPE_DOM_OUT
def __create_forecast( self, data ):
def __create_forecast( self, data, dWriteHandlers ):
#
# DOM and root element creation
......@@ -123,7 +151,7 @@ class Metro_metro2dom( Metro_module ):
sHeader_xpath = metro_config.get_value('XML_FORECAST_XPATH_HEADER')
self.__create_header(domDoc, nodeRoot, sHeader_xpath,
lHeader_keys, data)
lHeader_keys, dWriteHandlers, data)
#
......@@ -142,12 +170,12 @@ class Metro_metro2dom( Metro_module ):
'XML_FORECAST_XPATH_PREDICTION')
self.__create_matrix(domDoc, nodeRoot, sPrediction_xpath,
lPrediction_keys, data)
lPrediction_keys, dWriteHandlers, data)
return domDoc
def __create_roadcast( self, data ):
def __create_roadcast( self, data, dWriteHandlers ):
#
# DOM and root element creation
......@@ -170,7 +198,7 @@ class Metro_metro2dom( Metro_module ):
sHeader_xpath = metro_config.get_value('XML_ROADCAST_XPATH_HEADER')
self.__create_header(domDoc, nodeRoot, sHeader_xpath,
lHeader_keys, data)
lHeader_keys, dWriteHandlers, data)
#
......@@ -187,13 +215,13 @@ class Metro_metro2dom( Metro_module ):
metro_config.get_value('XML_ROADCAST_XPATH_PREDICTION')
self.__create_matrix(domDoc, nodeRoot, sPrediction_xpath,
lPrediction_keys, data)
lPrediction_keys, dWriteHandlers, data)
return domDoc
def __create_header( self, domDoc, nodeRoot, sHeader_xpath,
lHeader_keys, metro_data ):
lHeader_keys, dWriteHandlers, metro_data ):
lHeader_xpath = string.split(sHeader_xpath,"/")
......@@ -210,11 +238,11 @@ class Metro_metro2dom( Metro_module ):
# header creation
metro_xml.create_node_tree_from_dict(domDoc, nodeHeader,
lHeader_keys, dHeader)
lHeader_keys, dWriteHandlers, dHeader)
def __create_matrix( self, domDoc, nodeRoot, sPrediction_xpath,
lPrediction_keys, metro_data):
lPrediction_keys, dWriteHandlers, metro_data):
lPrediction_xpath = string.split(sPrediction_xpath,"/")
......@@ -242,4 +270,5 @@ class Metro_metro2dom( Metro_module ):
metro_xml.create_node_tree_from_matrix(domDoc, nodePrediction,
sPrediction_node_name,
lPrediction_keys,
dWriteHandlers,
metro_data, naMatrix)
......@@ -129,6 +129,9 @@ def metro_get_execution_sequence():
"""
Fetch the METRo execution sequence.
"""
metro_logger.print_init_message(metro_logger.LOGGER_INIT_MESSAGE,
_("Construct METRo execution sequence:"))
lModule_execution_sequence = metro_config.get_value\
("INIT_MODULE_EXECUTION_SEQUENCE")
......@@ -140,9 +143,13 @@ def metro_get_execution_sequence():
for sModule_name in lModule_execution_sequence:
module = __import__(sModule_name)
sClass_name = string.capitalize(sModule_name)
#print sClass_name
exec "tmp_object = module." + sClass_name + "()"
lObject_sequence.append(tmp_object)
metro_logger.print_init_message(metro_logger.LOGGER_INIT_SUCCESS,
_("METRo execution sequence ready"))
return lObject_sequence
......
......@@ -567,15 +567,15 @@ def set_default_value( ):
'WRITE':"toolbox.metro_metro2dom_handler.write_real"},
'DATE': \
{'READ' :"metro_dom2metro_handler.read_date",
'WRITE':"metro_metro2dom_handler.write_date"},
{'READ' :"toolbox.metro_dom2metro_handler.read_date",
'WRITE':"toolbox.metro_metro2dom_handler.write_date"},
'STRING': \
{'READ' :"metro_dom2metro_handler.read_string",
'WRITE':"metro_metro2dom_handler.write_string"},
{'READ' :"toolbox.metro_dom2metro_handler.read_string",
'WRITE':"toolbox.metro_metro2dom_handler.write_string"},
'COORDINATE': \
{'READ' :"metro_dom2metro_handler.read_coordinate",
{'READ' :"toolbox.metro_dom2metro_handler.read_coordinate",
'WRITE':"",
'CHILD':[{'NAME':"LATITUDE",
'XML_TAG':"latitude",
......@@ -588,7 +588,7 @@ def set_default_value( ):
'LIST_LEVEL_TEMP': \
{'READ' :"",
'WRITE':"metro_metro2dom_handler.write_list",
'WRITE':"toolbox.metro_metro2dom_handler.write_list",
'CHILD':[{'NAME':"LEV-TEMP",
'XML_TAG':"lev-temp",
'DATA_TYPE':'REAL',
......@@ -597,7 +597,7 @@ def set_default_value( ):
'VERTICAL_LEVELS': \
{'READ' :"",
'WRITE':"metro_metro2dom_handler.write_list",
'WRITE':"toolbox.metro_metro2dom_handler.write_list",
'CHILD':[{'NAME':"DEPTH",
'XML_TAG':"depth",
'DATA_TYPE':'REAL',
......@@ -606,12 +606,12 @@ def set_default_value( ):
'ROADLAYER_TYPE': \
{'READ' :"metro_dom2metro_handler.read_roadlayer_type",
{'READ' :"toolbox.metro_dom2metro_handler.read_roadlayer_type",
'WRITE':""},
'DEFAULT': \
{'READ' :"metro_dom2metro_handler.read_string",
'WRITE':"metro_metro2dom_handler.write_string"}
{'READ' :"toolbox.metro_dom2metro_handler.read_string",
'WRITE':"toolbox.metro_metro2dom_handler.write_string"}
},
'FROM' :CFG_INTERNAL,
'COMMENTS':_("standard data type for METRo XML files")}
......
......@@ -66,17 +66,17 @@ def read_date(sTag,nodeBranch):
fDate = 0.0
return fDate
def read_coordinate(lChild,nodeBranch):
def read_coordinate(lChild,nodeBranch,dReadHandler):
if nodeBranch:
lRslt = metro_xml.extract_data(lChild,nodeBranch[0])
lRslt = metro_xml.extract_data(lChild,dReadHandler,nodeBranch[0])
else:
lRslt = []
return list_to_coordinate(lRslt)
def read_station_components(lChild,nodeBranch):
def read_station_components(lChild,nodeBranch,dReadHandler):
if nodeBranch:
rslt = metro_xml.extract_data(lChild,nodeBranch[0])
rslt = metro_xml.extract_data(lChild,dReadHandler,nodeBranch[0])
else:
rslt = None
return rslt
......
......@@ -46,7 +46,15 @@ import types
import metro_util
_ = metro_util.init_translation('metro_error')
# remove comment, need it because received the error:
# File "/home/phobos/software/cmc/metro/svn/metro/src/frontend/toolbox/metro_error.py", line 49, in <module>
# _ = metro_util.init_translation('metro_error')
# File "/home/phobos/software/cmc/metro/svn/metro/src/frontend/toolbox/metro_util.py", line 635, in init_translation
# '/usr/share/locale')
# File "/usr/lib/python2.6/gettext.py", line 469, in translation
# raise IOError(ENOENT, 'No translation file found for domain', domain)
#IOError: [Errno 2] No translation file found for domain: 'metro_error'
#_ = metro_util.init_translation('metro_error')
class Metro_error(Exception):
"""
......
......@@ -55,7 +55,7 @@ def write_date( sXml_tag, fDate ):
sDate = metro_date.seconds2iso8601(fDate)
return metro_xml.create_text_node(sXml_tag, sDate)
def write_list( sXml_list_tag, lChildList, lValues ):
def write_list( sXml_list_tag, lChildList, lValues, dWriteHandlers ):
listNode = metro_xml.create_node( sXml_list_tag)
......@@ -63,10 +63,8 @@ def write_list( sXml_list_tag, lChildList, lValues ):
# Get handler
#
dDef = lChildList[0]
sWriteHandler = metro_xml.get_handler('WRITE', dDef)
sImportHandlerCode = metro_xml.get_handler_import_code(sWriteHandler)
exec sImportHandlerCode
sData_type_name = dDef['DATA_TYPE']
sXml_tag = dDef['XML_TAG']
#
......@@ -74,11 +72,8 @@ def write_list( sXml_list_tag, lChildList, lValues ):
#
num=0
for val in lValues:
# Construction of instruction doing the function call that will
# create the node
sCode = "nodeData = " + sWriteHandler + "(sXml_tag,val)"
exec sCode
nodeData = dWriteHandlers[sData_type_name](sXml_tag,val)
metro_xml.set_attribute(nodeData,"num",str(num))
metro_xml.append_child(listNode,nodeData)
......
......@@ -127,6 +127,7 @@ def test_import( sModule_name ):
"""
Used to test check if a module is present.
"""
try:
sCode = "import " + sModule_name
exec sCode
......@@ -137,6 +138,7 @@ def test_function_existence( sModule_name, sFunction_name ):
"""
Utiliser pour tester la presence d'une fonction dans un module.
"""
try:
sCode_import = "import " + sModule_name
exec sCode_import
......
......@@ -117,12 +117,15 @@ def xpath( domDom, sXpath ):
sRslt = None
return sRslt
def extract_xpath(lDefs, domDom, sXpath, bIs_matrix=False):
def extract_xpath(lDefs, dReadHandlers, domDom, sXpath, bIs_matrix=False):
"""
Name: extract_xpath
Arguments: [I] lDefs: list of dictionnaries containing the data definitions
as defined in metro_config.py
[I] dReadHandlers: dictionnay of read handler.
key = data type name (ex: INTEGER),
value = handler (ex: toolbox.metro_dom2metro_handler.read_integer)
[I] domDOM: DOM in which data has to be extracted.
[I] sXPath: Path of the node containing all the subnodes
(header, prediction, etc.)
......@@ -133,7 +136,7 @@ def extract_xpath(lDefs, domDom, sXpath, bIs_matrix=False):
Description: Return one or all the value contained under the xpath in the DOM.
"""
node_items = metro_xml_lib.xpath(metro_xml_lib.get_dom_root(domDom),
sXpath)
......@@ -141,18 +144,18 @@ def extract_xpath(lDefs, domDom, sXpath, bIs_matrix=False):
if len(node_items) > 0:
llExtractedValue = []
for node_item in node_items:
lData = extract_data(lDefs, node_item)
lData = extract_data(lDefs, dReadHandlers, node_item)
llExtractedValue.append(lData)
else:
llExtractedValue = None
else:
if len(node_items) == 1:
node_item = node_items[0]
llExtractedValue = extract_data(lDefs, node_item)
llExtractedValue = extract_data(lDefs, dReadHandlers, node_item)
elif len(node_items) > 1:
llExtractedValue = []
for node_item in node_items:
lData = extract_data(lDefs, node_item)
lData = extract_data(lDefs, dReadHandlers, node_item)
llExtractedValue.append(lData)
else:
llExtractedValue = None
......@@ -160,12 +163,15 @@ def extract_xpath(lDefs, domDom, sXpath, bIs_matrix=False):
return llExtractedValue
def extract_data(lDefs, nodeItems):
def extract_data(lDefs, dReadHandlers, nodeItems):
"""
Name: extract_data
Arguments: [I] lDefs: list of dictionary containing the
definition of XML element in metro_config.py
[I] dReadHandlers: dictionnay of read handler.
key = data type name (ex: INTEGER),
value = handler (ex: toolbox.metro_dom2metro_handler.read_integer)
[I] nodeItems: fecth the elements in theses nodes
Output: lData :: list of values, one per definition in lDefs.
......@@ -173,7 +179,6 @@ def extract_data(lDefs, nodeItems):
Description: Extract the data from one node containing more nodes.
"""
lData = []
# Retrieve the informations about the data type
......@@ -189,23 +194,15 @@ def extract_data(lDefs, nodeItems):
sTag = dDef['XML_TAG']
sData_type_name = dDef['DATA_TYPE']
sReadHandler = get_handler('READ', dDef, dData_type)
sImportHandlerCode = get_handler_import_code(sReadHandler)
exec sImportHandlerCode
if dData_type[sData_type_name].has_key('CHILD'):
# Construction of instruction doing the function call that will
# extract the data containing a list of "sub-data"
nodeTmp = metro_xml_lib.xpath(nodeItems,
sTag)
nodeTmp = metro_xml_lib.xpath(nodeItems,sTag)
lChildList = dData_type[sData_type_name]['CHILD']
sCode = "data = " + sReadHandler + "(lChildList,nodeTmp)"
data = dReadHandlers[sData_type_name](lChildList,nodeTmp,dReadHandlers)
else:
# Construction of instruction doing the function call that will
# extract the data
sCode = "data = " + sReadHandler + "(sTag,nodeItems)"
# extract the data
data = dReadHandlers[sData_type_name](sTag,nodeItems)
exec sCode
lData.append(data)
# If there is only one definition of data (lDefs), it is an "array".
......@@ -282,7 +279,7 @@ def write_to_file( domDoc, sFilename ):
metro_xml_lib.write_xml_file(domDoc, sFilename)
def create_node_tree_from_dict( domDoc, nodeParent, lDefs, dData ):
def create_node_tree_from_dict( domDoc, nodeParent, lDefs, dWriteHandlers, dData ):
# Retrieve the informations on the data types
dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
......@@ -297,28 +294,19 @@ def create_node_tree_from_dict( domDoc, nodeParent, lDefs, dData ):
sXml_tag = dDef['XML_TAG']
sData_type_name = dDef['DATA_TYPE']
sWriteHandler = get_handler('WRITE', dDef, dData_type)
sImportHandlerCode = get_handler_import_code(sWriteHandler)
exec sImportHandlerCode
if dData_type[sData_type_name].has_key('CHILD'):
# Construction of instruction doing the function call that will
# create the node containing a list of "sub-node"
lChildList = dData_type[sData_type_name]['CHILD']
sCode = "nodeData = " + sWriteHandler + \
"(sXml_tag,lChildList,dData[sTag])"
nodeData = dWriteHandlers[sData_type_name](sXml_tag,lChildList,dData[sTag],dWriteHandlers)
else:
# Construction of instruction doing the function call that will
# create the node
sCode = "nodeData = " + sWriteHandler + \
"(sXml_tag,dData[sTag])"
exec sCode
nodeData = dWriteHandlers[sData_type_name](sXml_tag,dData[sTag])
append_child(nodeParent, nodeData)
def create_node_tree_from_matrix( domDoc, nodeParent, sPrediction_xpath,
lDefs, metro_data_object, npMatrix ):
lDefs, dWriteHandlers, metro_data_object, npMatrix ):
"""
Each prediction will be contained in a node that will have the name
given by sPrediction_xpath.
......@@ -345,10 +333,6 @@ def create_node_tree_from_matrix( domDoc, nodeParent, sPrediction_xpath,
sXml_tag = dDef['XML_TAG']
sData_type_name = dDef['DATA_TYPE']
sWriteHandler = get_handler('WRITE', dDef)
sImportHandlerCode = get_handler_import_code(sWriteHandler)
exec sImportHandlerCode
# Extraction of the data from the matrix
lIndexList = metro_data_object.index_of_matrix_col(sTag)
if not metro_data_object.is_multi_col(sTag):
......@@ -364,17 +348,12 @@ def create_node_tree_from_matrix( domDoc, nodeParent, sPrediction_xpath,
val.append(npData[i])
if dData_type[sData_type_name].has_key('CHILD'):
# Construction of instruction doing the function call that will
# create the node containing a list of "sub-node"
lChildList = dData_type[sData_type_name]['CHILD']
sCode = "nodeData = " + sWriteHandler + \
"(sXml_tag,lChildList,val)"
nodeData = dWriteHandlers[sData_type_name](sXml_tag,lChildList,val,dWriteHandlers)
else:
# Construction of instruction doing the function call that will
# create the node
sCode = "nodeData = " + sWriteHandler + "(sXml_tag,val)"
exec sCode
nodeData = dWriteHandlers[sData_type_name](sXml_tag,val)
append_child(nodePrediction, nodeData)
......@@ -405,12 +384,3 @@ def get_handler(sHandlerType, dDefData, dAllDefData=None):
return sHandler
def get_handler_import_code(sHandler):
# Creation of code needed to import the module in which we can find
# the function used to create the node.
lFunctionPart = string.split(sHandler,'.')
sFunction_module = string.join(lFunctionPart[:-1],".")
sCode = "import " +sFunction_module
return sCode
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment