metro_xml.py 13 KB
Newer Older
1
2
# METRo : Model of the Environment and Temperature of Roads
# METRo is Free and is proudly provided by the Government of Canada
Miguel Tremblay's avatar
Miguel Tremblay committed
3
# Copyright (C) Her Majesty The Queen in Right of Canada, Environment Canada, 2006
Sasa Zhang's avatar
Sasa Zhang committed
4
#
5
#  Questions or bugs report: metro@ec.gc.ca
6
#  METRo repository: https://framagit.org/metroprojects/metro
7
#  Documentation: https://framagit.org/metroprojects/metro/wikis/home
8
9
10
11
12
13
14
#
# Code contributed by:
#  Miguel Tremblay - Canadian meteorological center
#  Francois Fortin - Canadian meteorological center
#
#  $LastChangedDate$
#  $LastChangedRevision$
Sasa Zhang's avatar
Sasa Zhang committed
15
###################################################################################
16
17
18
19
20
21
22
23
24
25
26
27
28
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
Sasa Zhang's avatar
Sasa Zhang committed
29

30

Miguel Tremblay's avatar
Miguel Tremblay committed
31
"""
Sasa Zhang's avatar
Sasa Zhang committed
32
33
34
35
36
37
38
39
40
41
42
    Name       : metro_xml.py

    Description: Methods to create and manipulate xml DOM

    Work on    :

    Notes      :

    Author     : Francois Fortin

    Date       : 2004
Miguel Tremblay's avatar
Miguel Tremblay committed
43
"""
Miguel Tremblay's avatar
Miguel Tremblay committed
44

45

Sasa Zhang's avatar
Sasa Zhang committed
46
import sys
47
48
import metro_config
import metro_logger
49
import metro_error
50
51
from toolbox import metro_util

Sasa Zhang's avatar
Sasa Zhang committed
52

53
54
_ = metro_util.init_translation('metro_xml')

Sasa Zhang's avatar
Sasa Zhang committed
55
56

def init(sMetro_xml_lib=""):
57
58
59
60
    global metro_xml_lib
    if sMetro_xml_lib != "":
        try:
            metro_xml_lib = __import__(sMetro_xml_lib)
61
        except metro_error.Metro_import_error:
Sasa Zhang's avatar
Sasa Zhang committed
62
63
            sMessage = _("Fatal error! Can't import '%s' xml library") % sMetro_xml_lib
            metro_logger.print_init_message(metro_logger.LOGGER_INIT_ERROR, sMessage)
64
        else:
Sasa Zhang's avatar
Sasa Zhang committed
65
66
            metro_logger.print_init_message(metro_logger.LOGGER_INIT_SUCCESS,
                                            _("XML library '%s' will be use.") % sMetro_xml_lib)
67
            metro_xml_lib = __import__(sMetro_xml_lib)
Sasa Zhang's avatar
Sasa Zhang committed
68

69
    else:
Sasa Zhang's avatar
Sasa Zhang committed
70
        metro_logger.print_init_message(metro_logger.LOGGER_INIT_MESSAGE, _("Auto configure METRo XML library"))
71
        try:
Sasa Zhang's avatar
Sasa Zhang committed
72
73
74
75
76
77
            metro_util.test_import("toolbox.metro_xml_libxml2")
        except metro_error.Metro_import_error as inst:
            sMessage = _("Fatal error! No METRo XML library can be use. ") + \
                       _("\nMETRo need one of the following XML library ") + \
                       _("installed on the system.\nSupported library:") + "python-libxml2"
            metro_logger.print_init_message(metro_logger.LOGGER_INIT_ERROR, sMessage)
Miguel Tremblay's avatar
   
Miguel Tremblay committed
78
            sys.exit(3)
79
        else:
Sasa Zhang's avatar
Sasa Zhang committed
80
81
            metro_logger.print_init_message(metro_logger.LOGGER_INIT_SUCCESS, _("metro_xml_libxml2 will be used."))
            metro_xml_libxml2 = metro_util.import_name('toolbox', "metro_xml_libxml2")
82
83
84
            metro_xml_lib = metro_xml_libxml2.Metro_xml_libxml2()
    metro_xml_lib.start()

Sasa Zhang's avatar
Sasa Zhang committed
85

86
87
88
def stop():
    sMessage = metro_xml_lib.stop()
    if sMessage:
Sasa Zhang's avatar
Sasa Zhang committed
89
        metro_logger.print_message(metro_logger.LOGGER_MSG_WARNING, sMessage)
90

Sasa Zhang's avatar
Sasa Zhang committed
91
92

def validate_string(sXml_content):
93
94
95
    if metro_xml_lib.get_name() == "Metro_xml_libxml2":
        metro_xml_lib.validate_xml_string(sXml_content)
    else:
96
        sMessage = _("Validation is only supported with metro_xml_libxml2")
97
        if metro_logger.is_initialised():
Sasa Zhang's avatar
Sasa Zhang committed
98
            metro_logger.print_message(metro_logger.LOGGER_MSG_WARNING, sMessage)
99
        else:
Sasa Zhang's avatar
Sasa Zhang committed
100
            metro_logger.print_init_message(metro_logger.LOGGER_INIT_ERROR, sMessage)
101
102


Sasa Zhang's avatar
Sasa Zhang committed
103
104
def xpath(domDom, sXpath):
    nodeLeaf = metro_xml_lib.xpath(metro_xml_lib.get_dom_root(domDom), sXpath)
105
    if nodeLeaf:
Sasa Zhang's avatar
Sasa Zhang committed
106
        sRslt = metro_xml_lib.get_node_value(nodeLeaf[0])
107
108
109
110
    else:
        sRslt = None
    return sRslt

Sasa Zhang's avatar
Sasa Zhang committed
111

112
def extract_xpath(lDefs, dReadHandlers, domDom, sXpath, bIs_matrix=False):
Miguel Tremblay's avatar
Miguel Tremblay committed
113
    """
Sasa Zhang's avatar
Sasa Zhang committed
114
115
116
117
118
119
120
121
122
        Name: extract_xpath

        Arguments: [I] lDefs: list of dictionaries containing the data definitions as defined in metro_config.py
                   [I] dReadHandlers: dictionary of read handler.
                                      key = data type name (ex: INTEGER),
                                      value = handler (ex: toolbox.metro_dom2metro_handler.read_integer)
                    [I] domDOM: DOM in which data has to be extracted.
                    [I] sXPath: Path of the node containing all the subnodes (header, prediction, etc.)
                    [I] bIs_matrix: Boolean indicating if there is more than one value to fetch
123

Sasa Zhang's avatar
Sasa Zhang committed
124
        Output:  llExtractedValue : A list of lists containing the expected values.
125

Sasa Zhang's avatar
Sasa Zhang committed
126
127
128
129
130
        Description: Return one or all the value contained under the xpath in the DOM.
    """

    node_items = metro_xml_lib.xpath(metro_xml_lib.get_dom_root(domDom), sXpath)
    if bIs_matrix:
Miguel Tremblay's avatar
Miguel Tremblay committed
131
132
133
        if len(node_items) > 0:
            llExtractedValue = []
            for node_item in node_items:
134
                lData = extract_data(lDefs, dReadHandlers, node_item)
Miguel Tremblay's avatar
Miguel Tremblay committed
135
                llExtractedValue.append(lData)
136
        else:
Miguel Tremblay's avatar
Miguel Tremblay committed
137
            llExtractedValue = None
138
    else:
Miguel Tremblay's avatar
Miguel Tremblay committed
139
140
        if len(node_items) == 1:
            node_item = node_items[0]
141
            llExtractedValue = extract_data(lDefs, dReadHandlers, node_item)
Miguel Tremblay's avatar
Miguel Tremblay committed
142
143
144
        elif len(node_items) > 1:
            llExtractedValue = []
            for node_item in node_items:
145
                lData = extract_data(lDefs, dReadHandlers, node_item)
Miguel Tremblay's avatar
Miguel Tremblay committed
146
                llExtractedValue.append(lData)
147
        else:
Miguel Tremblay's avatar
Miguel Tremblay committed
148
149
150
            llExtractedValue = None
    return llExtractedValue

Sasa Zhang's avatar
Sasa Zhang committed
151

152
def extract_data(lDefs, dReadHandlers, nodeItems):
Miguel Tremblay's avatar
Miguel Tremblay committed
153
    """
Sasa Zhang's avatar
Sasa Zhang committed
154
        Name: extract_data
Miguel Tremblay's avatar
Miguel Tremblay committed
155

Sasa Zhang's avatar
Sasa Zhang committed
156
157
158
159
160
        Arguments:  [I] lDefs: list of dictionary containing the definition of XML element in metro_config.py
                    [I] dReadHandlers: dictionary of read handler.
                                       key = data type name (ex: INTEGER),
                                       value = handler (ex: toolbox.metro_dom2metro_handler.read_integer)
                    [I] nodeItems: fetch the elements in theses nodes
Miguel Tremblay's avatar
Miguel Tremblay committed
161

Sasa Zhang's avatar
Sasa Zhang committed
162
        Output:   lData :: list of values, one per definition in lDefs.
163

Sasa Zhang's avatar
Sasa Zhang committed
164
        Description: Extract the data from one node containing more nodes.
Miguel Tremblay's avatar
Miguel Tremblay committed
165
    """
166
    lData = []
Sasa Zhang's avatar
Sasa Zhang committed
167
    # Retrieve the information about the data type
168
169
    dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
    dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
Sasa Zhang's avatar
Sasa Zhang committed
170
    dData_type = metro_util.join_dictionaries(dStandard_data_type, dExtended_data_type)
171

Miguel Tremblay's avatar
Miguel Tremblay committed
172
    # For each definitions of item
173
    for dDef in lDefs:
Miguel Tremblay's avatar
Miguel Tremblay committed
174
        # Get the name and the type of the item
175
176
        sTag = dDef['XML_TAG']
        sData_type_name = dDef['DATA_TYPE']
Sasa Zhang's avatar
Sasa Zhang committed
177
178

        if 'CHILD' in dData_type[sData_type_name]:
Miguel Tremblay's avatar
Miguel Tremblay committed
179
            #  extract the data containing a list of "sub-data"
Sasa Zhang's avatar
Sasa Zhang committed
180
            nodeTmp = metro_xml_lib.xpath(nodeItems, sTag)
181
            lChildList = dData_type[sData_type_name]['CHILD']
Sasa Zhang's avatar
Sasa Zhang committed
182
            data = dReadHandlers[sData_type_name](lChildList, nodeTmp, dReadHandlers)
183
        else:
184
            #  extract the data
Sasa Zhang's avatar
Sasa Zhang committed
185
            data = dReadHandlers[sData_type_name](sTag, nodeItems)
186
187
        lData.append(data)

Miguel Tremblay's avatar
Miguel Tremblay committed
188
    # If there is only one definition of data (lDefs), it is an "array".
189
    # In this case, we must extract the "array" from the list [[1,2,3,...,N]]  ==>  [1,2,3,...,N]
190
191
192
193
194
195
    if len(lDefs) == 1:
        if lData:
            lData = lData[0]
    return lData


Sasa Zhang's avatar
Sasa Zhang committed
196
197
def extract_data_from_node(sTag, nodeBranch):
    nodeLeaf = metro_xml_lib.xpath(nodeBranch, sTag + "/text()")
198
    if nodeLeaf:
Sasa Zhang's avatar
Sasa Zhang committed
199
        sRslt = metro_xml_lib.get_node_value(nodeLeaf[0])
200
201
202
203
    else:
        sRslt = None
    return sRslt

Sasa Zhang's avatar
Sasa Zhang committed
204
205

def read_dom(sFilename):
206
207
    return metro_xml_lib.read_dom(sFilename)

Sasa Zhang's avatar
Sasa Zhang committed
208
209

def dom_from_string(sXml):
210
211
    return metro_xml_lib.dom_from_string(sXml)

Sasa Zhang's avatar
Sasa Zhang committed
212
213

def free_dom(domDoc):
214
215
    metro_xml_lib.free_dom(domDoc)

Sasa Zhang's avatar
Sasa Zhang committed
216
217

def create_dom(sDom_root_name):
218
219
    return metro_xml_lib.create_dom(sDom_root_name)

Sasa Zhang's avatar
Sasa Zhang committed
220
221

def get_dom_root(domDoc):
222
223
224
    return metro_xml_lib.get_dom_root(domDoc)


Sasa Zhang's avatar
Sasa Zhang committed
225
def mkdir_xpath(domDoc, nodeBranch, sXpath):
226
    nodeParent = nodeBranch
Sasa Zhang's avatar
Sasa Zhang committed
227
    lNode_name = sXpath.split('/')
228
    for sNode_name in lNode_name:
Miguel Tremblay's avatar
   
Miguel Tremblay committed
229
        nodeChild = metro_xml_lib.create_node(sNode_name)
Sasa Zhang's avatar
Sasa Zhang committed
230
        metro_xml_lib.append_child(nodeParent, nodeChild)
231
232
233
234
        nodeParent = nodeChild
    return nodeParent


Sasa Zhang's avatar
Sasa Zhang committed
235
def cd_xpath(nodeBranch, sXpath):
236
    lChild = metro_xml_lib.xpath(nodeBranch, sXpath)
Sasa Zhang's avatar
Sasa Zhang committed
237
    if lChild:
238
239
240
241
242
        nodeChild = lChild[0]
    else:
        nodeChild = None
    return nodeChild

Sasa Zhang's avatar
Sasa Zhang committed
243
244

def create_node(sNode_name):
245
    return metro_xml_lib.create_node(sNode_name)
246
247


Sasa Zhang's avatar
Sasa Zhang committed
248
249
250
251
252
def create_text_node(sNode_name, sNode_value):
    return metro_xml_lib.create_text_node(sNode_name, sNode_value)


def append_child(nodeParent, nodeChild):
253
254
    metro_xml_lib.append_child(nodeParent, nodeChild)

Sasa Zhang's avatar
Sasa Zhang committed
255
256

def set_attribute(node, sAttributeName, sAttributeValue):
257
258
    metro_xml_lib.set_attribute(node, sAttributeName, sAttributeValue)

259

Sasa Zhang's avatar
Sasa Zhang committed
260
261
def write_to_file(domDoc, sFilename):
    metro_xml_lib.write_xml_file(domDoc, sFilename)
262
263


Sasa Zhang's avatar
Sasa Zhang committed
264
265
def create_node_tree_from_dict(domDoc, nodeParent, lDefs, dWriteHandlers, dData):
    # Retrieve the information on the data types
266
267
    dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
    dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
Sasa Zhang's avatar
Sasa Zhang committed
268
    dData_type = metro_util.join_dictionaries(dStandard_data_type, dExtended_data_type)
269
270

    for dDef in lDefs:
Miguel Tremblay's avatar
Miguel Tremblay committed
271
        # Retrieve the name and the item type
272
273
274
275
        sTag = dDef['NAME']
        sXml_tag = dDef['XML_TAG']
        sData_type_name = dDef['DATA_TYPE']

Sasa Zhang's avatar
Sasa Zhang committed
276
        if 'CHILD' in list(dData_type[sData_type_name].keys()):
Miguel Tremblay's avatar
Miguel Tremblay committed
277
            #  create the node containing a list of "sub-node"
278
            lChildList = dData_type[sData_type_name]['CHILD']
Sasa Zhang's avatar
Sasa Zhang committed
279
            nodeData = dWriteHandlers[sData_type_name](sXml_tag, lChildList, dData[sTag], dWriteHandlers)
280
        else:
Sasa Zhang's avatar
Sasa Zhang committed
281
282
            #  create the node
            nodeData = dWriteHandlers[sData_type_name](sXml_tag, dData[sTag])
283
284
285
        append_child(nodeParent, nodeData)


Sasa Zhang's avatar
Sasa Zhang committed
286
287
def create_node_tree_from_matrix(domDoc, nodeParent, sPrediction_xpath, lDefs,
                                 dWriteHandlers, metro_data_object, npMatrix):
Miguel Tremblay's avatar
Miguel Tremblay committed
288
    """
Sasa Zhang's avatar
Sasa Zhang committed
289
        Each prediction will be contained in a node that will have the name given by sPrediction_xpath.
Miguel Tremblay's avatar
Miguel Tremblay committed
290
    """
Sasa Zhang's avatar
Sasa Zhang committed
291
    # Retrieve the information about the data types
292
293
    dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
    dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
Sasa Zhang's avatar
Sasa Zhang committed
294
    dData_type = metro_util.join_dictionaries(dStandard_data_type, dExtended_data_type)
295

296
    for npData in npMatrix:
Miguel Tremblay's avatar
Miguel Tremblay committed
297
        # If needed, creation of a node to contain the prediction
Sasa Zhang's avatar
Sasa Zhang committed
298
        if sPrediction_xpath is not None and sPrediction_xpath != "":
299
300
301
302
            nodePrediction = mkdir_xpath(domDoc, nodeParent, sPrediction_xpath)
        else:
            nodePrediction = nodeParent
        for dDef in lDefs:
Miguel Tremblay's avatar
Miguel Tremblay committed
303
            # Get the name and the type of the item
304
305
306
307
            sTag = dDef['NAME']
            sXml_tag = dDef['XML_TAG']
            sData_type_name = dDef['DATA_TYPE']

308
            # Extraction of the data from the matrix
309
310
311
312
313
314
315
316
317
318
            lIndexList = metro_data_object.index_of_matrix_col(sTag)
            if not metro_data_object.is_multi_col(sTag):
                # single col
                val = npData[lIndexList[0]]
            else:
                # multi col
                # FFTODO good candidate for optimisation
                val = []
                for i in lIndexList:
                    val.append(npData[i])
Sasa Zhang's avatar
Sasa Zhang committed
319
            if 'CHILD' in dData_type[sData_type_name]:
Miguel Tremblay's avatar
Miguel Tremblay committed
320
                #  create the node containing a list of "sub-node"
321
                lChildList = dData_type[sData_type_name]['CHILD']
Sasa Zhang's avatar
Sasa Zhang committed
322
                nodeData = dWriteHandlers[sData_type_name](sXml_tag, lChildList, val, dWriteHandlers)
323
            else:
Sasa Zhang's avatar
Sasa Zhang committed
324
325
                #  create the node
                nodeData = dWriteHandlers[sData_type_name](sXml_tag, val)
326
            append_child(nodePrediction, nodeData)
327
328


Sasa Zhang's avatar
Sasa Zhang committed
329
330
331
def get_handler(sHandlerType, dDefData, dAllDefData=None):
    if dAllDefData is None:
        # Retrieve the information about the data types
332
333
        dStandard_data_type = metro_config.get_value('XML_DATATYPE_STANDARD')
        dExtended_data_type = metro_config.get_value('XML_DATATYPE_EXTENDED')
Sasa Zhang's avatar
Sasa Zhang committed
334
335
        dAllDefData = metro_util.join_dictionaries(dStandard_data_type, dExtended_data_type)

336
337
338
339
    sTag = dDefData['NAME']
    sXml_tag = dDefData['XML_TAG']
    sData_type_name = dDefData['DATA_TYPE']

Sasa Zhang's avatar
Sasa Zhang committed
340
341
342
343
    if sData_type_name not in list(dAllDefData.keys()):
        sMessage = _("Invalid data_type: (%s) for the following ") % sData_type_name + \
                   _("tag:(%s). Default data type will be used.") % sTag
        metro_logger.print_message(metro_logger.LOGGER_MSG_WARNING, sMessage)
344
        sData_type_name = 'DEFAULT'
Sasa Zhang's avatar
Sasa Zhang committed
345

346
347
348
    # The data type needs the call of a function to create the node
    sHandler = dAllDefData[sData_type_name][sHandlerType]
    return sHandler