Vispack restructuring (part 3, removing Tixi functions)

Former-commit-id: d45edbfffb517a890050f65ed541dabd9e53deb2

kadmos/graph/mixin_vispack.py

 # Imports
 import json
 import os
-import warnings
 import shutil
 import logging
 import tempfile
@@ -9,11 +8,11 @@ import progressbar
 
 import kadmos.vispack as vispack
 
-from kadmos.external.TIXI_2_2_4.additional_tixi_functions import get_element_details
-from kadmos.external.TIXI_2_2_4.tixiwrapper import Tixi
+from lxml import etree
 
-from ..utilities.xml import get_element_dict, merge
-from ..utilities.general import export_as_json, make_camel_case, get_list_entries, format_string_for_d3js
+from ..utilities.xml import get_element_details, recursively_unique_attribute, merge
+from ..utilities.general import export_as_json, make_camel_case, get_list_entries, format_string_for_d3js, \
+    get_element_dict
 
 
 # Settings for the logger
@@ -21,6 +20,8 @@ logger = logging.getLogger(__name__)
 
 
 class VispackMixin(object):
+    # TODO: The structure of this Mixin can be improved.
+    # TODO: Maybe the functions should be splitted into smaller sub functions.
 
     def vispack_add(self, file_path, mpg=None, function_order=None, reference_file=None, compress=False,
                     remove_after_compress=True, replacement_index=0):
@@ -447,14 +448,11 @@ class VispackMixin(object):
         logger.debug('Creating variableTree_dataschema.json...')
         variable_tree_dataschema = dict()
         if reference_file:
-            # Open XML with Tixi
-            tixi = Tixi()
-            tixi.openDocument(reference_file)
+            # Parse reference XML
+            reference_xml = etree.parse(reference_file)
             # Check validity of the CPACS file
-            try:
-                tixi.uIDCheckDuplicates()
-            except:
-                warnings.warn('WARNING: Reference file ' + reference_file + ' contains UID duplicates.')
+            # noinspection PyUnusedLocal
+            reference_valid = recursively_unique_attribute(reference_xml)
         for key in full_graph:
             if key is not 'attributes' and key is not coordinator_str:
                 if self.node[key]['category'] == 'variable':
@@ -465,7 +463,9 @@ class VispackMixin(object):
                             uidpath = self.node[key]['related_to_schema_node']
                         else:
                             uidpath = key
-                        var_value, var_dim = get_element_details(tixi, uidpath)
+                        # Get element details
+                        # noinspection PyUnboundLocalVariable
+                        var_value, var_dim = get_element_details(reference_xml, uidpath)
                     else:
                         var_value = 'unknown'
                         var_dim = None

kadmos/utilities/general.py

@@ -205,7 +205,7 @@ def test_attr_cond(attr_value, operator, test_value):
         return True if attr_value in test_value else False
 
 
-def export_as_json(data, filename, indent=None, sort_keys=True, cwd=None):
+def export_as_json(data, filename, indent=4, sort_keys=True, cwd=None):
     """
     Function to export a data object to a json file.
 
@@ -495,3 +495,38 @@ def get_schema(version):
     schema = etree.XMLSchema(etree.XML(schema_string))
 
     return schema
+
+
+def get_element_dict(xpath, var_value=None, var_dim=None, include_reference_data=False):
+    """
+    Function to create a D3.js-type dictionary for a nested tree based on an xpath.
+
+    :param xpath: xpath for the element
+    :param var_value: value of the element in a reference file
+    :param var_dim: dimension of the element in a reference file
+    :param include_reference_data: setting on whether reference data should be include in the path
+    :return: nested dictionary
+    """
+
+    # Make tree dictionary
+    xpath_list = xpath.split('/')[1:]
+    xpath_list.reverse()
+    max_depth = len(xpath_list) - 1
+
+    for idx, element in enumerate(xpath_list):
+        if idx == 0:
+            if include_reference_data:
+                element_dict = dict(name=element, level=max_depth - idx, type='variable',
+                                    value=var_value, dimension=var_dim)
+            else:
+                element_dict = dict(name=element, level=max_depth - idx, type='variable')
+        else:
+            if idx != max_depth:
+                # TODO: Should this not be a different type? Like group?
+                # noinspection PyUnboundLocalVariable
+                element_dict = dict(name=element, level=max_depth - idx, type='variable', children=[element_dict])
+            else:
+                # noinspection PyUnboundLocalVariable
+                element_dict = dict(name=element, level=max_depth - idx, children=[element_dict])
+    # noinspection PyUnboundLocalVariable
+    return element_dict

kadmos/utilities/xml.py

 import re
 import ast
+import logging
 
 from lxml.etree import Element, SubElement
 
 from general import make_camel_case, unmake_camel_case, string_eval
 
 
+# Settings for the logger
+logger = logging.getLogger(__name__)
+
+
 # noinspection PyPep8Naming, PyDefaultArgument
 def Child(parent, tag, content='', attrib={}, **extra):
     """
@@ -120,19 +125,6 @@ def ChildGroup(graph, attr_name, attr_value, data_list):
     return element
 
 
-def recursively_empty(e):
-    """
-    Utility function to check recursively if a ElementTree object is empty.
-
-    :param e: Input ElementTree object
-    :return: Result of the check
-    """
-
-    if e.text:
-        return False
-    return all((recursively_empty(c) for c in e.iterchildren()))
-
-
 def recursively_stringify(tree):
     """
     Utility function to recursively stringify a ElementTree object (for file comparison).
@@ -209,44 +201,120 @@ def recursively_dictify(element, key_dict={}):
     return dictionary
 
 
-def get_element_dict(xpath, var_value=None, var_dim=None, include_reference_data=False):
+def recursively_empty(element):
     """
-    Function to create a D3.js-type dictionary for a nested tree based on an xpath.
-
-    :param xpath: xpath for the element
-    :type xpath: str
-    :param var_value: value of the element in a reference file
-    :type var_value: str
-    :param var_dim: dimension of the element in a reference file
-    :type var_dim: str
-    :param include_reference_data: setting on whether reference data should be include in the path
-    :type include_reference_data: bool
-    :return: nested dictionary
-    :rtype: dict
+    Utility function to check recursively if a ElementTree object is empty.
+
+    :param element: Input ElementTree object
+    :return: Result of the check
+    """
+
+    if element.text:
+        return False
+    return all((recursively_empty(c) for c in element.iterchildren()))
+
+
+def recursively_unique_attribute(element, attribute='uID'):
+    """
+    Utility function to check recursively if the values of an attribute of an ElementTree object are unique.
+
+    :param element: Input ElementTree object
+    :param attribute: Name of the attribute being checked for uniqueness
+    :return: Result of the check
     """
 
-    # Make tree dictionary
-    xpath_list = xpath.split('/')[1:]
-    xpath_list.reverse()
-    max_depth = len(xpath_list) - 1
+    attribute_list = [e.get(attribute) for e in element.findall('.//*[@' + attribute + ']')]
+    attribute_list_unique = list(set(attribute_list))
+    result = len(attribute_list) == len(attribute_list_unique)
+    if not result:
+        duplicate_list = ['"'+attribute+'"' for attribute in attribute_list_unique
+                          if attribute_list.count(attribute) > 1]
+        logger.warning('There are several attributes with the same uIDs. The (reference) file is not valid. '
+                       'The duplicate uIDs are: ' + ', '.join(duplicate_list))
+    return result
 
-    for idx, element in enumerate(xpath_list):
-        if idx == 0:
-            if include_reference_data:
-                element_dict = dict(name=element, level=max_depth - idx, type='variable',
-                                    value=var_value, dimension=var_dim)
+
+def get_xpath_from_uxpath(tree, uxpath):
+    """
+    Utility function to determine the XPath belonging to a UXPath for a given ElementTree object.
+
+    :param tree: ElementTree object used for finding the XPath
+    :param uxpath: UXPath
+    :return: XPath
+    """
+
+    # Determine the element uxpath
+    xpath_elements = uxpath.split('/')[1:]
+    xpath_elements_rev = xpath_elements[::-1]
+    uid = ''
+
+    for idx, el in enumerate(xpath_elements_rev):
+        if '[' in el and ']' in el:
+            # Determine what's between the brackets
+            locator = el[el.find('[') + 1:el.rfind(']')]
+            if not locator.isdigit():
+                uid = locator
+                break
+
+    if len(uid) > 0:
+        xelement = tree.getroot().find('.//*[@uID="' + uid + '"]')
+        if xelement is not None:
+            xpath = tree.getpath(xelement)
+            if xpath:
+                if idx != 0:
+                    xpath = xpath + '/' + '/'.join(xpath_elements_rev[:idx][::-1])
             else:
-                element_dict = dict(name=element, level=max_depth - idx, type='variable')
+                xpath = None
         else:
-            if idx != max_depth:
-                # TODO: Should this not be a different type? Like group?
-                # noinspection PyUnboundLocalVariable
-                element_dict = dict(name=element, level=max_depth - idx, type='variable', children=[element_dict])
+            xpath = None
+    else:
+        xpath = uxpath
+
+    # TODO: Check whether the following is actually needed
+    # In the following empty uIDs will be removed (because the XPath is not valid otherwise).
+    if xpath is not None:
+        xpath = xpath.replace('[]', '')
+
+    return xpath
+
+
+def get_element_details(tree, uxpath):
+    """
+    Function to determine the value and dimension of an UXPath element in a reference file.
+
+    :param tree: ElementTree object used for finding the XPath
+    :param uxpath: UXPath
+    :return: element value and dimension
+    """
+
+    # Input assertions
+    assert isinstance(uxpath, basestring)
+
+    # Determine the element uxpath
+    xpath = get_xpath_from_uxpath(tree, uxpath)
+
+    if xpath:
+        try:
+            values = tree.getroot().xpath(xpath)
+            if len(values) > 1:
+                logger.warning('The XPath '+xpath+' is not unique in the reference file. Only the first value is used.')
+            value = values[0].text
+            separators = value.count(';')
+            if separators == 0:
+                dim = 1
             else:
-                # noinspection PyUnboundLocalVariable
-                element_dict = dict(name=element, level=max_depth - idx, children=[element_dict])
-    # noinspection PyUnboundLocalVariable
-    return element_dict
+                if value[-1] == ';':
+                    dim = separators
+                else:
+                    dim = separators + 1
+        except (IndexError, AttributeError):
+            value = 'The XPath "' + xpath + '" could not be found in the reference file.'
+            dim = None
+    else:
+        value = 'The XPath with UXPath "' + uxpath + '" could not be found in the reference file.'
+        dim = None
+
+    return value, dim
 
 
 def merge(a, b):