Added check_database() to perform database checks without the need to load()...

Added check_database() to perform database checks without the need to load() and include option in both to keep_running in case of an error.


Former-commit-id: cb75a3bd56f53bd09f81d427ecc3ef792482dc42

examples/scripts/ssbj/CMDOWS/RCG.xml

@@ -3,14 +3,14 @@
   <header>
     <creator>Lukas Mueller</creator>
     <description>RCG CMDOWS file of the super-sonic business jet test case optimization problem</description>
-    <timestamp>2018-03-17T11:44:59.526000</timestamp>
+    <timestamp>2018-03-28T17:18:10.460000</timestamp>
     <fileVersion>0.1</fileVersion>
     <cmdowsVersion>0.7</cmdowsVersion>
     <updates>
       <update>
         <modification>KADMOS export of a repository connectivity graph (RCG).</modification>
         <creator>Lukas Mueller</creator>
-        <timestamp>2018-03-17T11:44:59.526000</timestamp>
+        <timestamp>2018-03-28T17:18:10.460000</timestamp>
         <fileVersion>0.1</fileVersion>
         <cmdowsVersion>0.7</cmdowsVersion>
       </update>

kadmos/graph/graph_kadmos.py

@@ -974,7 +974,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin):
     #                                             LOAD METHODS                                                         #
     # ---------------------------------------------------------------------------------------------------------------- #
 
-    def load_cmdows(self, cmdows, check_list, ignore_modes=False):
+    def load_cmdows(self, cmdows, check_list, ignore_modes=False, keep_running=False):
 
         # Create organization node
         self._load_cmdows_header(cmdows)
@@ -983,7 +983,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin):
         self._load_cmdows_parameters(cmdows)
 
         # Create function nodes
-        self._load_cmdows_executables(cmdows, check_list, ignore_modes=ignore_modes)
+        self._load_cmdows_executables(cmdows, check_list, ignore_modes=ignore_modes, keep_running=keep_running)
 
         # Create architecture element nodes
         if hasattr(self, '_load_cmdows_architecture_elements'):
@@ -1004,12 +1004,12 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin):
 
         return
 
-    def _load_cmdows_executables(self, cmdows, check_list, ignore_modes=False):
+    def _load_cmdows_executables(self, cmdows, check_list, ignore_modes=False, keep_running=False):
 
-        self._load_cmdows_competences(cmdows, check_list, ignore_modes=ignore_modes)
+        self._load_cmdows_competences(cmdows, check_list, ignore_modes=ignore_modes, keep_running=keep_running)
         self._load_cmdows_equations(cmdows)
 
-    def _load_cmdows_competences(self, cmdows, check_list=None, ignore_modes=False):
+    def _load_cmdows_competences(self, cmdows, check_list=None, ignore_modes=False, keep_running=False):
 
         inputs_list = []
         outputs_list = []
@@ -1061,41 +1061,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin):
 
         io_list = inputs_list+outputs_list
 
-        root_check = False
-        leaf_check = False
-        xsd_check = False
-        # Check which checks need to be performed
-        if check_list:
-            checks_detected = 0
-            if 'consistent_root' in check_list:
-                root_check = True
-                checks_detected += 1
-            if 'invalid_leaf_elements' in check_list:
-                leaf_check = True
-                checks_detected += 1
-            if 'schemas' in check_list:
-                xsd_check = True
-                checks_detected += 1
-            # Check if checks were added correctly
-            if checks_detected is not len(check_list):
-                logger.warning("Invalid check_list element input detected. Valid options: 'consistent_root',"
-                               " 'invalid_leaf_elements' or 'schemas'")
-
-        parse_check(io_list)
-
-        # Check consistency of root
-        if root_check:
-            _check_roots(io_list)
-
-        # Check for invalid leaf elements
-        if leaf_check:
-            leafs = _get_leafs(io_list)
-            _check_leafs(io_list, leafs)
-
-        # Check file against XSD schema
-        if xsd_check:
-            schema_path = _get_data_schema(os.path.split(io_list[0][0])[0])
-            _validate_file_against_schema(io_list[0][0], schema_path)
+        _perform_check_list(io_list, check_list, keep_running=keep_running)
 
         for input_el in inputs_list:
             inputs = _read_io_xml_file(input_el[0], input_el[1].findtext('modeID'), ignore_modes=ignore_modes)
@@ -4249,7 +4215,8 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin):
 
 
 # noinspection PyUnboundLocalVariable
-def load(file_name, file_type=None, check_list=None, file_check_critical=True, source_folder=None, ignore_modes=False):
+def load(file_name, file_type=None, check_list=None, file_check_critical=True, source_folder=None, ignore_modes=False,
+         keep_running=False):
     """Function to load a KadmosGraph from a file.
 
     Different input file types are implemented. They are listed in the following.
@@ -4302,7 +4269,7 @@ def load(file_name, file_type=None, check_list=None, file_check_critical=True, s
     # TODO: Load also header info and/or metadata of files
     logger.info('Loading the ' + file_type.upper() + ' file ' + file_path + '...')
     if file_type == 'cmdows':
-        graph, mpg = _load_cmdows(file_path, check_list, ignore_modes=ignore_modes)
+        graph, mpg = _load_cmdows(file_path, check_list, ignore_modes=ignore_modes, keep_running=keep_running)
     elif file_type == 'kdms':
         graph = _load_kdms(file_path)
     elif file_type == 'graphml':
@@ -4333,6 +4300,130 @@ def load(file_name, file_type=None, check_list=None, file_check_critical=True, s
         return graph, mpg
 
 
+def check_database(file_name, source_folder=None, check_list=None, keep_running=False):
+    """Method to check the XML-files inside the database for parse errors, inconsistent roots, invalid leaf elements
+     and schema inconsistencies.
+
+     :param file_name: name of the cmdows file.
+     :type file_name: str
+     :param file_type: file_type
+     :type file_type: str
+     :param source_folder: source folder which contains the input file
+     :type source_folder: str
+     :param check_list: list with checks to be performed on in case of CMDOWS file with in- and output XML files.
+     check options:
+     'consistent_root': to check XML input/output files have the same, single root element.
+     'invalid_leaf_elements': to check XML input/output files for invalid leaf elements.
+     'schemas': to check XML input/output files against a schema file (XSD) in case of CMDOWS file in combination
+     with in- and output XML files.
+     :type check_list: list
+
+     :return: graph loaded from file
+     :rtype: KadmosGraph
+     """
+
+    # Try to detect the file type automatically
+    if file_name.lower().endswith(tuple(file_extensions)):
+        file_type = file_types[file_extensions.index(os.path.splitext(file_name)[1].lower()[1:])]
+    else:
+        raise IOError('The file type of ' + file_name + ' could not be detected automatically. '
+                                                        'Please provide it with the file_type input argument.')
+
+    # Check if the file type is valid
+    file_type = file_type.lower()
+    if file_type == 'cmdows':
+        pass
+    else:
+        raise IOError('The specified file type ' + file_type + ' cannot be checked. Please use the file type: cmdows.')
+
+    # Get path to file
+    if source_folder is None:
+        file_path = file_name
+    else:
+        file_path = os.path.join(source_folder, file_name)
+
+    cmdows = CMDOWS(file_path=file_path)
+    if not cmdows.check():
+        logger.warning('The CMDOWS file ' + file_path + ' has some flaws. You may use a previous version of KADMOS to'
+                                                        ' ensure proper loading of the CMDOWS file.')
+
+    # Parse
+    cmdows = etree.parse(file_path, parser).getroot()
+
+    # Clean
+    cmdows.clean()
+
+    inputs_list = []
+    outputs_list = []
+
+    for function in cmdows.findall('executableBlocks/designCompetences/designCompetence'):
+        if not function.findall('inputs/input'):
+            # Determine assumed input file location (same folder as CMDOWS file)
+            input_file_path = os.path.join(os.path.split(os.path.normpath(function.base))[0],
+                                           function.findtext('ID') + '-input.xml').replace('file:' + os.path.sep,
+                                                                                           '')
+        if os.path.isfile(input_file_path):
+            inputs_list.append([input_file_path, function])
+        else:
+            logger.warning('Could not find inputs for function: ' + function.get('uID'))
+
+        if not function.findall('inputs/input'):
+            # Determine assumed input file location (same folder as CMDOWS file)
+            output_file_path = os.path.join(os.path.split(os.path.normpath(function.base))[0],
+                                            function.findtext('ID') + '-output.xml').replace('file:' + os.path.sep,
+                                                                                             '')
+        if os.path.isfile(input_file_path):
+            outputs_list.append([output_file_path, function])
+        else:
+            logger.warning('Could not find outputs for function: ' + function.get('uID'))
+
+    io_list = inputs_list + outputs_list
+
+    _perform_check_list(io_list, check_list, keep_running=keep_running)
+
+    return
+
+
+def _perform_check_list(io_list, check_list, keep_running=False):
+
+    root_check = False
+    leaf_check = False
+    xsd_check = False
+    # Check which checks need to be performed
+    if check_list:
+        checks_detected = 0
+        if 'consistent_root' in check_list:
+            root_check = True
+            checks_detected += 1
+        if 'invalid_leaf_elements' in check_list:
+            leaf_check = True
+            checks_detected += 1
+        if 'schemas' in check_list:
+            xsd_check = True
+            checks_detected += 1
+        # Check if checks were added correctly
+        if checks_detected is not len(check_list):
+            logger.warning("Invalid check_list element input detected. Valid options: 'consistent_root',"
+                           " 'invalid_leaf_elements' or 'schemas'")
+
+    parse_check(io_list)
+
+    # Check consistency of root
+    if root_check:
+        _check_roots(io_list, keep_running=keep_running)
+
+    # Check for invalid leaf elements
+    if leaf_check:
+        leafs = _get_leafs(io_list)
+        _check_leafs(io_list, leafs, keep_running=keep_running)
+
+    # Check file against XSD schema
+    if xsd_check:
+        schema_path = _get_data_schema(os.path.split(io_list[0][0])[0])
+        _validate_file_against_schema(io_list[0][0], schema_path, keep_running=keep_running)
+
+    return
+
 def _load_kdms(file_path):
 
     # Parse
@@ -4370,7 +4461,7 @@ def _load_graphml(file_path):
 
 
 # noinspection PyUnboundLocalVariable
-def _load_cmdows(file_path, check_list=None, ignore_modes=False):
+def _load_cmdows(file_path, check_list=None, ignore_modes=False, keep_running=False):
 
     from graph_data import RepositoryConnectivityGraph, FundamentalProblemGraph, MdaoDataGraph
 
@@ -4400,7 +4491,7 @@ def _load_cmdows(file_path, check_list=None, ignore_modes=False):
         IOError('The CMDOWS file ' + file_path + ' is missing basic elements and cannot be loaded.')
 
     # Load the graph (and MPG in case this one is provided)
-    mpg = graph.load_cmdows(cmdows, check_list, ignore_modes=ignore_modes)
+    mpg = graph.load_cmdows(cmdows, check_list, ignore_modes=ignore_modes, keep_running=keep_running)
 
     # Clean up graph
     try:
@@ -4628,7 +4719,7 @@ def _get_data_schema(kb_dir):
     return os.path.join(kb_dir, data_schema)
 
 
-def _validate_file_against_schema(file_path, schema_path):
+def _validate_file_against_schema(file_path, schema_path, keep_running=False):
     """
     (PRIVATE) Check the read-write XML file in the knowledge base against the XML Schema.
     Argument is list/tuple of nodes to ignore in validation. Root node can not be ignored.
@@ -4653,11 +4744,12 @@ def _validate_file_against_schema(file_path, schema_path):
         for error in xmlschema.error_log:
             logger.debug('ERROR ON LINE {} in file {}: {}'.format(error.line, os.path.split(file_path)[1],
                                                                   error.message.encode("utf-8")))
-        raise AssertionError('The provided file {} is not valid w.r.t. the schema {} (set logger to debug for '
-                             'additional info).'.format(os.path.split(file_path)[1], os.path.split(schema_path)[1]))
+        if not keep_running:
+            raise AssertionError('The provided file {} is not valid w.r.t. the schema {} (set logger to debug for '
+                                 'additional info).'.format(os.path.split(file_path)[1], os.path.split(schema_path)[1]))
     return
 
-def _check_roots(io_list):
+def _check_roots(io_list, keep_running=False):
     """
     This function checks if there are inconsistent roots in the XML input and output files.
     :rtype: Warning
@@ -4677,8 +4769,9 @@ def _check_roots(io_list):
                     logger.debug('Inconsistent root of element found in XML file {}'.format(file_name))
                     logger.debug('Inconsistent root: {}'.format(current_root.tag))
                     logger.debug('Inconsistent w.r.t. standard root: {}'.format(previous_root.tag))
-                    raise IOError('Inconsistent root in file {} (root found: {}, expected root: {})'
-                                  .format(file_name, current_root.tag, previous_root.tag))
+                    if not keep_running:
+                        raise IOError('Inconsistent root in file {} (root found: {}, expected root: {})'
+                                      .format(file_name, current_root.tag, previous_root.tag))
             previous_root = current_root
             first_file = False
     return
@@ -4718,7 +4811,7 @@ def _get_leafs(io_list):
     return leafs
 
 
-def _check_leafs(io_list, leafs):
+def _check_leafs(io_list, leafs, keep_running=False):
     """
     This function checks if there are elements in other files with the same element that still have child elements.
     :rtype: Warning
@@ -4739,13 +4832,14 @@ def _check_leafs(io_list, leafs):
                         logger.debug('Invalid leaf element path {}'.format(leaf[1]))
                         logger.debug('Reason for invalidity: longer path {} found in file {}'.format(path, file_name))
                         leafs.remove(leaf)
-                        raise AssertionError('Invalid leaf element found in XML file {}\n'.format(leaf[2]) +
-                                             'Invalid leaf element path {}\n'.format(leaf[1]) +
-                                             'Reason for invalidity: longer path {} found in file {}'.format(path, file_name))
+                        if not keep_running:
+                            raise AssertionError('Invalid leaf element found in XML file {}\n'.format(leaf[2]) +
+                                                 'Invalid leaf element path {}\n'.format(leaf[1]) +
+                                                 'Reason for invalidity: longer path {} found in file {}'.format(path, file_name))
     return
 
 
-def parse_check(io_list):
+def parse_check(io_list, keep_running=False):
     logger.info('Checking for parsing errors...')
     for entry in io_list:
         file_path=entry[0]
@@ -4754,7 +4848,7 @@ def parse_check(io_list):
     return
 
 
-def _try_parsing(file_path):
+def _try_parsing(file_path, keep_running=False):
     try:
         tree = etree.parse(file_path, parser)
     except etree.XMLSyntaxError as e:
@@ -4763,3 +4857,5 @@ def _try_parsing(file_path):
         logger.warning('ERROR: {}'.format(e))
         raise IOError('Could not parse XML file {}. Check file structure.'.format(os.path.split(file_path)[1]))
     return tree
+
+