From a350dc87c86ff844962f2a1f69be53dbe8f7b0b7 Mon Sep 17 00:00:00 2001 From: imcovangent <I.vanGent@tudelft.nl> Date: Tue, 18 Sep 2018 18:30:10 +0200 Subject: [PATCH] Added automatic isolation of surrogate models for BLISS-2000 for TUD wing design case study. Adjusted create_dsm() method w.r.t. determination of the node_style (separate method used now). Further improvements for BLISS-2000 architecture. Former-commit-id: 64eac2ce7bdd73c838a569e20f720836afaf709b --- kadmos/graph/graph_data.py | 377 +++++++++++++++++++--------------- kadmos/graph/graph_kadmos.py | 175 +++++++++------- kadmos/graph/graph_process.py | 30 +-- 3 files changed, 320 insertions(+), 262 deletions(-) diff --git a/kadmos/graph/graph_data.py b/kadmos/graph/graph_data.py index 49ca52487..a23074c9c 100644 --- a/kadmos/graph/graph_data.py +++ b/kadmos/graph/graph_data.py @@ -4396,6 +4396,128 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): 'functions_dicts': [sys_functions_dict, subsys_functions_dicts], 'functions_lists': functions_lists} + def _isolate_surrogate_models(self, sa): + """Method to isolate surrogate models and determine where certain functions belong within a distributed + system.""" + + # For each subgroup (SM) determine which pre-desvar functions are between the group and the design variables + # to be used for the surrogate model. Add these to the subgroup. + n_groups = len(sa['functions_dicts'][1]) + # Get the design variables of the group + for g in range(n_groups): + group_uncoupled_functions = set() + group_des_vars = [des_var for des_var, groups in sa['des_vars']['groups'].items() if g in groups] + des_vars_funs = set() + for g_des_var in group_des_vars: + des_var_targets = self.get_targets(g_des_var) + des_var_targets = [t for t in des_var_targets if self.nodes[t]['problem_role'] != self.FUNCTION_ROLES[1]] + des_vars_funs.update(des_var_targets) + subgraph = self.get_subgraph_by_function_nodes(sa['functions_dicts'][0][self.FUNCTION_ROLES[4]]+sa['functions_lists'][1][g]) + subgraph = subgraph.merge_functions(sa['functions_lists'][1][g], new_label='__merged_node__') + for des_vars_fun in des_vars_funs: + simple_paths = [p for p in nx.all_simple_paths(subgraph, des_vars_fun, '__merged_node__')] + for path in simple_paths: + for node in path: + if subgraph.nodes[node]['category'] == 'function' and node != '__merged_node__': + group_uncoupled_functions.add(node) + group_postdesvar_functions = [f for f in sa['functions_dicts'][0][self.FUNCTION_ROLES[4]] if f in group_uncoupled_functions] + sa['functions_dicts'][1][g][self.FUNCTION_ROLES[4]] = group_postdesvar_functions + sa['functions_dicts'][1][g][self.FUNCTION_ROLES[4]] + + # For extended couplings from uncoupled-DVD to post-coupling determine whether they should come from one of the + # surrogate models. + if sa['functions_dicts'][0][self.FUNCTION_ROLES[4]]: + subgraph1 = self.get_subgraph_by_function_nodes(sa['functions_dicts'][0][self.FUNCTION_ROLES[4]] + + sa['functions_dicts'][0][self.FUNCTION_ROLES[2]]) + subgraph2 = subgraph1.merge_functions(sa['functions_dicts'][0][self.FUNCTION_ROLES[4]], new_label='__merged_node__') + additional_couplings = set([n[2] for n in subgraph2.get_direct_coupling_nodes(['__merged_node__'] + sa['functions_dicts'][0][self.FUNCTION_ROLES[2]])]) + else: + additional_couplings = set() + subgraph1 = None + # First check upon which design variables the additional coupling depends + for add_coup in additional_couplings: + depends_on = set() + # Check whether there is a dependency between the + for des_var, gs in sa['des_vars']['groups'].items(): + if subgraph1.has_node(des_var): + if nx.has_path(subgraph1, des_var, add_coup): + depends_on.add(des_var) + # Extract the common groups from the dependencies + common_groups = set() + for dependency in depends_on: + dg = set(sa['des_vars']['groups'][dependency]) + if not common_groups: + common_groups.update(dg) + else: + common_groups.intersection(dg) + # Raise error if there are no common groups + if not common_groups: + raise NotImplementedError('An extended coupling from post-desvars functions without any common groups ' + 'has not been implemented (yet).') + elif len(common_groups) == 1: + # Else if there is one common group, then check if all required functions are already part of that group + # and add them, if they are not. + add_coup_group = list(common_groups)[0] + add_coup_source = self.get_source(add_coup) + group_des_vars = [des_var for des_var, groups in sa['des_vars']['groups'].items() if add_coup_group in groups] + des_vars_funs = set() + for g_des_var in group_des_vars: + des_var_targets = self.get_targets(g_des_var) + des_var_targets = [t for t in des_var_targets if + self.nodes[t]['problem_role'] != self.FUNCTION_ROLES[1]] + des_vars_funs.update(des_var_targets) + required_functions = [] + for des_vars_fun in des_vars_funs: + simple_paths = [p for p in nx.all_simple_paths(subgraph1, des_vars_fun, add_coup_source)] + for path in simple_paths: + for node in path: + if subgraph1.nodes[node]['category'] == 'function': + if node not in required_functions: + required_functions.append(node) + for fun in required_functions: + if fun not in sa['functions_dicts'][1][add_coup_group][self.FUNCTION_ROLES[4]]: + sa['functions_dicts'][1][add_coup_group][self.FUNCTION_ROLES[4]].append(fun) + else: # multiple common groups + # If there are multiple common groups, then check if the function is already part of one of them. + # Raise error if the function is part of multiple (not implemented, but can be) + add_coup_source = self.get_source(add_coup) + found_groups = set() + for common_group in common_groups: + group_funs = sa['functions_dicts'][1][common_group][self.FUNCTION_ROLES[4]] + if add_coup_source in group_funs: + found_groups.add(common_group) + if len(found_groups) != 1: + raise NotImplementedError('For multiple common groups the function has to be used in one of them at the moment.') + else: + add_coup_group = list(found_groups)[0] + + # Adjust the system analysis dictionary + sa['couplings']['extended'].append(add_coup) + sa['couplings']['groups'][add_coup] = add_coup_group + + # Remove the uncoupled-DVD functions from the global level + sa['functions_dicts'][0][self.FUNCTION_ROLES[4]] = [] + + # Create/update also list with all functions at a certain level + # TODO: this function is a repeat of an earlier function in analyze_distributed_system + functions_lists = [] + functions_list = [] + sys_functions_dict = sa['functions_dicts'][0] + subsys_functions_dicts = sa['functions_dicts'][1] + for key, item in sys_functions_dict.items(): + functions_list.extend(item) + functions_lists.append(functions_list) + for function_dict in subsys_functions_dicts: + functions_list = [] + for key, item in function_dict.items(): + functions_list.extend(item) + if len(functions_lists) == 1: + functions_lists.append([functions_list]) + else: + functions_lists[1].append(functions_list) + sa['functions_lists'] = functions_lists + + return sa + def get_objective_node(self): """Method to get the single (or non-existent) objective node from a graph. @@ -4680,13 +4802,15 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): elif mdao_arch == graph.OPTIONS_ARCHITECTURES[8]: # BLISS-2000 coupled_functions_groups = graph.graph['problem_formulation']['coupled_functions_groups'] n_groups = len(coupled_functions_groups) - sys_opt, sys_conv, sys_sms, sub_smbds, sub_does, sub_opts, sub_smbs = \ + sys_opt, sys_conv, sys_sms, sub_does, sub_opts, sub_convs = \ self.get_architecture_node_ids(mdao_arch, number_of_groups=n_groups) - sys_opt_label, sys_conv_label, sys_sms_labels, sub_smbds_labels, sub_does_labels, sub_opts_labels, \ - sub_smbs_labels = self.get_architecture_node_labels(mdao_arch, number_of_groups=n_groups) + sys_opt_label, sys_conv_label, sys_sms_labels, sub_does_labels, sub_opts_labels, sub_convs_labels = \ + self.get_architecture_node_labels(mdao_arch, number_of_groups=n_groups) sa = self._analyze_distributed_system(des_var_nodes, objective_node, constraint_nodes, mg_function_ordering) + sa = self._isolate_surrogate_models(sa) + # Determine any required function instances, add them and adjust subsys_functions_dict accordingly sa = mdg._split_functions(sa) @@ -4701,6 +4825,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): sms_ins = [] weight_nodes2 = [] sm_inps_lists = [] + doe_inps_list = [] sa['functions_dicts'][0][self.FUNCTION_ROLES[1]].extend(sys_sms) for idx, subsys_functions_dict in enumerate(sa['functions_dicts'][1]): # Get global and local design nodes and local constraint nodes involved in the group @@ -4719,13 +4844,33 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): # Add the weighted couplings objective function according to BLISS-2000 group_wcf_node, group_wcf_obj_node, weight_nodes = \ - mdg.connect_weighted_couplings_objective_function(idx, local_group_couplings_group) + mdg.connect_weighted_couplings_objective_function(idx, [f for f in local_group_couplings_group if f in sa['couplings']['basic']]) sa['functions_dicts'][1][idx][mdg.FUNCTION_ROLES[2]].append(group_wcf_node) # Add and connect the sub-level optimizer fin_des_vars, _, _, _ = mdg.connect_optimizer(sub_opts[idx], local_des_vars_group, group_wcf_obj_node, local_cnstrnt_vars_group, label=sub_opts_labels[idx]) + # (optionally) add a converger or check for the removal of feedback? + coupled_funs = sa['functions_dicts'][1][idx][mdg.FUNCTION_ROLES[1]] + coupled_subgraph = mdg.get_subgraph_by_function_nodes(coupled_funs) + try: + cycles = nx.find_cycle(coupled_subgraph) + except NetworkXNoCycle: + cycles = [] + + if cycles: + # TODO: temp fix since coupled_functions_groups now also indicates partitions + del coupled_subgraph.graph['problem_formulation']['coupled_functions_groups'] + best_order = coupled_subgraph.get_possible_function_order(method='single-swap') + sa['functions_dicts'][1][idx][mdg.FUNCTION_ROLES[1]] = best_order + mdg.connect_converger(sub_convs[idx], 'Gauss-Seidel', best_order, False, label=sub_convs_labels[idx]) + else: + for f in sa['functions_dicts'][1][idx][mdg.FUNCTION_ROLES[1]]: + mdg.nodes[f]['architecture_role'] = self.ARCHITECTURE_ROLES_FUNS[6] + for f in sa['functions_dicts'][1][idx][mdg.FUNCTION_ROLES[2]]: + mdg.nodes[f]['architecture_role'] = self.ARCHITECTURE_ROLES_FUNS[6] + # Add local coupling nodes as final output in the graph lgcg_finals = [] for node in local_group_couplings_group: @@ -4740,14 +4885,9 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): doe_inps, doe_outs = mdg.connect_doe_block(sub_does[idx], external_group_couplings_copies_group + local_des_vars_copies_group+weight_nodes, - lgcg_finals+fin_des_vars) - - # Add and connect the surrogate model boundary determinator - mdg.connect_boundary_determinator(sub_smbds[idx], [], doe_inps, label=sub_smbds_labels[idx]) - - # Add and connect the surrogate model builder - sm_def_node = mdg.connect_surrogate_model_builder(sub_smbs[idx], doe_inps, doe_outs, - label=sub_smbs_labels[idx]) + lgcg_finals+fin_des_vars, label=sub_does_labels[idx]) + # Keep track of DOE inputs for convergence check + doe_inps_list.extend(doe_inps) # Add and connect the surrogate model itself sm_inps = [] @@ -4768,7 +4908,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): mdg.mark_as_design_variable(node2, ignore_outdegree=True) sm_inps.extend(global_des_vars_group) sm_out_originals = [mdg.nodes[node]['related_to_schema_node'] for node in lgcg_finals+fin_des_vars] - sm_outs = mdg.connect_surrogate_model(sys_sms[idx], sm_def_node, sm_inps, sm_out_originals, + sm_outs = mdg.connect_surrogate_model(sys_sms[idx], doe_inps, doe_outs, sm_inps, sm_out_originals, label=sys_sms_labels[idx]) sm_inps_lists.append(sm_inps) sms_ins.extend(sm_inps) @@ -4781,13 +4921,13 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): sms_outs_related = [mdg.nodes[node]['related_to_schema_node'] for node in sms_outs] for func in sa['functions_dicts'][0][self.FUNCTION_ROLES[2]]: sources = mdg.get_sources(func) + mdg.nodes[func]['architecture_role'] = mdg.ARCHITECTURE_ROLES_FUNS[6] for source in sources: if 'related_to_schema_node' in mdg.nodes[source]: rel_node = mdg.nodes[source]['related_to_schema_node'] if rel_node in sms_outs_related: sm_node = sms_outs[sms_outs_related.index(rel_node)] # Reconnect the source to the SM node - assert mdg.in_degree(source) == 0, 'This node is supposed to be an input.' mdg.remove_edge(source, func) mdg.add_edge(sm_node, func) @@ -4806,52 +4946,22 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin): ccf_mapping[sms_out] = map_node else: raise NotImplementedError('Could not find the right map node somehow...') - ccf_node, cc_nodes = mdg.connect_consistency_constraint_function(ccf_mapping) + ccf_node, cc_nodes = mdg.connect_consistency_constraint_function(ccf_mapping, + architecture_role=self.ARCHITECTURE_ROLES_FUNS[6]) sa['functions_dicts'][0][self.FUNCTION_ROLES[2]].append(ccf_node) # Connect the system-level optimizer - fin_des_vars, _, _, ini_guess_nodes = mdg.connect_optimizer(sys_opt, + fin_des_vars, fin_obj, _, ini_guess_nodes = mdg.connect_optimizer(sys_opt, list(sys_lev_des_vars)+weight_nodes2 + ccf_mapping.values(), objective_node, list(sys_lev_cnstrnts)+cc_nodes, label=sys_opt_label) - # Connect converger check + # Connect distributed system converger fin_sys_lev_des_vars = [node for node in fin_des_vars if mdg.nodes[node]['related_to_schema_node'] in sys_lev_des_vars] - ini_guess_nodes_filt = [node for node in ini_guess_nodes if - mdg.nodes[node]['related_to_schema_node'] in sys_lev_des_vars] - mdg.connect_convergence_check(sys_conv, fin_sys_lev_des_vars+ini_guess_nodes_filt, label=sys_conv_label) - - # Connect the initial guesses and final values to the surrogate model boundary determinator - ini_guess_nodes_related = [mdg.nodes[node]['related_to_schema_node'] for node in ini_guess_nodes] - fin_val_nodes_related = [mdg.nodes[node]['related_to_schema_node'] for node in fin_des_vars] - for idx, smbo in enumerate(sub_smbds): - smbo_sources = sm_inps_lists[idx] - for smbo_source in smbo_sources: - # First the initial guesses - if 'related_to_schema_node' in mdg.nodes[smbo_source]: - if mdg.nodes[smbo_source]['related_to_schema_node'] in ini_guess_nodes_related: - ini_inp = ini_guess_nodes[ini_guess_nodes_related.index(mdg.nodes[smbo_source] - ['related_to_schema_node'])] - else: - raise NotImplementedError('Could not find related node.') - else: - ini_inp = ini_guess_nodes[ini_guess_nodes_related.index(smbo_source)] - assert ini_inp, 'Could not find the right node.' - mdg.add_edge(ini_inp, smbo) - - # Then the final values - if 'related_to_schema_node' in mdg.nodes[smbo_source]: - if mdg.nodes[smbo_source]['related_to_schema_node'] in fin_val_nodes_related: - fin_inp = fin_des_vars[fin_val_nodes_related.index(mdg.nodes[smbo_source] - ['related_to_schema_node'])] - else: - raise NotImplementedError('Could not find related node.') - else: - fin_inp = fin_des_vars[fin_val_nodes_related.index(smbo_source)] - assert fin_inp, 'Could not find the right node.' - mdg.add_edge(fin_inp, smbo) + mdg.connect_distributed_system_converger(sys_conv, fin_des_vars + [fin_obj], + doe_inps_list + ini_guess_nodes, label=sys_conv_label) # Finally, connect the coordinator mdg.connect_coordinator(additional_inputs=fin_sys_lev_des_vars) @@ -5264,6 +5374,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): sub_funs = sa_fun_lists[1] function_instances = dict() + # First function instances between system level and sub-level for idx in range(len(sub_funs)): for sub_fun in sub_funs[idx]: if sub_fun in sys_funs: @@ -5271,6 +5382,16 @@ class MdaoDataGraph(DataGraph, MdaoMixin): 'serving': [fun for fun in sub_funs[idx] if set(self.get_targets(sub_fun)) .intersection(set(self.get_sources(fun)))]} + # Then function instances between sub-levels + # TODO: Now the instance creation is limited to one instance for a certain function while this could be more... + for idx in range(len(sub_funs)-1): + for idy in range(1, len(sub_funs)): + for sub_funx in sub_funs[idx]: + if sub_funx in sub_funs[idy]: + function_instances[sub_funx] = {'group_idx': idy, + 'serving': [fun for fun in sub_funs[idy] if + set(self.get_targets(sub_funx)) + .intersection(set(self.get_sources(fun)))]} for f_ins, f_det in function_instances.items(): new_instance = self.add_function_instance(f_ins, serves=f_det['serving']) @@ -5486,6 +5607,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): category='variable', label=var2.split('/')[-1] + '_' + var1.split('/')[-1], instance=1, + problem_role=self.PROBLEM_ROLES_VARS[0], architecture_role=self.ARCHITECTURE_ROLES_VARS[11]) weight_nodes.append(var2) eq_lab1 = 'y{}'.format(idx) @@ -5507,7 +5629,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): self.add_equation((new_function_node, new_obj_node), math_expression, 'Python') return new_function_node, new_obj_node, weight_nodes - def connect_consistency_constraint_function(self, ccv_mappings): + def connect_consistency_constraint_function(self, ccv_mappings, architecture_role=None): """Method to add a consistency constraint function to an MDG. :param ccv_mappings: mapping between nodes that should be made consistent @@ -5526,7 +5648,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): label=self.CONSCONS_LABEL, instance=1, problem_role=self.FUNCTION_ROLES[2], # post-coupling - architecture_role=self.ARCHITECTURE_ROLES_FUNS[8], # post-coupling analysis + architecture_role=self.ARCHITECTURE_ROLES_FUNS[8] if architecture_role is None else architecture_role, function_type='consistency') # Connect the variable inputs for the function new_con_nodes = [] @@ -5715,13 +5837,15 @@ class MdaoDataGraph(DataGraph, MdaoMixin): return - def connect_convergence_check(self, converger, inputs, label='SYS-CONV'): + def connect_distributed_system_converger(self, converger, inputs, outputs, label='SYS-CONV'): """Method to add a convergence check in the MDG. :param converger: node ID of the converger block :type converger: basestring :param inputs: input nodes to be connected for convergence check :type inputs: list + :param outputs: output nodes to be connected for convergence check + :type outputs: list :param label: label for the converger block :type label: basestring :return: the convergence check node @@ -5733,7 +5857,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): if not self.has_node(converger): self.add_node(converger, category='function', - architecture_role=self.ARCHITECTURE_ROLES_FUNS[2], + architecture_role=self.ARCHITECTURE_ROLES_FUNS[10], # distributed system converger label=label, instance=1) assert isinstance(inputs, list) @@ -5743,13 +5867,10 @@ class MdaoDataGraph(DataGraph, MdaoMixin): for input in inputs: self.add_edge(input, converger) - # Add convergence check output - conv_check_node = '/{}/mdoData/systemConvergenceCheck'.format(self.get_schema_root_name(input)) - self.assert_node_exists_not(conv_check_node) - self.add_node(conv_check_node, category='variable', instance=1, label='conv_check') - self.add_edge(converger, conv_check_node) + for output in outputs: + self.add_edge(converger, output) - return conv_check_node + return def connect_converger(self, converger, conv_type, coupling_functions, include_couplings_as_final_output, system_converger=False, label='CONV', conv_is_optimizer=False): @@ -5934,9 +6055,10 @@ class MdaoDataGraph(DataGraph, MdaoMixin): self.add_edge(var, optimizer) # Create a final value copy and connect it as output of the associated functions fin_value_node = self.copy_node_as(var, architecture_role=self.ARCHITECTURE_ROLES_VARS[5]) - if fin_obj is not None: + if fin_obj is None: + fin_obj = fin_value_node + else: fin_cnstrnts.append(fin_value_node) - fin_obj = fin_value_node self.copy_edge([self.get_sources(var)[0], var], [self.get_sources(var)[0], fin_value_node]) # If the graph contains consistency constraint variables, then connect these to the optimizer as well consconcs_nodes = self.find_all_nodes(category='variable', @@ -6058,56 +6180,7 @@ class MdaoDataGraph(DataGraph, MdaoMixin): return inps, outs - def connect_surrogate_model_builder(self, sm_builder, input_x_data, input_y_data, label=None): - """Method to connect a surrogate model builder node in the MDG. The surrogate model builder takes input data - (generally from a DOE) and gives the definition of a surrogate model as output. - - :param sm_builder: new function node - :type sm_builder: basestring - :param input_x_data: list of nodes for which results were obtained (x-axis) - :type input_x_data: list - :param input_y_data: list of nodes with analysis results (y-axis) - :type input_y_data: list - :param label: label of the new function node - :type label: basestring - :return: output node with surrogate model definition - :rtype: basestring - """ - - # Input assertions - self.assert_node_exists_not(sm_builder) - self.assert_node_exists(input_x_data) - self.assert_node_exists(input_y_data) - - # Set label - if label is None: - label = str(sm_builder) - - # Add the surrogate model builder - self.add_node(sm_builder, - category='function', - architecture_role=self.ARCHITECTURE_ROLES_FUNS[11], # Surrogate model builder - label=label, - instance=1, - metadata=dict(input_x_data=[], input_y_data=[])) - - # Add the data for building as input to the builder - for input_x_entry in input_x_data: - self.add_edge(input_x_entry, sm_builder) - self.nodes[sm_builder]['metadata']['input_x_data'].append(input_x_entry) - for input_y_entry in input_y_data: - self.add_edge(input_y_entry, sm_builder) - self.nodes[sm_builder]['metadata']['input_y_data'].append(input_y_entry) - - # Add the surrogate model definition as output of the block - output_node = '/{}/surrogateModels/{}/definition'.format(self.get_schema_root_name(input_x_entry[0]), - label.replace('-', '')) - assert not self.has_node(output_node), 'The node {} already exists.'.format(output_node) - self.add_node(output_node, category='variable', instance=1, label='def{}'.format(label.replace('-', ''))) - self.add_edge(sm_builder, output_node) - return output_node - - def connect_surrogate_model(self, sm, def_node, sm_inputs, sm_out_originals, label=None): + def connect_surrogate_model(self, sm, train_inputs, train_outputs, sm_inputs, sm_out_originals, label=None): """Method to connect a surrogate to the right nodes in the MDG. :param sm: new function node of surrogate model @@ -6126,7 +6199,6 @@ class MdaoDataGraph(DataGraph, MdaoMixin): # Input assertions assert not self.has_node(sm), 'Node {} already exists in the graph.'.format(sm) - assert self.has_node(def_node), 'Node {} is missing in the graph.'.format(def_node) # Set label if label is None: @@ -6135,13 +6207,16 @@ class MdaoDataGraph(DataGraph, MdaoMixin): # Add the surrogate model self.add_node(sm, category='function', - architecture_role=self.ARCHITECTURE_ROLES_FUNS[12], # Surrogate model + architecture_role=self.ARCHITECTURE_ROLES_FUNS[9], # Surrogate model label=label, instance=1) # Connect the surrogate model - # Connect model definition as input - self.add_edge(def_node, sm) + # Connect train data as input + for train_input in train_inputs: + self.add_edge(train_input, sm) + for train_output in train_outputs: + self.add_edge(train_output, sm) # Connect the model inputs for sm_input in sm_inputs: @@ -6156,47 +6231,6 @@ class MdaoDataGraph(DataGraph, MdaoMixin): return node_apprs - def connect_boundary_determinator(self, bd, inputs, bounds, label=None): - """Method to connect a boundary determinator method for building surrogate models. This boundary determinator is - used in architectures like BLISS-2000 to adjust bounds for different cycles. - - :param bd: new function node - :type bd: basestring - :param inputs: inputs of the function - :type inputs: list - :param bounds: bounds output of the function - :type bounds: list - :param label: label of the new function node - :type label: basestring - :return: - :rtype: - """ - - # Input assertions - self.assert_node_exists_not(bd) - self.assert_node_exists(bounds) - - # Set label - if label is None: - label = str(bd) - - # Add the surrogate model - self.add_node(bd, - category='function', - architecture_role=self.ARCHITECTURE_ROLES_FUNS[10], # Surrogate model boundary determinator - label=label, - instance=1) - - # Connect boundary determination inputs - for input in inputs: - self.add_edge(input, bd) - - # Connect the surrogate model boundary determinator output - for bound in bounds: - self.add_edge(bd, bound) - - return - def connect_partitions(self, mdao_arch, sub_func_orderings, coup_functions): """Method to connect partitions in a data graph of a monolithic architecture """ @@ -6619,19 +6653,32 @@ class MdaoDataGraph(DataGraph, MdaoMixin): elif mdao_arch == mdg.OPTIONS_ARCHITECTURES[8]: # BLISS-2000 distr_function_ordering = mdg.graph['distr_function_ordering'] n_groups = len(distr_function_ordering[1]) - sys_opt, sys_conv, _, sub_smbds, sub_does, sub_opts, sub_smbs = \ + sys_opt, sys_conv, _, sub_does, sub_opts, sub_convs = \ self.get_architecture_node_ids(mdao_arch, number_of_groups=n_groups) sequence1 = [coor] + distr_function_ordering[0][self.FUNCTION_ROLES[3]] + [sys_conv] - mpg.add_process(sequence1, 0, mdg) + sub_does_cs = [] for idx, subgroup in enumerate(distr_function_ordering[1]): - sequence2 = [sys_conv] + subgroup[self.FUNCTION_ROLES[3]] + [sub_smbds[idx]] + [sub_does[idx]] + \ - [sub_opts[idx]] + subgroup[self.FUNCTION_ROLES[4]] + subgroup[self.FUNCTION_ROLES[1]] + \ - subgroup[self.FUNCTION_ROLES[2]] - mpg.add_process(sequence2, mpg.nodes[sequence1[-1]]['process_step'], mdg, - end_in_iterative_node=sub_opts[idx]) + if not mpg.has_node(sub_convs[idx]): + sequence2 = sequence1 + subgroup[self.FUNCTION_ROLES[3]] + [sub_does[idx]] + \ + [sub_opts[idx]] + subgroup[self.FUNCTION_ROLES[4]] + subgroup[self.FUNCTION_ROLES[1]] + \ + subgroup[self.FUNCTION_ROLES[2]] + mpg.add_process(sequence2, 0, mdg, + end_in_iterative_node=sub_opts[idx]) + else: + sequence2a = sequence1 + subgroup[self.FUNCTION_ROLES[3]] + [sub_does[idx]] + \ + [sub_opts[idx]] + subgroup[self.FUNCTION_ROLES[4]] + [sub_convs[idx]] + mpg.add_process(sequence2a, 0, mdg) + sequence2b = [sub_convs[idx]] + subgroup[self.FUNCTION_ROLES[1]] + mpg.add_process(sequence2b, mpg.nodes[sequence2a[-1]]['process_step'], mdg, + end_in_iterative_node=sub_convs[idx]) + sequence2c = [sub_convs[idx]] + subgroup[self.FUNCTION_ROLES[2]] + mpg.add_process(sequence2c, mpg.nodes[sequence2b[-1]]['process_step'], mdg, + end_in_iterative_node=sub_opts[idx]) mpg.connect_nested_iterators(sub_does[idx], sub_opts[idx]) - sequence3 = [sub_does[idx]] + [sub_smbs[idx]] + [sys_opt] + sequence3 = [sub_does[idx]] + [sys_opt] mpg.add_process(sequence3, mpg.nodes[sub_does[idx]]['converger_step'], mdg) + sub_does_cs.append(mpg.nodes[sub_does[idx]]['converger_step']) + mpg.nodes[sys_opt]['process_step'] = max(sub_does_cs) + 1 sequence4 = [sys_opt] + distr_function_ordering[0][self.FUNCTION_ROLES[4]] + \ distr_function_ordering[0][self.FUNCTION_ROLES[1]] + \ distr_function_ordering[0][self.FUNCTION_ROLES[2]] diff --git a/kadmos/graph/graph_kadmos.py b/kadmos/graph/graph_kadmos.py index 84291fac6..d1dc1d790 100644 --- a/kadmos/graph/graph_kadmos.py +++ b/kadmos/graph/graph_kadmos.py @@ -68,6 +68,11 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): 'Box-Behnken design'] # 5 OPTIONS_CONVERGERS = ['Jacobi', 'Gauss-Seidel', None] FUNCTION_ROLES = ['pre-coupling', 'coupled', 'post-coupling', 'pre-desvars', 'post-desvars'] + FUNCTION_ROLES_NODESTYLES = ['PreAnalysisDVI', # 0 + 'CoupledAnalysis', # 1 + 'PostAnalysis', # 2 + 'PreAnalysisDVI', # 3 + 'PreAnalysisDVD'] # 4 CMDOWS_VERSION = '0.9' CMDOWS_ATTRIBUTES = ['nominal_value', 'valid_ranges', 'constraint_type', 'constraint_operator', 'reference_value', 'required_equality_precision', 'samples', 'variable_type'] @@ -110,10 +115,8 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): 'post-iterator analysis', # 6 'coupled analysis', # 7 'post-coupling analysis', # 8 - 'consistency constraint function', # 9 - 'boundary determinator', # 10 - 'surrogate model builder', # 11 - 'surrogate model'] # 12 + 'surrogate model', # 9 + 'distributed system converger'] # 10 ARCHITECTURE_ROLES_NODESTYLES = ['Coordinator', # 0 'Optimization', # 1 'Converger', # 2 @@ -123,11 +126,9 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): 'PreAnalysisDVD', # 6 'CoupledAnalysis', # 7 'PostAnalysis', # 8 - 'PostAnalysis', # 9 - 'Metamodel', # 10 - 'Metamodel', # 11 - 'Metamodel'] # 12 - CMDOWS_ARCHITECTURE_ROLE_SPLITTER = get_list_entries(ARCHITECTURE_ROLES_FUNS, 0, 1, 2, 3, 9) + 'Metamodel', # 9 + 'Converger'] # 10 + CMDOWS_ARCHITECTURE_ROLE_SPLITTER = get_list_entries(ARCHITECTURE_ROLES_FUNS, 0, 1, 2, 3) SYS_PREFIX = 'Sys-' SUBSYS_PREFIX = 'Sub-' SUBSYS_SUFFIX = '-' @@ -648,7 +649,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): # ---------------------------------------------------------------------------------------------------------------- # def create_dsm(self, file_name, destination_folder=None, open_pdf=False, mpg=None, include_system_vars=True, summarize_vars=False, function_order=None, keep_tex_file=False, abbreviate_keywords=False, - compile_pdf=True, colors_based_on='problem_roles'): + compile_pdf=True, colors_based_on='function_roles'): """Method to create a (X)DSM PDF file :param file_name: name of the file to be saved @@ -684,7 +685,12 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): assert keep_tex_file or compile_pdf, 'The settings do not make sense, set either keep_tex_file or compile_pdf' \ ' to True, or both!' if colors_based_on == 'partitions': - assert 'coupled_functions_groups' in self.graph['problem_formulation'], 'Graph is not partitioned' + assert 'coupled_functions_groups' in self.graph['problem_formulation'], 'Graph is not partitioned.' + if colors_based_on == 'function_roles': + if isinstance(self, MdaoDataGraph): + colors_based_on = 'architecture_roles' + else: + colors_based_on = 'problem_roles' # Check if MPG is applicable if mpg is not None: @@ -767,42 +773,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): # Add analysis blocks for idx, item in enumerate(diagonal_nodes): node = diagonal_nodes[idx] - if isinstance(graph, FundamentalProblemGraph): - if colors_based_on == 'partitions' and node in [node1 for nodes in self.graph[ - 'problem_formulation']['coupled_functions_groups'] for node1 in nodes]: - partitions = self.graph['problem_formulation']['coupled_functions_groups'] - part_id = [i for i in range(len(partitions)) if node in partitions[i]] - node_style = 'EvenPartitions' if part_id[0] % 2 == 0 else 'OddPartitions' - elif 'problem_role' in graph.node[node]: - if graph.node[node]['problem_role'] == graph.FUNCTION_ROLES[0]: - node_style = 'PreAnalysis' - elif graph.nodes[node]['problem_role'] == graph.FUNCTION_ROLES[1]: - node_style = 'CoupledAnalysis' - elif graph.nodes[node]['problem_role'] == graph.FUNCTION_ROLES[2]: - node_style = 'PostAnalysis' - else: - logger.warning('An invalid FPG has been provided: problem_role missing for: %s.' % node) - node_style = 'RcgAnalysis' - elif isinstance(graph, MdaoDataGraph): - if 'architecture_role' in graph.nodes[node]: - try: - role_index = self.ARCHITECTURE_ROLES_FUNS.index(graph.nodes[node]['architecture_role']) - except ValueError: - raise AssertionError('Architecture role %s is not supported for creation of XDSMs.' - % graph.nodes[node]['architecture_role']) - if colors_based_on == 'partitions' and node in [node1 for nodes in self.graph[ - 'problem_formulation']['coupled_functions_groups'] for node1 in nodes]: - partitions = self.graph['problem_formulation']['coupled_functions_groups'] - part_id = [i for i in range(len(partitions)) if node in partitions[i]] - node_style = 'EvenPartitions' if part_id[0] % 2 == 0 else 'OddPartitions' - else: - node_style = self.ARCHITECTURE_ROLES_NODESTYLES[role_index] - else: - logger.warning('An invalid MDG has been provided: architecture_role missing for: %s.' % node) - node_style = 'RcgAnalysis' - else: - node_style = 'RcgAnalysis' - + node_style = self._get_node_style(node, colors_based_on) node_text = graph.nodes[node].get('label', str(node)) if '^{' in node_text: node_text = node_text.replace('^{', '$^{').replace('}', '}$') @@ -817,18 +788,7 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): assert len(node_list) == 1, 'Somehow, a unique diagonal position {} could not be found in the MDG.'\ .format(idx) node = node_list[0] - try: - role_index = self.ARCHITECTURE_ROLES_FUNS.index(graph_mpg.nodes[node]['architecture_role']) - except ValueError: - raise AssertionError('Architecture role %s is not supported for creation of XDSMs.' - % graph_mpg.nodes[node]['architecture_role']) - if colors_based_on == 'partitions' and node in [node1 for nodes in self.graph['problem_formulation'][ - 'coupled_functions_groups'] for node1 in nodes]: - partitions = self.graph['problem_formulation']['coupled_functions_groups'] - part_id = [i for i in range(len(partitions)) if node in partitions[i]] - node_style = 'EvenPartitions' if part_id[0] % 2 == 0 else 'OddPartitions' - else: - node_style = self.ARCHITECTURE_ROLES_NODESTYLES[role_index] + node_style = self._get_node_style(node, colors_based_on) # Determine node text node_text = graph_mpg.get_node_text(node) if '^{' in node_text: @@ -949,6 +909,53 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): # Return return + def _get_node_style(self, node, colors_based_on): + """Method to retrieve the right node style of a given node. + + :param node: Node from graph + :type node: str + :param colors_based_on: setting to assess on which node property the color should be based + :type colors_based_on: str + :return: node_style + :rtype: str + """ + from graph_data import FundamentalProblemGraph, MdaoDataGraph + + if colors_based_on == 'partitions': + if node in [node1 for nodes in self.graph['problem_formulation']['coupled_functions_groups'] for node1 in + nodes]: + partitions = self.graph['problem_formulation']['coupled_functions_groups'] + part_id = [i for i in range(len(partitions)) if node in partitions[i]] + node_style = 'EvenPartitions' if part_id[0] % 2 == 0 else 'OddPartitions' + else: + if isinstance(self, FundamentalProblemGraph) and 'problem_role' in self.nodes[node]: + role_index = self.FUNCTION_ROLES.index(self.nodes[node]['problem_role']) + node_style = self.FUNCTION_ROLES_NODESTYLES[role_index] + elif isinstance(self, MdaoDataGraph) and 'architecture_role' in self.nodes[node]: + role_index = self.ARCHITECTURE_ROLES_FUNS.index(self.nodes[node]['architecture_role']) + node_style = self.ARCHITECTURE_ROLES_NODESTYLES[role_index] + else: + if isinstance(self, (FundamentalProblemGraph, MdaoDataGraph)): + logger.warning('An invalid graph has been provided, expected role missing for: %s.' % node) + node_style = 'RcgAnalysis' + elif colors_based_on == 'problem_roles': + if 'problem_role' in self.nodes[node]: + role_index = self.FUNCTION_ROLES.index(self.nodes[node]['problem_role']) + node_style = self.FUNCTION_ROLES_NODESTYLES[role_index] + else: + logger.warning('An incomplete graph has been provided: problem_role missing for: %s.' % node) + node_style = 'RcgAnalysis' + elif colors_based_on == 'architecture_roles': + if 'architecture_role' in self.nodes[node]: + role_index = self.ARCHITECTURE_ROLES_FUNS.index(self.nodes[node]['architecture_role']) + node_style = self.ARCHITECTURE_ROLES_NODESTYLES[role_index] + else: + logger.warning('An incomplete graph has been provided: architecture_role missing for: %s.' % node) + node_style = 'RcgAnalysis' + else: + node_style = 'RcgAnalysis' + return node_style + def _create_cmdows_header(self, description, modification, creator, version, cmdows_version, timestamp=datetime.now()): @@ -2723,6 +2730,24 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): return subgraph + def get_source(self, node): + """Function to determine the single source of a given node. Throws error if node has multiple sources. + + :param node: node for which source should be found + :type node: str + :return: source + :rtype: str + """ + + self.assert_node_exists(node) + sources = [edge[0] for edge in self.in_edges(node)] + if len(sources) > 1: + raise AssertionError('Node has multiple sources, use get_sources() method instead.') + elif len(sources) == 1: + return sources[0] + else: + return None + def get_sources(self, node): """Function to determine the sources of a given node. @@ -2737,6 +2762,24 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): return sources + def get_target(self, node): + """Function to determine the single target of a given node. Throws error if node has multiple targets. + + :param node: node for which target should be found + :type node: str + :return: target + :rtype: str + """ + + self.assert_node_exists(node) + targets = [edge[1] for edge in self.out_edges(node)] + if len(targets) > 1: + raise AssertionError('Node has multiple targets, use get_targets() method instead.') + elif len(targets) == 1: + return targets[0] + else: + return None + def get_targets(self, node): """Function to determine the targets of a given node. @@ -3213,15 +3256,13 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): sys_conv = '{}{}'.format(self.SYS_PREFIX, self.CONVERGER_STRING) sys_sms = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SM_STRING, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - sub_smbds = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SMBD_STRING, self.SUBSYS_SUFFIX, item) for - item in range(number_of_groups)] sub_does = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.DOE_STRING, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] sub_opts = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.OPTIMIZER_STRING, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - sub_smbs = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SMB_STRING, self.SUBSYS_SUFFIX, item) for + sub_convs = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.CONVERGER_STRING, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - return sys_opt, sys_conv, sys_sms, sub_smbds, sub_does, sub_opts, sub_smbs + return sys_opt, sys_conv, sys_sms, sub_does, sub_opts, sub_convs def get_architecture_node_labels(self, mdao_architecture, number_of_groups=None): """Method to get the labels of architecture nodes specific for a certain MDAO architecture. @@ -3259,16 +3300,13 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): sys_conv_label = '{}{}'.format(self.SYS_PREFIX, self.CONVERGER_LABEL) sys_sms_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SM_LABEL, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - sub_smbds_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SMBD_LABEL, self.SUBSYS_SUFFIX, item) - for item in range(number_of_groups)] sub_does_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.DOE_LABEL, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] sub_opts_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.OPTIMIZER_LABEL, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - sub_smbs_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.SMB_LABEL, self.SUBSYS_SUFFIX, item) + sub_convs_labels = ['{}{}{}{}'.format(self.SUBSYS_PREFIX, self.CONVERGER_LABEL, self.SUBSYS_SUFFIX, item) for item in range(number_of_groups)] - return sys_opt_label, sys_conv_label, sys_sms_labels, sub_smbds_labels, sub_does_labels, sub_opts_labels, \ - sub_smbs_labels + return sys_opt_label, sys_conv_label, sys_sms_labels, sub_does_labels, sub_opts_labels, sub_convs_labels def add_objective_function_by_nodes(self, *args, **kwargs): """This function adds objective functions to the graph using lists of variable nodes. @@ -3962,7 +4000,6 @@ class KadmosGraph(nx.DiGraph, EquationMixin, VistomsMixin): args = list(args[0]) # Input assertions - assert len(args) > 1, 'More than 1 input is required for this function.' for arg in args: assert isinstance(arg, basestring) diff --git a/kadmos/graph/graph_process.py b/kadmos/graph/graph_process.py index 0246a1c15..5cd609a8b 100644 --- a/kadmos/graph/graph_process.py +++ b/kadmos/graph/graph_process.py @@ -294,9 +294,8 @@ class MdaoProcessGraph(ProcessGraph): # BLISS-2000: Append system-level convergence check if bliss2000: - convs = self.find_all_nodes(attr_cond=['architecture_role', '==', self.ARCHITECTURE_ROLES_FUNS[2]]) - sys_conv = [item for item in convs if self.SYS_PREFIX in item] - assert len(sys_conv) == 1, '{} system convergers found, one expected.'.format(len(sys_conv)) + sys_conv = self.find_all_nodes(attr_cond=['architecture_role', '==', self.ARCHITECTURE_ROLES_FUNS[10]]) + assert len(sys_conv) == 1, '{} distributed system convergers found, one expected.'.format(len(sys_conv)) diagonal_order.extend(sys_conv) # Append system level optimizer and/or DOE block @@ -345,19 +344,6 @@ class MdaoProcessGraph(ProcessGraph): # Append sublevel functions here for idx, subsyslevel_ord in enumerate(subsyslevel_orderings): - # BLISS-2000: add surrogate model boundary determinator - if bliss2000: - smbds = self.find_all_nodes( - attr_cond=['architecture_role', '==', - self.ARCHITECTURE_ROLES_FUNS[10]]) # boundary determinator - if len(smbds) > 1: - sub_smbd = [item for item in smbds if - self.SUBSYS_SUFFIX in item and self.SUBSYS_SUFFIX + str(idx) in item] - assert len(sub_smbd) == 1, '{} subsystem boundary determinators found, one expected.'.format( - len(sub_smbd)) - smbds = sub_smbd - diagonal_order.extend(smbds) - # Append subsystem-level pre-coupling functions if self.FUNCTION_ROLES[0] in subsyslevel_ord: diagonal_order.extend(subsyslevel_ord[self.FUNCTION_ROLES[0]]) @@ -406,18 +392,6 @@ class MdaoProcessGraph(ProcessGraph): if self.FUNCTION_ROLES[2] in subsyslevel_ord: diagonal_order.extend(subsyslevel_ord[self.FUNCTION_ROLES[2]]) - # Append subsystem-level surrogate model builder - if bliss2000: - smbs = self.find_all_nodes( - attr_cond=['architecture_role', '==', self.ARCHITECTURE_ROLES_FUNS[11]]) # SM builder - if len(smbs) > 1: - sub_smb = [item for item in smbs if - self.SUBSYS_SUFFIX in item and self.SUBSYS_SUFFIX + str(idx) in item] - assert len(sub_smb) == 1, '{} subsystem boundary determinators found, one expected.'.format( - len(sub_smb)) - smbs = sub_smb - diagonal_order.extend(smbs) - # Append system-level post-coupling functions if distr_conv: if self.FUNCTION_ROLES[2] in syslevel_ordering: -- GitLab