From bab232dbb8a167d39305ea19e9e6dd2a2e9ca631 Mon Sep 17 00:00:00 2001
From: Anne-Liza <a.m.r.m.bruggeman@student.tudelft.nl>
Date: Mon, 21 May 2018 11:25:00 +0200
Subject: [PATCH] Added runtime - feedback trade-off to partitioning. Fixed
small bug in sequencing algorithms.
Former-commit-id: 3341b0086223aa911b04a4bbcd00b4603369f5fb
---
kadmos/graph/graph_data.py | 466 +++++++++++++++++++++++--------------
1 file changed, 293 insertions(+), 173 deletions(-)
diff --git a/kadmos/graph/graph_data.py b/kadmos/graph/graph_data.py
index a2c8e5a18..78aa56874 100644
--- a/kadmos/graph/graph_data.py
+++ b/kadmos/graph/graph_data.py
@@ -594,10 +594,7 @@ class DataGraph(KadmosGraph):
# Merge the nodes in the partitions into the super node
for partition in partitions:
for function_id in partition:
- function_graph = nx.contracted_nodes(function_graph, 'super_node', function_id)
-
- # Remove selfloops that arise due to the merging
- function_graph.remove_edges_from(function_graph.selfloop_edges())
+ function_graph = nx.contracted_nodes(function_graph, 'super_node', function_id, self_loops=False)
# Check if all coupled nodes are assigned to a partition
if not nx.is_directed_acyclic_graph(function_graph):
@@ -622,8 +619,8 @@ class DataGraph(KadmosGraph):
for function_id in functions_in_cycle:
if function_id != 'super_node':
coupled_functions.append(function_id)
- function_graph = nx.contracted_nodes(function_graph, 'super_node', function_id)
- function_graph.remove_edges_from(function_graph.selfloop_edges())
+ function_graph = nx.contracted_nodes(function_graph, 'super_node', function_id,
+ self_loops=False)
# Find a topological function order
function_order = list(nx.topological_sort(function_graph))
@@ -641,10 +638,12 @@ class DataGraph(KadmosGraph):
partitions[partition] = nodes
coupled_functions_order.extend(nodes)
self.graph['problem_formulation']['coupled_functions_groups'] = partitions
- else:
+ elif coupled_functions:
coupled_functions_order = self.minimize_feedback(coupled_functions, method, multi_start=multi_start,
rcb=rcb, use_runtime_info=use_runtime_info,
coupling_dict=coupling_dict)
+ else:
+ coupled_functions_order = []
# Get post-coupling functions and sort
post_coupling_functions = function_order[function_order.index('super_node') + 1:]
@@ -783,8 +782,8 @@ class DataGraph(KadmosGraph):
coupling_dict = self.get_coupling_dictionary()
# Get total number of couplings and total runtime
- total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
- function2 in coupling_dict[function1]])
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info else\
len(nodes)
@@ -875,8 +874,8 @@ class DataGraph(KadmosGraph):
'resources')
# Get total number of couplings and total runtime
- total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
- function2 in coupling_dict[function1]])
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info \
else len(nodes)
@@ -914,8 +913,8 @@ class DataGraph(KadmosGraph):
converged = False
# Get total number of couplings and total runtime
- total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
- function2 in coupling_dict[function1]])
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info \
else len(nodes)
@@ -985,8 +984,8 @@ class DataGraph(KadmosGraph):
n_eval = 0
# Get total number of couplings and total runtime
- total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
- function2 in coupling_dict[function1]])
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info \
else len(nodes)
@@ -1046,8 +1045,8 @@ class DataGraph(KadmosGraph):
n_eval = 0
# Get total number of couplings and maximum runtime of graph
- total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
- function2 in coupling_dict[function1]])
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info \
else len(nodes)
@@ -1125,7 +1124,10 @@ class DataGraph(KadmosGraph):
return n_feedback, run_time_branch
- def _genetic_algorithm(self, nodes, coupling_dict):
+ # noinspection PyUnresolvedReferences
+ def _genetic_algorithm(self, nodes, coupling_dict, rcb, use_runtime_info):
+
+ import array
from deap import base
from deap import creator
@@ -1133,18 +1135,27 @@ class DataGraph(KadmosGraph):
from deap import algorithms
# Settings GA
- CXPB = 0.63
- MUTPB = 0.4
- NGEN = 250
- INDPB = 0.11
- pop_size = 300
+ cxpb = 0.5
+ mutpb = 0.65
+ ngen = 750
+ indpb = 0.02
+ pop_size = 250
# Make mapping of nodes to sequence of integers
mapping = copy.deepcopy(nodes)
- # Create fitness and individual class
- creator.create('FitnessMin', base.Fitness, weights=(-1.0,))
- creator.create('Individual', list, fitness=creator.FitnessMin)
+ # Get total number of couplings and maximum runtime of graph
+ total_couplings = max(sum([coupling_dict[function1][function2] for function1 in nodes for function2 in nodes if
+ function2 in coupling_dict[function1]]), 1)
+ total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes]) if use_runtime_info \
+ else len(nodes)
+
+ # Create fitness class based on the optimization objective (= rcb value)
+ creator.create('Fitness', base.Fitness, weights=(-0.99, -0.01)) if rcb in [0, 1] else \
+ creator.create('Fitness', base.Fitness, weights=(-1.0,))
+
+ # Create individual class
+ creator.create('Individual', array.array, typecode='i', fitness=creator.Fitness)
# Get toolbox
toolbox = base.Toolbox()
@@ -1154,10 +1165,11 @@ class DataGraph(KadmosGraph):
toolbox.register('individual', tools.initIterate, creator.Individual, toolbox.indices)
toolbox.register('population', tools.initRepeat, list, toolbox.individual)
toolbox.register('mate', tools.cxOrdered)
- toolbox.register('mutate', tools.mutShuffleIndexes, indpb=INDPB)
- #toolbox.register('select', tools.selBest)
+ toolbox.register('mutate', tools.mutShuffleIndexes, indpb=indpb)
toolbox.register('select', tools.selTournament, tournsize=3)
- toolbox.register('evaluate', self._get_fitness_individual, mapping=mapping, coupling_dict=coupling_dict)
+ toolbox.register('evaluate', self._get_fitness_individual, mapping=mapping, coupling_dict=coupling_dict,
+ rcb=rcb, use_runtime_info=use_runtime_info, total_couplings=total_couplings,
+ total_time=total_time)
population = toolbox.population(pop_size)
hof = tools.HallOfFame(1)
@@ -1166,24 +1178,35 @@ class DataGraph(KadmosGraph):
stats.register("std", np.std)
stats.register("min", np.min)
stats.register("max", np.max)
- algorithms.eaSimple(population, toolbox, CXPB, MUTPB, NGEN, halloffame=hof,
- verbose=True, stats=stats)
+ pop, logbook = algorithms.eaSimple(population, toolbox, cxpb, mutpb, ngen, halloffame=hof, verbose=True,
+ stats=stats)
+ n_eval = sum([logbook[generation]['nevals'] for generation in range(len(logbook))])
+
+ # Get best results
function_order = []
for idx in hof[0]:
function_order.append(mapping[idx])
- return function_order
+ return function_order, n_eval
- def _get_fitness_individual(self, individual, mapping, coupling_dict):
+ def _get_fitness_individual(self, individual, mapping, coupling_dict, rcb, use_runtime_info, total_couplings,
+ total_time):
""" Function to evaluate the fitness of an individual. Needed for the genetic algorithm
"""
function_order = []
for idx in individual:
function_order.append(mapping[idx])
- feedback, time = self.get_feedback_info(function_order, coupling_dict)
+ feedback, time = self.get_feedback_info(function_order, coupling_dict, use_runtime_info)
- return feedback,
+ f = rcb * (feedback / float(total_couplings)) + (1 - rcb) * (time / float(total_time))
+
+ if rcb == 1:
+ return f, time
+ elif rcb == 0:
+ return f, feedback
+ else:
+ return f,
def get_feedback_info(self, function_order, coupling_dict=None, use_runtime_info=False):
"""Function to determine the number of feedback loops and a runtime estimation for a given function order
@@ -1467,8 +1490,11 @@ class RepositoryConnectivityGraph(DataGraph):
self.add_node('D{0}'.format(discipline + 1), category='function', instance=1, function_type='regular')
if 'runtime_range' in kwargs:
runtime_range = kwargs['runtime_range']
- self.nodes['D{0}'.format(discipline + 1)]['performance_info'] = {'run_time': random.randint(
- runtime_range[0], runtime_range[1])}
+ if runtime_range:
+ self.nodes['D{0}'.format(discipline + 1)]['performance_info'] = {'run_time': random.randint(
+ runtime_range[0], runtime_range[1])}
+ else:
+ self.nodes['D{0}'.format(discipline + 1)]['performance_info'] = {'run_time': 1}
for local_constraint in range(n_local_constraints[discipline]): # Local constraints
self.add_node('G{0}_{1}'.format(discipline+1, local_constraint+1), category='function', instance=1,
function_type='regular')
@@ -2760,8 +2786,8 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
if 'doe_runs' in doe_settings:
self.graph['problem_formulation']['doe_settings']['doe_runs'] = doe_settings['doe_runs']
- def partition_graph(self, n_parts, include_run_time=False, tpwgts=None, recursive=True, contig=False,
- local_convergers=False):
+ def partition_graph(self, n_parts, use_runtime_info=False, tpwgts=None, recursive=True, contig=False,
+ local_convergers=False, coupling_dict=None, rcb=1.0, node_selection=None):
"""Partition a graph using the Metis algorithm (http://glaros.dtc.umn.edu/gkhome/metis/metis/overview).
Note that partitioning can only be performed on undirected graphs. Therefore every graph input is translated
@@ -2769,8 +2795,8 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
:param n_parts: number of partitions requested (algorithm might provide less)
:type n_parts: int
- :param include_run_time:
- :type include_run_time: bool
+ :param use_runtime_info:
+ :type use_runtime_info: bool
:param tpwgts: list of target partition weights
:type tpwgts: list
:param recursive: option to use the recursive bisection or k-way partitioning algorithm
@@ -2784,160 +2810,229 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
import metis
# Input assertions
- assert 'function_ordering' in self.graph['problem_formulation'], 'Function ordering is missing'
- pre_couplings = self.graph['problem_formulation']['function_ordering'][self.FUNCTION_ROLES[0]]
- coupled_nodes = self.graph['problem_formulation']['function_ordering'][self.FUNCTION_ROLES[1]]
- post_couplings = self.graph['problem_formulation']['function_ordering'][self.FUNCTION_ROLES[2]]
- if include_run_time:
- for node in coupled_nodes:
- assert 'run_time' in self.nodes[node]['performance_info'], 'Run time missing for function ' \
- '{}'.format(node)
+ if not node_selection:
+ assert 'function_ordering' in self.graph['problem_formulation'], 'Function ordering is missing'
- # Get subgraph
- if 'system_architecture' in self.graph['problem_formulation'] and \
+ # Get coupling dictionary
+ if not coupling_dict:
+ coupling_dict = self.get_coupling_dictionary()
+
+ # Get the nodes that need to be partitioned
+ if node_selection:
+ nodes_to_partition = list(node_selection)
+ # For IDF, all functions in the optimizer loop are partitioned except for the objective and constraint functions
+ elif 'system_architecture' in self.graph['problem_formulation'] and \
self.graph['problem_formulation']['system_architecture'] == self.OPTIONS_ARCHITECTURES[2]:
+
+ # Get post-design-variables, coupled and post-coupling functions
+ mg_function_ordering = self.get_mg_function_ordering()
+ post_des_vars = mg_function_ordering[self.FUNCTION_ROLES[4]]
+ post_couplings = mg_function_ordering[self.FUNCTION_ROLES[2]]
+ coupled_nodes = mg_function_ordering[self.FUNCTION_ROLES[1]]
+
+ # Get objective and constraint functions
constr_obj_vars = self.find_all_nodes(attr_cond=['problem_role', '==', self.PROBLEM_ROLES_VARS[2]]) + \
- self.find_all_nodes(attr_cond=['problem_role', '==', self.PROBLEM_ROLES_VARS[1]])
- constr_obj_funcs = []
- for variable in constr_obj_vars:
- constr_obj_funcs.extend(self.get_sources(variable))
- post_coupling_disciplines = [node for node in post_couplings if node not in constr_obj_funcs]
- pre_coupling_disciplines = [node for node in pre_couplings if node not in constr_obj_funcs]
- subgraph = self.get_subgraph_by_function_nodes(pre_coupling_disciplines + coupled_nodes +
- post_coupling_disciplines)
+ self.find_all_nodes(attr_cond=['problem_role', '==', self.PROBLEM_ROLES_VARS[1]])
+ constr_obj_funcs = [self.get_sources(variable)[0] for variable in constr_obj_vars]
+
+ # Get the nodes that need to be partitioned
+ nodes_to_partition = [node for node in post_des_vars if node not in constr_obj_funcs] + coupled_nodes + \
+ [node for node in post_couplings if node not in constr_obj_funcs]
+ # When a converger is present or no system architecture selected, only the coupled functions are partitioned
else:
- subgraph = self.get_subgraph_by_function_nodes(coupled_nodes)
+ function_ordering = self.graph['problem_formulation']['function_ordering']
+ coupled_nodes = function_ordering[self.FUNCTION_ROLES[1]]
+ nodes_to_partition = coupled_nodes
- graph = subgraph.deepcopy()
- coupling_dict = graph.get_coupling_dictionary()
+ # Check runtime
+ if use_runtime_info:
+ for node in nodes_to_partition:
+ assert 'run_time' in self.nodes[node]['performance_info'], 'Run time missing for function ' \
+ '{}'.format(node)
- best_solution = {'partitions': [], 'variables': float("inf"), 'run_time': float("inf")}
+ # Get initial function graph of the nodes that need to be partitioned
+ subgraph = self.get_subgraph_by_function_nodes(nodes_to_partition)
+ initial_function_graph = subgraph.get_function_graph()
+
+ # Get total number of couplings and the maximum runtime of the graph
+ total_couplings = sum([coupling_dict[function1][function2] for function1 in nodes_to_partition for function2 in
+ nodes_to_partition if function2 in coupling_dict[function1]])
+ total_time = sum([self.nodes[node]['performance_info']['run_time'] for node in nodes_to_partition]) if \
+ use_runtime_info else len(nodes_to_partition)
+
+ # Initialize variables
+ best_partitions = []
+ min_f, min_variables, min_time = float("inf"), float("inf"), float("inf")
number_of_iterations_not_improved = 0
+ function_graph = initial_function_graph.deepcopy()
while True:
+ # Combine coupling strengths of feedforward and feedback connections between two nodes to get an undirected
+ # graph with the correct edge weights
+ remove_edges = []
+ for edge in function_graph.edges():
+ if (edge[0], edge[1]) in remove_edges:
+ continue
+ elif (edge[1], edge[0]) in function_graph.edges():
+ function_graph.edges[edge[0], edge[1]]['coupling_strength'] += function_graph.edges[
+ edge[1], edge[0]]['coupling_strength']
+ remove_edges.append((edge[1], edge[0]))
+ for edge in remove_edges:
+ function_graph.remove_edge(edge[0], edge[1])
# Get undirected graph
- g_und = graph.to_undirected()
+ g_und = function_graph.to_undirected()
- # Add run time to the nodes of the undirected graph
+ # Add runtime to the nodes of the undirected graph for metis
for node in g_und.nodes():
- if g_und.nodes[node]['category'] == 'variable':
- g_und.nodes[node]['run_time'] = 0
- elif g_und.nodes[node]['category'] == 'function':
- g_und.nodes[node]['run_time'] = g_und.nodes[node]['performance_info']['run_time'] if \
- include_run_time else 1
+ g_und.nodes[node]['run_time'] = g_und.nodes[node]['performance_info']['run_time'] if \
+ use_runtime_info else 1
+
+ # Set the runtime as node weight and the coupling strength as edge weight for metis
g_und.graph['node_weight_attr'] = 'run_time'
+ g_und.graph['edge_weight_attr'] = 'coupling_strength'
- # Partition graph
+ # Partition graph using metis
(edgecuts, parts) = metis.part_graph(g_und, n_parts, tpwgts=tpwgts, recursive=recursive, contig=contig)
- # Get partition dict
+ # Create a list with the nodes in each partition
partitions = []
for part in range(n_parts):
+
+ # Get nodes in this partition
nodes = []
- # Get function nodes in partition
for idx, node in enumerate(g_und.nodes):
- if parts[idx] == part and graph.nodes[node]['category'] == 'function':
+ if parts[idx] == part:
nodes.extend(node.split('--') if '--' in node else [node])
+
# Minimize feedback within the partition
- graph_partition = self.get_subgraph_by_function_nodes(nodes)
- if local_convergers and not nx.is_directed_acyclic_graph(graph_partition):
- nodes = self.get_possible_function_order('single-swap', node_selection=nodes)
+ if not nodes:
+ logger.warning('Metis returned less than {} partitions. Some partitions will be empty'.format(
+ n_parts))
else:
- nodes = self.minimize_feedback(nodes, 'single-swap')
- # Add nodes to the partition dict
- partitions.append(nodes)
+ nodes = self.get_possible_function_order('single-swap', node_selection=nodes, rcb=rcb,
+ coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info) if local_convergers \
+ else self.minimize_feedback(nodes, 'single-swap', rcb=rcb, coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info)
- # Reset graph
- graph = subgraph.deepcopy()
+ # Add nodes to the partition list
+ partitions.append(nodes)
# Evaluate the properties of the partitioning
- partition_variables, system_variables, runtime = graph.get_partition_info(partitions,
- include_run_time=include_run_time)
+ partition_variables, system_variables, runtime = subgraph.get_partition_info(
+ partitions, coupling_dict=coupling_dict, use_runtime_info=use_runtime_info,
+ local_convergers=local_convergers)
+ n_variables = system_variables + sum(partition_variables)
- # Merge nodes that can be merged based on process and calculate runtime of each partition
+ # Decide whether new solution is better than the best solution found so far
+ f = rcb * (n_variables / float(total_couplings)) + (1 - rcb) * (max(runtime) / float(total_time))
+ if (n_variables == min_variables and max(runtime) < min_time) or \
+ (max(runtime) == min_time and n_variables < min_variables) or (f < min_f):
+ best_partitions, min_f, min_variables, min_time = partitions, f, n_variables, max(runtime)
+ number_of_iterations_not_improved = 0
+ else:
+ number_of_iterations_not_improved += 1
+
+ # If the third iteration does not give an improvement the iterations are stopped
+ if number_of_iterations_not_improved > 2:
+ break
+
+ # Merge the nodes that can be merged based on process
+ function_graph = initial_function_graph.deepcopy()
for partition in partitions:
nodes = list(partition)
while nodes:
merge_nodes, run_times = [], []
for idx, node in enumerate(nodes):
+ # If the nodes before the current node do not supply input to the current node, the nodes can
+ # be merged
if not set(nodes[:idx]).intersection(coupling_dict[node]):
merge_nodes.append(node)
- run_times.append(self.nodes[node]['performance_info']['run_time'] if include_run_time else
+ run_times.append(self.nodes[node]['performance_info']['run_time'] if use_runtime_info else
1)
else:
break
if len(merge_nodes) > 1:
- new_node = '--'.join(merge_nodes)
+ new_node_label = '--'.join(merge_nodes)
try:
- graph = graph.merge_parallel_functions(merge_nodes, new_label=new_node)
- graph.nodes[new_node]['performance_info'] = {'run_time': max(run_times)}
+ function_graph = function_graph.merge_parallel_functions(merge_nodes,
+ new_label=new_node_label)
+ function_graph.nodes[new_node_label]['performance_info'] = {'run_time': max(run_times)}
except AssertionError:
pass
for node in merge_nodes:
nodes.pop(nodes.index(node))
- n_variables = len(system_variables) + sum([len(variables) for variables in partition_variables])
-
- # Decide whether new solution is better than the best solution found so far
- accept_solution = False
- variable_change = abs((best_solution['variables'] - n_variables)/float(best_solution['variables']))
- run_time_change = abs((best_solution['run_time'] - max(runtime))/float(best_solution['run_time']))
- if n_variables <= best_solution['variables']:
- if max(runtime) < best_solution['run_time']:
- accept_solution = True
- else:
- if variable_change*1.5 > run_time_change:
- accept_solution = True
- elif max(runtime) <= best_solution['run_time']:
- if run_time_change > variable_change*1.5:
- accept_solution = True
-
- if accept_solution:
- best_solution = {'partitions': partitions, 'variables': n_variables, 'run_time': max(runtime)}
- number_of_iterations_not_improved = 0
- else:
- number_of_iterations_not_improved += 1
-
- # Remember current partition
- if accept_solution:
- best_solution['partitions'] = partitions
- best_solution['variables'] = n_variables
- best_solution['runtime'] = max(runtime)
-
- # If the third iteration does not give an improvement the iterations are stopped
- if number_of_iterations_not_improved > 2:
- break
+ # Get correct coupling strengths in case merged nodes exist in the graph
+ for node1 in function_graph.nodes():
+ for node2 in function_graph.nodes():
+ coupling_strength = 0
+ source_nodes = node1.split('--') if '--' in node1 else [node1]
+ target_nodes = node2.split('--') if '--' in node2 else [node2]
+ for source in source_nodes:
+ for target in target_nodes:
+ if (source, target) in initial_function_graph.edges():
+ coupling_strength += initial_function_graph.edges[source, target][
+ 'coupling_strength']
+ if coupling_strength != 0:
+ function_graph.edges[node1, node2]['coupling_strength'] = coupling_strength
# Add local convergers if there are feedback loops in the partitions
convergers = []
if local_convergers:
- for part_nr, partition in enumerate(best_solution['partitions']):
- subgraph = self.get_subgraph_by_function_nodes(partition)
- if not nx.is_directed_acyclic_graph(subgraph):
+ for part_nr, partition in enumerate(best_partitions):
+ converger = False
+ for idx, node in enumerate(partition):
+ if not set(partition[idx:]).intersection(coupling_dict[node]):
+ continue
+ else:
+ converger = True
+ if converger:
convergers.append(part_nr)
- # Update the function order
-
+ # Update the function order # todo: check whether the non partitioned nodes were already in the right sequence?
+ if node_selection:
+ function_order = [node for partition in partitions for node in partition]
+ elif 'system_architecture' in self.graph['problem_formulation'] and \
+ self.graph['problem_formulation']['system_architecture'] == self.OPTIONS_ARCHITECTURES[2]:
+ # noinspection PyUnboundLocalVariable
+ pre_desvars_order = self.sort_nodes_for_process(mg_function_ordering[self.FUNCTION_ROLES[3]])
+ partitioned_nodes_order = [node for partition in partitions for node in partition]
+ # noinspection PyUnboundLocalVariable
+ constr_obj_order = self.sort_nodes_for_process(constr_obj_funcs)
+ function_order = pre_desvars_order + partitioned_nodes_order + constr_obj_order
+ else:
+ pre_coupled_order = self.sort_nodes_for_process(function_ordering[self.FUNCTION_ROLES[0]])
+ partitioned_nodes_order = [node for partition in partitions for node in partition]
+ post_coupling_order = self.sort_nodes_for_process(function_ordering[self.FUNCTION_ROLES[2]])
+ function_order = pre_coupled_order + partitioned_nodes_order + post_coupling_order
- function_order = []
+ # Add partition to the input graph
+ if not 'problem_formulation' in self.graph:
+ self.graph['problem_formulation'] = dict()
- self.graph['problem_formulation']['coupled_functions_groups'] = best_solution['partitions']
+ self.graph['problem_formulation']['coupled_functions_groups'] = best_partitions
self.graph['problem_formulation']['local_convergers'] = convergers
self.graph['problem_formulation']['partition_convergence'] = []
self.graph['problem_formulation']['sequence_partitions'] = []
- self.graph['problem_formulation']['function_order'] = function_order
+ if not node_selection:
+ self.graph['problem_formulation']['function_order'] = function_order
return
- def get_partition_info(self, partitions, include_run_time=False, coupling_dict=None):
+ def get_partition_info(self, partitions, coupling_dict=None, use_runtime_info=False, local_convergers=False):
""" Function to get the number of feedback variables in the partitions and the number system variables (feedback
and feedforward variables between partitions)
:param partitions: list which indicates which nodes are in which partition
:type partitions: list
- :param include_run_time:
- :type include_run_time: bool
+ :param coupling_dict:
+ :type coupling_dict: dict
+ :param use_runtime_info:
+ :type use_runtime_info: bool
+ :param local_convergers:
+ :param local_convergers: bool
:return partition_variables:
:rtype partition_variables: list of sets
:return system_variables:
@@ -2948,7 +3043,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
function_order = [func for partition in partitions for func in partition]
# Input assertions
- if include_run_time:
+ if use_runtime_info:
for node in function_order:
assert 'run_time' in self.nodes[node]['performance_info'], 'Run time missing for function ' \
'{}'.format(node)
@@ -2959,46 +3054,57 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
# For each node in the partitions check whether its input comes from the same partition, another partition or
# neither
- partition_variables = [set() for _ in partitions]
- system_variables = set()
+ system_connections = 0
+ partition_connections = []
for partition, nodes in enumerate(partitions):
+ partition_feedback = 0
for idx, target in enumerate(nodes):
for source in coupling_dict[target]:
if source in nodes[idx+1:]:
- paths = nx.all_shortest_paths(self, source, target)
- for path in paths:
- partition_variables[partition].update([path[1].split('/')[-1]])
+ partition_feedback += coupling_dict[target][source]
elif source in function_order and source not in nodes:
- paths = nx.all_shortest_paths(self, source, target)
- for path in paths:
- system_variables.update([path[1].split('/')[-1]])
+ system_connections += coupling_dict[target][source]
+ partition_connections.append(partition_feedback)
- # Calculate run time of each partition
+ # Calculate runtime # todo: can this more simple?
run_time_partitions = []
- for partition in partitions:
- nodes = list(partition)
- run_time_partition = 0
- while nodes:
- parallel_nodes = []
- run_times = []
+ for partition, nodes in enumerate(partitions):
+ if local_convergers and self.check_for_coupling(nodes, only_feedback=True):
+ # Get runtime pre-coupled nodes
+ pre_coupling_nodes = []
for idx, node in enumerate(nodes):
- if not set(nodes[:idx]).intersection(coupling_dict[node]):
- parallel_nodes.append(node)
- if include_run_time:
- run_times.append(self.nodes[node]['performance_info']['run_time'])
- else:
- run_times.append(1)
+ if not set(nodes[idx + 1:]).intersection(coupling_dict[node]):
+ pre_coupling_nodes = nodes[:idx + 1]
+ else:
+ break
+ # Get runtime post-coupled nodes
+ nodes = [node for node in nodes if node not in pre_coupling_nodes]
+ input_nodes = []
+ post_coupling_nodes = []
+ coupled_nodes = list(nodes)
+ input_nodes.extend([node for node in coupling_dict[node] if node in nodes[idx:]] for idx, node in
+ enumerate(nodes))
+ for idx, node in reversed(list(enumerate(nodes))):
+ if node not in input_nodes:
+ post_coupling_nodes = nodes[idx:]
+ coupled_nodes = nodes[:idx]
else:
break
- run_time_partition += max(run_times)
- for node in parallel_nodes:
- nodes.pop(nodes.index(node))
+ run_time_partition = self.get_runtime_sequence(pre_coupling_nodes, coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info) + \
+ self.get_runtime_sequence(coupled_nodes, coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info) + \
+ self.get_runtime_sequence(post_coupling_nodes, coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info)
+ else:
+ run_time_partition = self.get_runtime_sequence(nodes, coupling_dict=coupling_dict,
+ use_runtime_info=use_runtime_info)
run_time_partitions.append(run_time_partition)
- return partition_variables, system_variables, run_time_partitions
+ return partition_connections, system_connections, run_time_partitions
- def select_number_of_partitions(self, partition_range, include_run_time=False, plot_pareto_front=False,
- local_convergers=False):
+ def select_number_of_partitions(self, partition_range, use_runtime_info=False, plot_pareto_front=False,
+ local_convergers=False, coupling_dict=None, rcb=1.0):
""" Function to evaluate the properties of different number of partitions and to select the best one.
:param partition_range: range of number of partitions that need to be evaluated
@@ -3013,7 +3119,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
# Input assertions
assert 'function_ordering' in self.graph['problem_formulation'], 'Function ordering is missing'
coupled_nodes = self.graph['problem_formulation']['function_ordering'][self.FUNCTION_ROLES[1]]
- if include_run_time:
+ if use_runtime_info:
for node in coupled_nodes:
assert 'run_time' in self.nodes[node]['performance_info'], 'Run time missing for function ' \
'{}'.format(node)
@@ -3021,24 +3127,32 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
partition_info = []
partition_results = dict()
+ if not coupling_dict:
+ coupling_dict = self.get_coupling_dictionary()
+
for idx, n_partitions in enumerate(partition_range):
+
# Partition graph
graph = self.deepcopy()
- partitions, convergers = graph.partition_graph(n_partitions,
- include_run_time=include_run_time,
- local_convergers=local_convergers)
+ print 'Number of partitions: ', n_partitions
+ graph.partition_graph(n_partitions, coupling_dict=coupling_dict, use_runtime_info=use_runtime_info,
+ local_convergers=local_convergers, rcb=rcb)
+ partitions = graph.graph['problem_formulation']['coupled_functions_groups']
+ local_convergers = graph.graph['problem_formulation']['local_convergers']
# Evaluate graph
partition_variables, system_variables, runtime = graph.get_partition_info(partitions,
- include_run_time=include_run_time)
- total_var = len(system_variables) + sum([len(variables) for variables in partition_variables])
+ use_runtime_info=use_runtime_info,
+ coupling_dict=coupling_dict,
+ local_convergers=local_convergers)
+ total_var = system_variables + sum(partition_variables)
# Save partition information
- partition_info.append([idx, n_partitions, [len(variables) for variables in partition_variables],
- len(system_variables), total_var, max(runtime)])
+ partition_info.append([idx, n_partitions, partition_variables, system_variables, total_var, max(runtime)])
partition_results[n_partitions] = dict()
partition_results[n_partitions]['partitions'] = partitions
- partition_results[n_partitions]['convergers'] = convergers
+ partition_results[n_partitions]['local_convergers'] = local_convergers
+ partition_results[n_partitions]['function_order'] = graph.graph['problem_formulation']['function_order']
# Print partition information in table
header = ['Option', '# partitions', '# feedback in partitions', '# system variables', 'Total # variables',
@@ -3064,16 +3178,21 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
plt.show()
# Select the number of partitions
- selmsg = 'Please slect number of partitions:'
+ selmsg = 'Please select number of partitions:'
sel = prompting.user_prompt_select_options(*partition_range, message=selmsg, allow_empty=False,
allow_multi=False)
idx = partition_range.index(int(sel[0]))
# Get result
- partitions = partition_results[partition_range[idx]]['partitions']
- convergers = partition_results[partition_range[idx]]['convergers']
+ self.graph['problem_formulation']['coupled_functions_groups'] = partition_results[partition_range[idx]][
+ 'partitions']
+ self.graph['problem_formulation']['local_convergers'] = partition_results[partition_range[idx]][
+ 'local_convergers']
+ self.graph['problem_formulation']['partition_convergence'] = []
+ self.graph['problem_formulation']['sequence_partitions'] = []
+ self.graph['problem_formulation']['function_order'] = partition_results[partition_range[idx]]['function_order']
- return partitions, convergers
+ return
def select_distributed_architecture(self):
""" Function for easy selection of a distributed architecture for a partitioned graph.
@@ -3093,7 +3212,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
partitions = self.graph['problem_formulation']['coupled_functions_groups']
coupling_dict = self.get_coupling_dictionary()
- # Select system architecture
+ # Select system architecture # todo: remove? System acrhictecture should be given before graph is partitioned
system_architectures = ['unconverged-MDA', 'converged-MDA', 'MDF', 'IDF', 'unconverged-OPT']
options = []
for idx, arch in enumerate(system_architectures):
@@ -3148,6 +3267,7 @@ class FundamentalProblemGraph(DataGraph, KeChainMixin):
continue
break
+ # todo: remove or improve
print 'local converger:', local_convergers
print 'system architecture:', system_architecture
print 'jacobi convergence:', jacobi_convergence
--
GitLab