adding fast access parameters to the dataset

core_tools/data/SQL/SQL_database_mgr.py

@@ -20,8 +20,10 @@ class SQL_database_manager:
 		if self.conn == None:
 			self.conn = psycopg2.connect(dbname=SQL_conn_info.dbname, user=SQL_conn_info.user, 
 				password=SQL_conn_info.passwd, host=SQL_conn_info.host, port=SQL_conn_info.port)
+			self.__init_database()
 			self.last_commit = time.time()
 
+
 	def __init_database(self):
 		'''
 		check if the database has been set up correctly. Will generate a new overview table

core_tools/data/SQL/buffer_writer.py

@@ -59,7 +59,6 @@ class buffer_writer:
 			# reset writing position
 			self.lobject.seek(self.cursor*8)
 
-
 class buffer_reader:
 	def __init__(self, SQL_conn, oid, shape):
 		'''
@@ -91,3 +90,10 @@ class buffer_reader:
 
 		data = np.frombuffer(binary_data)
 		self.buffer.flat[start:start+data.size] = data
+
+class buffer_reference:
+	'''
+	object in case a user want to take a copy of the reader/writer
+	'''
+	def __init__(self, data):
+		self.buffer = data
\ No newline at end of file

core_tools/data/ds/data_set.py

@@ -19,7 +19,7 @@ def create_new_data_set(experiment_name, *m_params):
         m_param.init_data_dataclass()
         ds.measurement_parameters += [m_param]
         ds.measurement_parameters_raw += m_param.to_SQL_data_structure()
-
+    print(ds.measurement_parameters_raw)
     SQL_mgr = SQL_database_manager()
     SQL_mgr.register_measurement(ds)
 

core_tools/data/ds/data_set_DataMgr.py

-'''
-
-'''
-
 import numpy as np
+import copy
+import string
+
+class m_param_origanizer():
+    def __init__(self, m_param_raw):
+        self.m_param_raw = m_param_raw
+
+    def get(self, key, nth_set):
+        items = self[key]
+        for i in items:
+            if i.nth_set == nth_set:
+                return i
+
+        raise ValueError('m_param with id {} and set {} not found in this data collection.'.format(key, nth_set))
+
+    def __getitem__(self, key):
+        '''
+        gets a list with parameters containing this key
+
+        Returns
+            list<m_param_raw> : raw parameters originating from this id.
+        '''
+        param_s = []
+        for m_param in self.m_param_raw:
+            if m_param.param_id == key:
+                param_s.append(m_param)
+
+        if len(param_s) != 0:
+            return param_s
+        raise ValueError('m_param with id {} not found in this data collection.'.format(key))
+
+    def get_m_param_id(self):
+        '''
+        get the measurement id's
+        '''
+        id_s = set()
+        for m_param in self.m_param_raw:
+            id_s.add(m_param.param_id_m_param)
+
+        return list(id_s)
+
+    def __copy__(self):
+        new_m_param = []
+        for i in self.m_param_raw:
+            new_m_param.append(copy.copy(i))
+
+        return m_param_origanizer(new_m_param)
+
+class data_descriptor: #autogenerate parameter info
+    def __set_name__(self, owner, name): # from python 3.6 (super handy :) )
+        self.name = name
+    def __get__(self, obj, objtype):
+        return getattr(obj.__dict__.get("_dataset_data_description__raw_data"), self.name)
+
+class dataset_data_description():
+    unit = data_descriptor()
+    label = data_descriptor()
+    name = data_descriptor()
+    shape = data_descriptor()
+
+    def __init__(self, name, m_param_raw, m_params_raw_collection):
+        '''
+        Args:
+            m_param_raw (m_param_raw) : pointer to the raw parameter to add
+            m_params_raw_collection (m_param_origanizer) : object containing a representation of all the data in the dataset
+        '''
+        self.name = name
+        self.__raw_data = m_param_raw
+        self.__raw_data_org =  m_params_raw_collection
+        self.__repr_attr_overview = []
+        self.ndim = len(m_param_raw.dependency) + 1
+        self.__populate_data()
+
+    def __populate_data(self):
+        for i in range(len(self.__raw_data.dependency)):
+            repr_attr_overview = []
+            raw_data = self.__raw_data_org[self.__raw_data.dependency[i]]
+
+            for j in range(len(raw_data)): #this is not pretty, but it works..
+                dataDescription = dataset_data_description('', raw_data[j], self.__raw_data_org)
+                
+                name = string.ascii_lowercase[23+i] + str(j+1)
+
+                self.__setattr__(name, dataDescription)
+                if j == 0:
+                    self.__setattr__(string.ascii_lowercase[23+i], dataDescription)
+                    if len(raw_data) == 1:
+                        name = string.ascii_lowercase[23+i]
+                
+                if self.ndim < 3:
+                    self.__setattr__(string.ascii_lowercase[8+i] + str(j+1), dataDescription)
+                    if len(raw_data) == 1:
+                        self.__setattr__(string.ascii_lowercase[8+i], dataDescription)
+                        repr_attr_overview += [(string.ascii_lowercase[8+i], dataDescription)]
+                    else:
+                        repr_attr_overview += [(string.ascii_lowercase[8+i] + str(j+1), dataDescription)]
+                else:
+                    repr_attr_overview += [(name, dataDescription)]
+
+            self.__repr_attr_overview += [repr_attr_overview]
+
+        if self.ndim <= 3:
+            name  = string.ascii_lowercase[23+self.ndim-1]
+        else:
+            name  = string.ascii_lowercase[8+self.ndim-1]
+
+        self.__setattr__(name, self)
+        self.__repr_attr_overview += [[(name, self)]]
 
-def get_m_param_id(m_params_raw):
-    id_s = set()
-    for m_param in m_params_raw:
-        id_s.append(m_param.param_id_m_param)
-
-    return id_s
-
-def get_m_params_from_id(m_params_raw, id_info):
-    m_params = []
-    for m_param in m_param_raw:
-        if m_param.id_info == id_info:
-            m_params.append(m_param)
-
-    return m_params
-
-def construct_low_level_data_structure(m_params_raw):
-    m_param_ids = get_m_param_id(m_params_raw)
-
-    for id_info in m_param_ids:
-        dataset_data_descriptor(m_param_raw, m_params_raw):
-
-
-
-class dataset_data_descriptor():
-    def __init__(self, id_info, m_params_raw):
-        # 1 :: get setpoints
-    
     def __call__(self):
-        return self.x_data
+        return self.__raw_data.data_buffer.data
 
     def average(self, dim):
         pass
@@ -40,53 +117,101 @@ class dataset_data_descriptor():
     def slice(self, dim, i):
         pass
 
+    def __repr__(self):
+        output_print = ""
+        output_print += "| " + "{:<15}".format(self.name) + " | " + "{:<15}".format(self.label) + " | " +  "{:<8}".format(self.unit)+ " | " +  "{:<25}".format(self.shape) + "|\n"
+        for i in self.__repr_attr_overview:
+            for j in i:
+                dataDescription = j[1]
+                if dataDescription.ndim == 1:
+                    output_print +=  "|  " +  "{:<14}".format(j[0]) + " | " +  "{:<15}".format(dataDescription.label) + " | " +   "{:<8}".format(dataDescription.unit)+ " | " +  "{:<25}".format(dataDescription.shape) + "|\n"
+
+        return output_print
+
+class data_set_property_intializer():
+    def __init__(self, m_params):
+        self.__repr_attr_overview = []
+        # m_meas_id's
+        m_id = m_params.get_m_param_id()
+
+        for i in range(len(m_id)): #this is not pretty.
+            n_sets = len(m_params[m_id[i]])
+            repr_attr_overview = []
+            for j in range(n_sets):
+                ds_descript = dataset_data_description('', m_params.get(m_id[i],  j), m_params)
+
+                name = 'm' + str(i+1) + string.ascii_lowercase[j]
+                setattr(self, name, ds_descript)
+
+                if j == 0:
+                    setattr(self, 'm' + str(i+1), ds_descript)
+                
+                if j == 0 and n_sets==1: #consistent printing
+                    repr_attr_overview += [('m' + str(i+1), ds_descript)]
+                    ds_descript.name = 'm' + str(i+1)
+                else:
+                    repr_attr_overview += [(name, ds_descript)]
+                    ds_descript.name = name
+
+            self.__repr_attr_overview += [repr_attr_overview]
 
     def __repr__(self):
-        pass
+        output_print = "DataSet :: my_measurement_name\n\nid = 1256\nTrueID = 1225565471200\n\n"
+        output_print += "| idn             | label           | unit     | size                     |\n"
+        output_print += "---------------------------------------------------------------------------\n"
+        for i in self.__repr_attr_overview:
+            for j in i:
+                output_print += j[1].__repr__()
+                output_print += "\n"
+        output_print += "database : vanderyspen\n"
+        output_print += "set_up : XLD\n"
+        output_print += "project : 6dot\n"
+        output_print += "sample_name : SQ19\n"
+        return output_print
 
-class test():
-    def __init__(self):
-        self.x = dataset_data_descriptor()
-        # self.x = 6    
+if __name__ == '__main__':
+        
 
+    from core_tools.data.ds.data_set_raw import m_param_raw
 
-from core_tools.data.ds.data_set_raw import m_param_raw
+    a = m_param_raw(param_id=1636274596872, nth_set=0, param_id_m_param=1636274596872, setpoint=False, setpoint_local=False, name_gobal='test', name='chan_1', label='keithley 1', unit='pA', dependency=[1635967634696, 1635967635080], shape='[100, 100, 10]', size=100000, oid=16478, data_buffer=None) 
+    b = m_param_raw(param_id=1636274596872, nth_set=1, param_id_m_param=1636274596872, setpoint=False, setpoint_local=False, name_gobal='test', name='chan_2', label='keithley 2', unit='pA', dependency=[1635967634696, 1635967635080, 32719381726400], shape='[100, 100, 10]', size=100000, oid=16479, data_buffer=None) 
+    c = m_param_raw(param_id=16359690863200, nth_set=0, param_id_m_param=1636274596872, setpoint=False, setpoint_local=True, name_gobal='local_var', name='P1', label='P1', unit='m', dependency=[], shape='[10]', size=10, oid=16476, data_buffer=None) 
+    d = m_param_raw(param_id=32719381726400, nth_set=0, param_id_m_param=1636274596872, setpoint=False, setpoint_local=True, name_gobal='local_var', name='P2', label='P2', unit='m', dependency=[], shape='[10]', size=10, oid=16477, data_buffer=None) 
+    e = m_param_raw(param_id=1635967634696, nth_set=0, param_id_m_param=1636274596872, setpoint=True, setpoint_local=False, name_gobal='name1', name='name1', label='B field', unit='mT', dependency=[], shape='[100, 100]', size=10000, oid=16474, data_buffer=None) 
+    f = m_param_raw(param_id=1635967635080, nth_set=0, param_id_m_param=1636274596872, setpoint=True, setpoint_local=False, name_gobal='name2', name='name2', label='P1', unit='mV', dependency=[], shape='[100, 100]', size=10000, oid=16475, data_buffer=None)
 
-a = m_param_raw(param_id=140455866128848, nth_set=0, param_id_m_param=140455866128848, setpoint=False, setpoint_local=False, name_gobal='name3', name='name3', label='name3', unit='', dependency=[140455866119360, 140455866119216], shape='[100, 100]', size=10000, oid=18249,data_buffer=None) 
-b = m_param_raw(param_id=140455866119360, nth_set=0, param_id_m_param=140455866128848, setpoint=True, setpoint_local=False, name_gobal='name1', name='name1', label='name1', unit='', dependency=[], shape='[100, 100]', size=10000, oid=18247,data_buffer=None)
-c = m_param_raw(param_id=140455866119216, nth_set=0, param_id_m_param=140455866128848, setpoint=True, setpoint_local=False, name_gobal='name2', name='name2', label='name2', unit='', dependency=[], shape='[100, 100]', size=10000, oid=18248,data_buffer=None)
+    l = m_param_origanizer([a,b,c,d,e,f])
 
-l = [a,b,c]
 
+    ds = data_set_property_intializer(l)
+    print(ds)
+    # print(ds.m1a.label)
+    # print(ds.m1b.unit)
+    # print(ds.m1b.label)
 
-ds = test()
-print(ds)
-print(ds.x())
-''' representation of the dataset.
+    ''' representation of the dataset.
 
-dataset :: my_measurement_name
+    dataset :: my_measurement_name
 
-id = 1256
-TrueID = 1225565471200
+    id = 1256
+    TrueID = 1225565471200
 
-| idn   | label | unit  | size      |
-------------------------------------- 
-| m1    | 'I1'  | 'A'   | (100,100) |
-| * x   | 'P1'  | 'mV'  | (100,)    |
-| * y   | 'P2'  | 'mV'  | (100,)    |
+    | idn   | label | unit  | size      |
+    ------------------------------------- 
+    | m1    | 'I1'  | 'A'   | (100,100) |
+    | * x   | 'P1'  | 'mV'  | (100,)    |
+    | * y   | 'P2'  | 'mV'  | (100,)    |
 
-| m2    | 'I2'  | 'A'   | (100)     |
-| * x   | 'P1'  | 'mV'  | (100,)    |
+    | m2    | 'I2'  | 'A'   | (100)     |
+    | * x   | 'P1'  | 'mV'  | (100,)    |
 
-| m3a   | 'I2'  | 'A'   | (100)     |
-|  x    | 'P2'  | 'mV'  | (100,)    |
+    | m3a   | 'I2'  | 'A'   | (100)     |
+    |  x    | 'P2'  | 'mV'  | (100,)    |
 
-| m3b   | 'I2'  | 'A'   | (100)     |
-|  x1   | 'P1'  | 'mV'  | (100,)    |
-|  x2   | 'P1'  | 'mV'  | (100,)    |
+    | m3b   | 'I2'  | 'A'   | (100)     |
+    |  x1   | 'P1'  | 'mV'  | (100,)    |
+    |  x2   | 'P1'  | 'mV'  | (100,)    |
 
-database : vanderyspen
-set_up : XLD
-project : 6dot
-sample_name : SQ19
-'''
\ No newline at end of file
+    
+    '''
\ No newline at end of file

core_tools/data/ds/data_set_raw.py

 from core_tools.data.SQL.connector import sample_info
+from core_tools.data.SQL.buffer_writer import buffer_reference
 from dataclasses import dataclass, field
+import copy
 
 @dataclass
 class data_set_raw:
@@ -44,4 +46,10 @@ class m_param_raw:
     shape : str
     size : int
     oid : int
-    data_buffer : any
\ No newline at end of file
+    data_buffer : any
+
+    def __copy__(self):
+        data_buffer = fake_data_buffer(self.data_buffer.buffer)
+        return buffer_reference(copy.copy(self.param_id), copy.copy(self.nth_set), copy.copy(self.param_id_m_param), copy.copy(self.setpoint), 
+            copy.copy(self.setpoint_local), copy.copy(self.name_gobal), copy.copy(self.name), copy.copy(self.label),
+            copy.copy(self.unit), copy.copy(self.dependency), copy.copy(self.shape), copy.copy(self.size), copy.copy(self.oid), data_buffer)
\ No newline at end of file

core_tools/data/lib/data_class.py

@@ -135,8 +135,9 @@ class m_param_dataclass(dataclass_raw_parent):
         data_items = []
 
         data_items += super().to_SQL_data_structure(self.id_info, False, False, self.dependencies)
-        for setpt in self.setpoints_local:
-            data_items += setpt.to_SQL_data_structure(self.id_info, False, True, setpt.dependencies)
+        for setpt_list in self.setpoints_local:
+            for setpt in setpt_list:
+                data_items += setpt.to_SQL_data_structure(self.id_info, False, True, setpt.dependencies)
 
         for setpt in self.setpoints:
             data_items += setpt.to_SQL_data_structure(self.id_info, True, False, setpt.dependencies)
@@ -154,26 +155,26 @@ class m_param_dataclass(dataclass_raw_parent):
         for setpoint in self.setpoints:
             setpoint.generate_data_buffer(setpoint_shape)  
 
-        for setpoint_local in self.setpoints_local:
-            setpoint_local.generate_data_buffer()  
+        for setpoint_local_list in self.setpoints_local:
+            for setpoint_local in setpoint_local_list:
+                setpoint_local.generate_data_buffer()  
 
         self.generate_data_buffer(setpoint_shape)
         self.__initialized = True
 
     @property
     def dependencies(self):
-        # setpoints external depencies
-        dep = []
-        for setpt in self.setpoints:
-            dep.append(setpt.id_info)
-
-        # setpoints internal dependencies
-        dep_tot = []
-        for setpt_loc in self.setpoints_local:
-            dep_tot.append(dep + [setpt_loc.id_info])
-        
-        if dep_tot == []:
-            return [dep]
+        dep_tot= []
+        for i in range(len(self.data)):
+            dep = []
+            for setpt in self.setpoints:
+                dep.append(setpt.id_info)
+
+            if len(self.setpoints_local) > i:
+                for setpt_l in self.setpoints_local[i]:
+                    dep.append(setpt_l.id_info)
+                    
+            dep_tot.append(dep)
 
         return dep_tot
     

core_tools/data/measurement.py

@@ -81,29 +81,26 @@ class Measurement:
             m_param_parameter_spec = m_param_dataclass(id(parameter), parameter.name, 
                 [parameter.name], [parameter.label], [parameter.unit])
 
-            my_setpoints = []
-
         if isinstance(parameter, qc.MultiParameter):
             m_param_parameter_spec = m_param_dataclass(id(parameter), parameter.name, 
                 list(parameter.names), list(parameter.labels), list(parameter.units), list(parameter.shapes))
 
             setpoint_local_parameter_spec = None
             for i in range(len(parameter.setpoint_names)):
-                my_setpoints = []
-                # a bit of a local hack, in setpoints, sometimes copies are made of the setpoint name
-                # this can cause in uniquess of the keys, therefore the extra multiplications (should more or less ensure uniqueness).
-                #cleaner solution?
-                setpoint_local_parameter_spec = setpoint_dataclass(id(parameter.setpoint_names[i][0])*10*(i+1), np.NaN, 
-                    'local_var', list(parameter.setpoint_names[i]), list(parameter.setpoint_labels[i]),
-                    list(parameter.setpoint_units[i]), [], [])
+                my_local_setpoints = []
                 for j in range(len(parameter.setpoints[i])):
+                    # a bit of a local hack, in setpoints, sometimes copies are made of the setpoint name
+                    # this can cause in uniquess of the keys, therefore the extra multiplications (should more or less ensure uniqueness).
+                    #cleaner solution?
+                    setpoint_local_parameter_spec = setpoint_dataclass(id(parameter.setpoint_names[i][j])*10*(i+1), np.NaN, 
+                        'local_var', list(parameter.setpoint_names[i][j]), list(parameter.setpoint_labels[i][j]),
+                        list(parameter.setpoint_units[i][j]), [], [])
                     data_array = parameter.setpoints[i][j]
                     setpoint_local_parameter_spec.data.append(np.asarray(data_array, order='C'))
                     shape = ( parameter.shapes[i][j],)
                     setpoint_local_parameter_spec.shapes.append(shape)
-
-                m_param_parameter_spec.setpoints_local.append(setpoint_local_parameter_spec)
-
+                    my_local_setpoints.append(setpoint_local_parameter_spec)
+                m_param_parameter_spec.setpoints_local.append(my_local_setpoints)
 
 
         for setpoint in setpoints:
@@ -203,7 +200,7 @@ if __name__ == '__main__':
     x = 100
     y = 100
 
-    m_param = m1
+    m_param = m4
     meas = Measurement()
     meas.register_set_parameter(a1, x)
     meas.register_set_parameter(a2, y)