add scope parameter to sample info

core_tools/data/SQL/connect.py

 '''
 definition of storage locations and initializer for storage.
 '''
+
+from typing import Optional
+
 from core_tools.data.name_validation import validate_data_identifier_value
 
+
 class SQL_descriptor(object):
     def __init__(self, required=False):
         self.val = None
@@ -15,21 +19,27 @@ class SQL_descriptor(object):
             raise ConnectionError('No sample information provided\n\n** Please check the docs (set up section). \n\n')
         return self.val
 
+
 class sample_info:
     project = SQL_descriptor(True)
     set_up = SQL_descriptor(True)
     sample = SQL_descriptor(True)
+    scope: Optional[str] = None
 
-    def __init__(self, project, set_up, sample):
+    def __init__(self, project, set_up, sample, scope):
         if project is not None:
             validate_data_identifier_value(project)
         if set_up is not None:
             validate_data_identifier_value(set_up)
         if sample is not None:
             validate_data_identifier_value(sample)
+        if scope is not None:
+            validate_data_identifier_value(scope)
+
         sample_info.project = project
         sample_info.set_up = set_up
         sample_info.sample = sample
+        sample_info.scope = scope
 
     def __str__(self) -> str:
         return f"{sample_info.project}: {sample_info.set_up}-{sample_info.sample}"
@@ -81,6 +91,7 @@ class SQL_conn_info_local:
     def __repr__(self):
         return f'{self.__class__}: host {self.host}, port {self.port}, user {self.user}, passwd *, dbname {self.dbname}, readonly {self.readonly}'
 
+
 class SQL_conn_info_remote:
     host = conn_info_descriptor()
     port = conn_info_descriptor()
@@ -100,7 +111,17 @@ class SQL_conn_info_remote:
     def __repr__(self):
         return f'{self.__class__}: host {self.host}, port {self.port}, user {self.user}, passwd *, dbname {self.dbname}, readonly {self.readonly}'
 
-def set_up_local_storage(user, passwd, dbname, project, set_up, sample, readonly=False):
+
+def set_up_local_storage(
+        user,
+        passwd,
+        dbname,
+        project,
+        set_up,
+        sample,
+        scope=None,
+        readonly=False
+):
     '''
     Set up the specification for the datastorage needed to store/retrieve measurements.
 
@@ -112,11 +133,24 @@ def set_up_local_storage(user, passwd, dbname, project, set_up, sample, readonly
         project (str) : project for which the data will be saved
         set_up (str) : set up at which the data has been measured
         sample (str) : sample name
+        scope (str|None) : SQDL scope name, can be None for setups that do not use SQDL.
     '''
     SQL_conn_info_local('localhost', 5432, user, passwd, dbname, readonly)
-    sample_info(project, set_up, sample)
-
-def set_up_remote_storage(host, port, user, passwd, dbname, project, set_up, sample, readonly=False):
+    sample_info(project, set_up, sample, scope)
+
+
+def set_up_remote_storage(
+    host,
+    port,
+    user,
+    passwd,
+    dbname,
+    project,
+    set_up,
+    sample,
+    scope=None,
+    readonly=False
+):
     '''
     Set up the specification for the datastorage needed to store/retrieve measurements.
 
@@ -130,15 +164,28 @@ def set_up_remote_storage(host, port, user, passwd, dbname, project, set_up, sam
         project (str) : project for which the data will be saved
         set_up (str) : set up at which the data has been measured
         sample (str) : sample name
+        scope (str|None) : SQDL scope name, can be None for setups that do not use SQDL.
     '''
     SQL_conn_info_remote(host, port, user, passwd, dbname, readonly)
-    sample_info(project, set_up, sample)
-
-def set_up_local_and_remote_storage(host, port,
-                                    user_local, passwd_local, dbname_local,
-                                    user_remote, passwd_remote, dbname_remote,
-                                    project, set_up, sample,
-                                    local_readonly=False, remote_readonly=False):
+    sample_info(project, set_up, sample, scope)
+
+
+def set_up_local_and_remote_storage(
+    host,
+    port,
+    user_local,
+    passwd_local,
+    dbname_local,
+    user_remote,
+    passwd_remote,
+    dbname_remote,
+    project,
+    set_up,
+    sample,
+    scope=None,
+    local_readonly=False,
+    remote_readonly=False
+):
     '''
     Set up the specification for the datastorage needed to store/retrieve measurements.
 
@@ -157,7 +204,8 @@ def set_up_local_and_remote_storage(host, port,
         project (str) : project for which the data will be saved
         set_up (str) : set up at which the data has been measured
         sample (str) : sample name
+        scope (str|None) : SQDL scope name, can be None for setups that do not use SQDL.
     '''
     SQL_conn_info_local('localhost', 5432, user_local, passwd_local, dbname_local, local_readonly)
     SQL_conn_info_remote(host, port, user_remote, passwd_remote, dbname_remote, remote_readonly)
-    sample_info(project, set_up, sample)
\ No newline at end of file
+    sample_info(project, set_up, sample, scope)

core_tools/data/SQL/model/versions/1_1_0.sql

@@ -58,13 +58,6 @@ CREATE INDEX ON upload_task_queue (coretools_uid);
 CREATE INDEX ON upload_task_queue (is_claimed_by, has_failed);
 CREATE INDEX ON upload_task_queue (is_claimed_by, should_retry);
 
----
-CREATE TABLE coretools_version (
-    major SMALLINT UNIQUE NOT NULL,
-    minor SMALLINT UNIQUE NOT NULL,
-    patch SMALLINT UNIQUE NOT NULL
-);
-
 ---
 CREATE TABLE upload_log (
     idx INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
@@ -100,3 +93,20 @@ CREATE INDEX ON sqdl_file (dataset_index);
 CREATE INDEX ON sqdl_file (sqdl_uuid);
 
 ---
+CREATE TABLE coretools_version (
+    major SMALLINT UNIQUE NOT NULL,
+    minor SMALLINT UNIQUE NOT NULL,
+    patch SMALLINT UNIQUE NOT NULL
+);
+
+INSERT INTO coretools_version (
+    major, minor, patch
+) VALUES (
+    1, 1, 0
+);
+
+--- alterations
+
+ALTER TABLE global_measurement_overview ADD COLUMN scope TEXT;
+ALTER TABLE sample_info_overview ADD COLUMN scope TEXT;
+    

core_tools/data/SQL/queries/dataset_creation_queries.py

@@ -114,14 +114,19 @@ class measurement_overview_queries:
             or not is_valid_info(sample_info.sample)):
             raise Exception(f'Sample info not valid: {sample_info}')
 
+        scope_value = sample_info.scope
+        if not is_valid_info(scope_value):
+            scope_value = None
+
         uuid = generate_uuid()
         # NOTE: column sync_location is abused for migration to new format
         var_names = (
-                'uuid', 'set_up', 'project', 'sample',
+                'uuid', 'set_up', 'scope', 'project', 'sample',
                 'creasted_by', 'exp_name', 'sync_location', 'exp_data_location',
                 'start_time')
+
         var_values = (
-                uuid, str(sample_info.set_up), str(sample_info.project), str(sample_info.sample),
+                uuid, str(sample_info.set_up), scope_value, str(sample_info.project), str(sample_info.sample),
                 SQL_conn_info_local.user, exp_name, 'New measurement_parameters', '',
                 psycopg2.sql.SQL("TO_TIMESTAMP({})").format(psycopg2.sql.Literal(start_time))
                 )

core_tools/startup/all.py

@@ -5,9 +5,9 @@ from datetime import datetime, timedelta
 
 from .config import load_configuration
 from .db_connection import (
-        connect_local_db,
-        connect_remote_db,
-        connect_local_and_remote_db)
+    connect_local_db,
+    connect_remote_db,
+    connect_local_and_remote_db)
 from .sample_info import set_sample_info
 
 logger = logging.getLogger(__name__)
@@ -24,7 +24,12 @@ def _configure_sample(cfg):
     project = cfg['project']
     setup = cfg['setup']
     sample = cfg['sample']
-    set_sample_info(project, setup, sample)
+    try:
+        scope = cfg['scope']
+    except KeyError:
+        logger.warning("Configuration warning: newer versions of core-tools expect the 'scope' config parameter for compatibility with SQDL. Either specify your setup's scope value, or to surpress this warning, add 'scope: ~' to your config file.")
+        scope = None
+    set_sample_info(project, setup, sample, scope)
 
 
 def _connect_to_db(cfg):

core_tools/startup/sample_info.py

@@ -4,20 +4,25 @@ from core_tools.data.SQL.SQL_connection_mgr import SQL_database_manager
 from core_tools.data.SQL.queries.dataset_creation_queries import sample_info_queries
 from core_tools.startup.db_connection import is_connected
 
-def set_sample_info(project=None, setup=None, sample=None):
+
+def set_sample_info(project=None, setup=None, sample=None, scope=None):
     if project is None:
         project = sample_info.project
     else:
         validate_data_identifier_value(project)
+
     if setup is None:
         setup = sample_info.set_up
     else:
         validate_data_identifier_value(setup)
+
     if sample is None:
         sample = sample_info.sample
     else:
         validate_data_identifier_value(sample)
-    sample_info(project, setup, sample)
+
+    sample_info(project, setup, sample, scope)
+
     if is_connected():
         db_mgr = SQL_database_manager()
         if not db_mgr.SQL_conn_info_local.readonly: