From 9cfa437ff16c103181ca49b1ffef42d6bc459b93 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 16:11:17 +0200 Subject: [PATCH 01/10] Cache source table ID mappings Read each source table once and look up row IDs in memory instead of running one SELECT per linked member. Warn about and skip IDs that are not found instead of aborting. --- relink_common_data_sets.py | 54 ++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 7e70b1b..1091a33 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -90,6 +90,46 @@ def check_rows_for_last_change(list_raw): print("funtion loaded!") +# function to get the mapping 'identifier -> row_id' for a source table (cached): +# ------------------------------------------------------------------------------ + +id_map_cache = {} + +def get_id_map(source_table, source_sql_tag): + """ + returns a dictionary that maps the values in column 'source_sql_tag' to the '_id' of the rows + in table 'source_table'. The table is read only once, later calls use the cached mapping. + """ + key = (source_table, source_sql_tag) + + if key not in id_map_cache: + print("... reading identifiers from table", source_table) + id_map = {} + + sql_limit = 10000 # max. number of rows returned by base.query() + offset = 0 + + while True: + query = f"SELECT _id, `{source_sql_tag}` FROM `{source_table}` LIMIT {sql_limit} OFFSET {offset}" + rows = base.query(query) + + for row in rows: + identifier = row.get(source_sql_tag) + if identifier is not None: + id_map.setdefault(str(identifier), row["_id"]) + + if len(rows) < sql_limit: # last page + break + + offset += sql_limit + + print("... identifiers found in table", source_table, ": ", str(len(id_map))) + id_map_cache[key] = id_map + + return id_map_cache[key] + +print("funtion loaded!") + # function to create linked records (specific for links in Common Data Sets): # ---------------------------------------------------------------------------- @@ -111,6 +151,9 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col print("Column to be linked:", column_to_link_name) print("Column ID is:", column_link_id) + # get the mapping 'identifier -> row_id' for the source table: + id_map = get_id_map(source_table, source_sql_tag) + # for current_row in table_rows[0:1]: # selected rows for testing for current_row in table_rows: # all rows # Note: from the CD, we expect a list exported as comma separated list and need to restore the list @@ -129,15 +172,16 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col members_id_list = members_string.split(", ") # print("members found:", members_id_list) - # 2. use SQL query to get the row_id's of the row that should be linked: + # 2. look up the row_id's of the rows that should be linked: members_row_id_list = [] for member in members_id_list: # print(member) - query = f"SELECT _id, `{source_sql_tag}` FROM `{source_table}` WHERE `{source_sql_tag}` = '{member}'" - rows = base.query(query) # should is a list with one dictionary - rows_dic = rows[0] - member_row_id = rows_dic['_id'] + member_row_id = id_map.get(member) + if member_row_id is None: + print("Warning: no row with", source_sql_tag, "=", member, "found in table", source_table, + "for row:", current_row[ref_label], "with row_id", current_row["_id"]) + continue # print(member_row_id) members_row_id_list.append(member_row_id) From d13e0f99b03ef4f829554db3f9cc2ae129792cfc Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 16:16:00 +0200 Subject: [PATCH 02/10] Print exception when link update fails The error handler printed 'response', which is unset or stale when update_link raises. --- relink_common_data_sets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 1091a33..0112ef0 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -208,7 +208,7 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col except Exception as e: # Ensure to catch any specific exceptions if the SDK specifies them: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(response) + "\n") + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(e) + "\n") # time dealy to avoid api overflow: # time.sleep(upload_delay_time) From 52bc0a0841aa377d650fa53c350392c9e22ccddb Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 16:47:15 +0200 Subject: [PATCH 03/10] Take server URL and API token from context Remove the hard-coded API token. Outside of a base, the environment variables dtable_web_url and api_token must be set. --- relink_common_data_sets.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 0112ef0..fab17de 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -28,9 +28,12 @@ room_table = "CD_Raumdaten_Mutter" # Name of the table with rooms -server_url = "https://seatable.molgen.mpg.de" # Our seatable server +# Server URL and API-token are provided by SeaTable when the script runs in a base. +# Outside of a base, set the environment variables 'dtable_web_url' (e.g. "https://seatable.molgen.mpg.de") +# and 'api_token' (base access API-token, can be created by the base admin only). +server_url = context.server_url # Our seatable server -api_token = "efe8" # Base access API-token (can be created by the base admin only) +api_token = context.api_token # Base access API-token time_check = "no" # yes: check, if a line that was changed recently within From 2d1a0ba71770c8df630f2e14a8cfc6946bc57ab4 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 16:54:00 +0200 Subject: [PATCH 04/10] Split exported ID lists more robustly Accept commas with or without spaces and ignore empty elements. --- relink_common_data_sets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index fab17de..7139f14 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -171,8 +171,8 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col else: # print("updating linked records for row:", current_row[ref_label], "with row_id", current_row["_id"]) - # split the string to get a list: - members_id_list = members_string.split(", ") + # split the string to get a list (ignore whitespace and empty elements): + members_id_list = [m.strip() for m in members_string.split(",") if m.strip()] # print("members found:", members_id_list) # 2. look up the row_id's of the rows that should be linked: From a649158eaf8ca08a2aca10b3a6f4f2bf2b161ce8 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:05:46 +0200 Subject: [PATCH 05/10] Rebuild links only where they differ from the text Compare the row IDs currently linked in each row with the ones given by the exported text field and update the link only if they differ. Creating links is expensive, and unlike the "Last Change" time filter this needs no time stamp column and no time window. It catches every mismatch, including links emptied by a common data set sync or edited by hand, and a repeated run does no updates at all. An empty text field now clears the links instead of leaving them untouched. This change in behavior is intentional. --- relink_common_data_sets.py | 107 ++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 7139f14..025db92 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -157,6 +157,8 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col # get the mapping 'identifier -> row_id' for the source table: id_map = get_id_map(source_table, source_sql_tag) + rows_unchanged = 0 + # for current_row in table_rows[0:1]: # selected rows for testing for current_row in table_rows: # all rows # Note: from the CD, we expect a list exported as comma separated list and need to restore the list @@ -164,57 +166,64 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col members_string = current_row[column_with_string] # print(members_string) - # check for empty fields (Note, if empty, seatable will return "None") + # empty fields (seatable will return "None") clear the links: if members_string is None: - print("no input data found, skipping row:", current_row["_id"]) - pass + members_string = "" + + # split the string to get a list (ignore whitespace and empty elements): + members_id_list = [m.strip() for m in members_string.split(",") if m.strip()] + # print("members found:", members_id_list) + + # 2. look up the row_id's of the rows that should be linked: + members_row_id_list = [] + + for member in members_id_list: + # print(member) + member_row_id = id_map.get(member) + if member_row_id is None: + print("Warning: no row with", source_sql_tag, "=", member, "found in table", source_table, + "for row:", current_row[ref_label], "with row_id", current_row["_id"]) + continue + # print(member_row_id) + members_row_id_list.append(member_row_id) + + # print("row_id's of the members:", members_row_id_list) + + # 3.) skip the row, if the current links already match (creating links is expensive): + current_links = current_row[column_to_link_name] or [] + current_row_id_set = {link["row_id"] for link in current_links} + + if current_row_id_set == set(members_row_id_list): + rows_unchanged += 1 + continue + + # 4.) get row_id for the line in which the link should be created: + current_row_id = current_row["_id"] + + # 5.) create linked records: + try: + response = base.update_link( + link_id = column_link_id, # id of the column in which the link should be created + table_name = table_to_link, # Name of the table in which the link should be created + other_table_name = source_table, # Name of the table that contains the data that should be linked + row_id = current_row_id, # '_id' of the row in which the link should be updated + other_rows_ids = members_row_id_list # '_id' of the row that that shouöd be linked + ) + + # check response (response will be "True", if the the link could be created or False if not): + if response: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"], response) + else: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed with response:", response) - else: - # print("updating linked records for row:", current_row[ref_label], "with row_id", current_row["_id"]) - # split the string to get a list (ignore whitespace and empty elements): - members_id_list = [m.strip() for m in members_string.split(",") if m.strip()] - # print("members found:", members_id_list) - - # 2. look up the row_id's of the rows that should be linked: - members_row_id_list = [] - - for member in members_id_list: - # print(member) - member_row_id = id_map.get(member) - if member_row_id is None: - print("Warning: no row with", source_sql_tag, "=", member, "found in table", source_table, - "for row:", current_row[ref_label], "with row_id", current_row["_id"]) - continue - # print(member_row_id) - members_row_id_list.append(member_row_id) - - # print("row_id's of the members:", members_row_id_list) - - # 3.) get row_id for the line in which the link should be created: - current_row_id = current_row["_id"] - - # 4.) create linked records: - try: - response = base.update_link( - link_id = column_link_id, # id of the column in which the link should be created - table_name = table_to_link, # Name of the table in which the link should be created - other_table_name = source_table, # Name of the table that contains the data that should be linked - row_id = current_row_id, # '_id' of the row in which the link should be updated - other_rows_ids = members_row_id_list # '_id' of the row that that shouöd be linked - ) - - # check response (response will be "True", if the the link could be created or False if not): - if response: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"], response) - else: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed with response:", response) - - except Exception as e: - # Ensure to catch any specific exceptions if the SDK specifies them: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(e) + "\n") - - # time dealy to avoid api overflow: - # time.sleep(upload_delay_time) + except Exception as e: + # Ensure to catch any specific exceptions if the SDK specifies them: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(e) + "\n") + + # time dealy to avoid api overflow: + # time.sleep(upload_delay_time) + + print("... rows with unchanged links:", rows_unchanged) # specify for testing: #table_to_link = "CD_OrgEinheit_Mutter" From 7d07715bb91d7a0d53e44f8944f4e8441a88a9a1 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:15:23 +0200 Subject: [PATCH 06/10] Remove the time based row filter Comparing links with the text fields makes the "Last Change" filter unnecessary. Drop it together with the task_status flag it controlled. --- relink_common_data_sets.py | 263 ++++++++++--------------------------- 1 file changed, 68 insertions(+), 195 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 025db92..1d2aeaf 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -5,16 +5,14 @@ # - CD_Stammdatenmutter: OrgEinh(Link) # - CD_Raumdaten_Mutter: OrgEinh(Link) # -# - Script can perform time check to update only rows that have been changed -# within a certain time limit -# +# - Links are only updated, if they differ from the exported text fields +# # version tm v2.0, 08.10.2026, running and tested! # # ---- FREEZE THIS VERSION ------- # ------------------------------------------------------------------------------------------------------ # import requried libraries: from seatable_api import Base, context -from datetime import datetime, timedelta import time # set parameters: @@ -35,10 +33,6 @@ api_token = context.api_token # Base access API-token -time_check = "no" # yes: check, if a line that was changed recently within - -time_cut_off = 2 # cut-off limit for last change (in days) - upload_delay_time = 0.5 # delay between Seatbale requets in seconds (worked with 0.5) print("... parameters set!") @@ -47,52 +41,11 @@ # now it should run # ---------------------------- -# for time check, calculate time cut-off: -cut_off = timedelta(days=time_cut_off) - # access the Seatable base using the API-token: print("... accessing data:") base = Base(api_token, server_url) base.auth() -# function to filter raw data for rows that have been changed within the time limit: -# ---------------------------------------------------------------------------------- - -def check_rows_for_last_change(list_raw): - filtered_rows = [] - - # for row in list_raw[0:1]: - for row in list_raw: - # print(row["_id"], row["LastChanged_in_CD"]) - time_flag = "go" - - # skip if 'LastChanged_in_CD' is empty: - if row["LastChanged_in_CD"] == None: - # print("No time stamp found, skipping line with row_id:", row["_id"]) - pass - - else: - # check for lines that have changed only recently within the time limit: - mtime_string = row['LastChanged_in_CD'] # get '_mtime' of the current row: - # print(mtime_string, type(mtime_string)) - mtime = datetime.fromisoformat(mtime_string) # convert string to datetime object: - # print(mtime, type(mtime)) - now = datetime.now(mtime.tzinfo).replace(microsecond=0) # get current time in isoformat with timezone: - # print(now, type(now)) - if now - mtime > cut_off: - time_flag = "stop" - - if time_flag == "stop": - # print(f"row was not changed within time cut_off, ignoring row {current_row.get('_id')} with _mtime = {mtime_string}") - pass - - else: - filtered_rows.append(row) - - return filtered_rows - -print("funtion loaded!") - # function to get the mapping 'identifier -> row_id' for a source table (cached): # ------------------------------------------------------------------------------ @@ -241,105 +194,76 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col # get data from the the table 'orgeinh_table' (Note: rows will be dictionaries): print("... creating linked records in table:", orgeinh_table) -orgeinh_rows_raw = base.list_rows(orgeinh_table) -print("... rows found in table", orgeinh_table, ": ", str(len(orgeinh_rows_raw))) - -task_status = "go" - -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - orgeinh_rows = check_rows_for_last_change(orgeinh_rows_raw) - - # check, if some rows are left: - if orgeinh_rows: - print("number of rows to update within the time limit:", len(orgeinh_rows)) - - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" - -else: - print("no time check, using all rows!") - orgeinh_rows = orgeinh_rows_raw - -print("task_status is:", task_status) +orgeinh_rows = base.list_rows(orgeinh_table) +print("... rows found in table", orgeinh_table, ": ", str(len(orgeinh_rows))) # for debugging: # for i in orgeinh_rows[0:1]: # print(i) # 1.) Update links for group leaders/Leitung: -if task_status == "go": +print("restoring links for PIs ...") - print("restoring links for PIs ...") +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Leitung(Link)" +column_with_string = "Leitung(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Leitung(Link)" - column_with_string = "Leitung(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" - - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 2.) Update links for secretaries/Sekretariate: -if task_status == "go": - - print("restoring links for secretaries ...") +print("restoring links for secretaries ...") - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Sek(Link)" - column_with_string = "Sekrertariat(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Sek(Link)" +column_with_string = "Sekrertariat(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 3.) Update links for directors: -if task_status == "go": +print("restoring links for directors ...") - print("restoring links for directors ...") +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Dir(Link)" +column_with_string = "Direktor(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Dir(Link)" - column_with_string = "Direktor(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" - - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 4.) Update links for HRAdmins: -if task_status == "go": - - print("restoring links for HR members ...") +print("restoring links for HR members ...") - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "SaBeHR(Link)" - column_with_string = "HRAdmin(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "SaBeHR(Link)" +column_with_string = "HRAdmin(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # get data from the the table 'staff_table' (Note: might be more than 1000 entries, thus we loop print("... creating linked records in table:", staff_table) -staff_rows_raw = [] # final list +staff_rows = [] # final list limit = 1000 # API limit for downloading rows using base.list_rows() start = 0 while True: block_of_rows = base.list_rows(staff_table, view_name = staff_table_view, start=start, limit=limit) - staff_rows_raw.extend(block_of_rows) + staff_rows.extend(block_of_rows) if len(block_of_rows) < limit: # last row break @@ -347,63 +271,35 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col start += limit # for debugging: -# for i in staff_rows_raw[0:1]: -# print(i["Pers-ID"], i['_id'], i["LastChanged_in_CD"]) - -print("... rows found in table", staff_table, ": ", str(len(staff_rows_raw))) - -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -task_status = "go" - -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - staff_rows = check_rows_for_last_change(staff_rows_raw) - - # check, if some rows are left: - if staff_rows: - print("number of rows to update within the time limit:", len(staff_rows)) - - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" - -else: - print("no time check, using all rows!") - staff_rows = staff_rows_raw - -print("task_status is:", task_status) +# for i in staff_rows[0:1]: +# print(i["Pers-ID"], i['_id'], i["OrgEinh(Link)"], i["OrgEinhReal(ID)"]) -# for debugging -#for i in staff_rows[0:1]: -# # print(i["Pers-ID"], i['_id'], i["LastChanged_in_CD"]) -# print(i["OrgEinh(link)"], i["OrgEinhReal(ID)"]) +print("... rows found in table", staff_table, ": ", str(len(staff_rows))) # 5.) Update links for OrgEinh in staff_table "Stammdaten": -if task_status == "go": +print("restoring links for people to the assigned groups ...") - print("restoring links for people to the assigned groups ...") +table_to_link = staff_table # should refer to CD "CD_Stammdaten_Mutter" +table_rows = staff_rows +ref_label = "Pers-ID" +column_to_link_name = "OrgEinh(Link)" +column_with_string = "OrgEinhReal(ID)" # or: "OrgEinh(CD String)" +source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +source_sql_tag = "Org-ID" # or: "OrgEinhName" - table_to_link = staff_table # should refer to CD "CD_Stammdaten_Mutter" - table_rows = staff_rows - ref_label = "Pers-ID" - column_to_link_name = "OrgEinh(Link)" - column_with_string = "OrgEinhReal(ID)" # or: "OrgEinh(CD String)" - source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - source_sql_tag = "Org-ID" # or: "OrgEinhName" - - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # get data from the the table 'room_table' (Note: we have more than 1000 entries and need to loop): print("... creating linked records in table:", room_table) -room_rows_raw = [] # final list +room_rows = [] # final list limit = 1000 # API limit for downloading rows using base.list_rows() start = 0 while True: block_of_rows = base.list_rows(room_table, view_name = "Default View", start=start, limit=limit) - room_rows_raw.extend(block_of_rows) + room_rows.extend(block_of_rows) if len(block_of_rows) < limit: # last page break @@ -411,45 +307,22 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col start += limit # for debugging: -#for i in room_rows_raw[0:1]: -# print(i["R-ID"], i['_id'], i["LastChanged_in_CD"]) - -print("... rows found in table", room_table, ": ", str(len(room_rows_raw))) - -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -task_status = "go" - -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - room_rows = check_rows_for_last_change(room_rows_raw) +#for i in room_rows[0:1]: +# print(i["R-ID"], i['_id']) - # check, if some rows are left: - if room_rows: - print("number of rows to update within the time limit:", len(room_rows)) - - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" - -else: - print("no time check, using all rows!") - room_rows = room_rows_raw - -print("task_status is:", task_status) +print("... rows found in table", room_table, ": ", str(len(room_rows))) # 6.) Update links for groups in room_table "CD_Raumdaten_Mutter": -if task_status == "go": - - print("restoring links for rooms to the assigned groups ...") +print("restoring links for rooms to the assigned groups ...") - table_to_link = room_table # should refer to CD "CD_Raumdaten_Mutter" - table_rows = room_rows - ref_label = "R-ID" - column_to_link_name = "OrgEinh(Link)" - column_with_string = "OrgEinhSeatableID(Exp)" - source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter""CD_OrgEinheit_Mutter" - source_sql_tag = "Org-ID" +table_to_link = room_table # should refer to CD "CD_Raumdaten_Mutter" +table_rows = room_rows +ref_label = "R-ID" +column_to_link_name = "OrgEinh(Link)" +column_with_string = "OrgEinhSeatableID(Exp)" +source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter""CD_OrgEinheit_Mutter" +source_sql_tag = "Org-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) print("... done!") From dda247cee5267b2c6e6cb4bfc7895a89a28d3b6c Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:19:58 +0200 Subject: [PATCH 07/10] Handle cells omitted from rows and check columns Read cells with get(), so cells left out of a row count as empty. As empty cells now clear links, stop with an error if a column name is not found in the table instead of clearing all links of a column. --- relink_common_data_sets.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 1d2aeaf..5661813 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -102,6 +102,12 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col """ print("... restoring links in table", table_to_link) + # stop, if a column is missing (e.g. a typo), as missing cells would clear all links: + column_names = [column["name"] for column in base.list_columns(table_to_link)] + for column_name in (column_to_link_name, column_with_string): + if column_name not in column_names: + raise ValueError(f"column '{column_name}' not found in table '{table_to_link}'") + # get the id of the column in the target table, in which the links should be created: column_link_id = base.get_column_link_id(table_to_link, column_to_link_name) print("Column to be linked:", column_to_link_name) @@ -116,10 +122,10 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col for current_row in table_rows: # all rows # Note: from the CD, we expect a list exported as comma separated list and need to restore the list # 1.) get the string for the member(s) = elements of the list: - members_string = current_row[column_with_string] + members_string = current_row.get(column_with_string) # print(members_string) - # empty fields (seatable will return "None") clear the links: + # empty fields (seatable will return "None" or omit the column) clear the links: if members_string is None: members_string = "" @@ -143,7 +149,7 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col # print("row_id's of the members:", members_row_id_list) # 3.) skip the row, if the current links already match (creating links is expensive): - current_links = current_row[column_to_link_name] or [] + current_links = current_row.get(column_to_link_name) or [] current_row_id_set = {link["row_id"] for link in current_links} if current_row_id_set == set(members_row_id_list): From df63ffeac1368f9a37d07aff96bb3d1a8148c37f Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:20:41 +0200 Subject: [PATCH 08/10] Read all rows of the OrgEinheit table list_rows() returns at most 1000 rows, so read the table in pages like the staff and room tables. --- relink_common_data_sets.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index 5661813..0540468 100644 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -200,7 +200,20 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col # get data from the the table 'orgeinh_table' (Note: rows will be dictionaries): print("... creating linked records in table:", orgeinh_table) -orgeinh_rows = base.list_rows(orgeinh_table) +orgeinh_rows = [] # final list + +limit = 1000 # API limit for downloading rows using base.list_rows() +start = 0 + +while True: + block_of_rows = base.list_rows(orgeinh_table, start=start, limit=limit) + orgeinh_rows.extend(block_of_rows) + + if len(block_of_rows) < limit: # last page + break + + start += limit + print("... rows found in table", orgeinh_table, ": ", str(len(orgeinh_rows))) # for debugging: From ccf1797840144a36118b75fa28d096f324e5540c Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:31:33 +0200 Subject: [PATCH 09/10] Make the script executable --- relink_common_data_sets.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 relink_common_data_sets.py diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py old mode 100644 new mode 100755 index 0540468..bfb0e78 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -1,3 +1,4 @@ +#!/bin/env python # Script restores linked records in the common data set 'CD_OrgEinheit_Mutter' # # Links that will be created: From d482dc8cda35d3392ef965688e6a3311d1c2d4f6 Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Mon, 12 Oct 2026 17:32:16 +0200 Subject: [PATCH 10/10] Strip trailing whitespace --- relink_common_data_sets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py index bfb0e78..c343b66 100755 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -1,6 +1,6 @@ #!/bin/env python -# Script restores linked records in the common data set 'CD_OrgEinheit_Mutter' -# +# Script restores linked records in the common data set 'CD_OrgEinheit_Mutter' +# # Links that will be created: # - CD_OrgEinheit_Mutter: Leitung(Link), Sek(Link), Dir(Link), SaBeHR(Link) # - CD_Stammdatenmutter: OrgEinh(Link) @@ -97,7 +97,7 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col table_rows = list with rows from table 'table_to_link' called via the API ref_label = name of a selected column used as identfier the current row (e.g. the Org-ID or Pers-ID) column_to_link_name = name of the column in which the links should be created - column_with_string = name of the column that contains the data as string (exporterd with the common data set) + column_with_string = name of the column that contains the data as string (exporterd with the common data set) source_table = name of the data, that should be linked source_sql_tag = name of the column in the source table, used as identifier (e.g. the Org-ID or Pers-ID) """