diff --git a/relink_common_data_sets.py b/relink_common_data_sets.py old mode 100644 new mode 100755 index 7e70b1b..c343b66 --- a/relink_common_data_sets.py +++ b/relink_common_data_sets.py @@ -1,20 +1,19 @@ -# Script restores linked records in the common data set 'CD_OrgEinheit_Mutter' -# +#!/bin/env python +# Script restores linked records in the common data set 'CD_OrgEinheit_Mutter' +# # Links that will be created: # - CD_OrgEinheit_Mutter: Leitung(Link), Sek(Link), Dir(Link), SaBeHR(Link) # - CD_Stammdatenmutter: OrgEinh(Link) # - CD_Raumdaten_Mutter: OrgEinh(Link) # -# - Script can perform time check to update only rows that have been changed -# within a certain time limit -# +# - Links are only updated, if they differ from the exported text fields +# # version tm v2.0, 08.10.2026, running and tested! # # ---- FREEZE THIS VERSION ------- # ------------------------------------------------------------------------------------------------------ # import requried libraries: from seatable_api import Base, context -from datetime import datetime, timedelta import time # set parameters: @@ -28,13 +27,12 @@ room_table = "CD_Raumdaten_Mutter" # Name of the table with rooms -server_url = "https://seatable.molgen.mpg.de" # Our seatable server - -api_token = "efe8" # Base access API-token (can be created by the base admin only) - -time_check = "no" # yes: check, if a line that was changed recently within +# Server URL and API-token are provided by SeaTable when the script runs in a base. +# Outside of a base, set the environment variables 'dtable_web_url' (e.g. "https://seatable.molgen.mpg.de") +# and 'api_token' (base access API-token, can be created by the base admin only). +server_url = context.server_url # Our seatable server -time_cut_off = 2 # cut-off limit for last change (in days) +api_token = context.api_token # Base access API-token upload_delay_time = 0.5 # delay between Seatbale requets in seconds (worked with 0.5) @@ -44,49 +42,48 @@ # now it should run # ---------------------------- -# for time check, calculate time cut-off: -cut_off = timedelta(days=time_cut_off) - # access the Seatable base using the API-token: print("... accessing data:") base = Base(api_token, server_url) base.auth() -# function to filter raw data for rows that have been changed within the time limit: -# ---------------------------------------------------------------------------------- - -def check_rows_for_last_change(list_raw): - filtered_rows = [] - - # for row in list_raw[0:1]: - for row in list_raw: - # print(row["_id"], row["LastChanged_in_CD"]) - time_flag = "go" - - # skip if 'LastChanged_in_CD' is empty: - if row["LastChanged_in_CD"] == None: - # print("No time stamp found, skipping line with row_id:", row["_id"]) - pass - - else: - # check for lines that have changed only recently within the time limit: - mtime_string = row['LastChanged_in_CD'] # get '_mtime' of the current row: - # print(mtime_string, type(mtime_string)) - mtime = datetime.fromisoformat(mtime_string) # convert string to datetime object: - # print(mtime, type(mtime)) - now = datetime.now(mtime.tzinfo).replace(microsecond=0) # get current time in isoformat with timezone: - # print(now, type(now)) - if now - mtime > cut_off: - time_flag = "stop" - - if time_flag == "stop": - # print(f"row was not changed within time cut_off, ignoring row {current_row.get('_id')} with _mtime = {mtime_string}") - pass +# function to get the mapping 'identifier -> row_id' for a source table (cached): +# ------------------------------------------------------------------------------ - else: - filtered_rows.append(row) +id_map_cache = {} + +def get_id_map(source_table, source_sql_tag): + """ + returns a dictionary that maps the values in column 'source_sql_tag' to the '_id' of the rows + in table 'source_table'. The table is read only once, later calls use the cached mapping. + """ + key = (source_table, source_sql_tag) + + if key not in id_map_cache: + print("... reading identifiers from table", source_table) + id_map = {} + + sql_limit = 10000 # max. number of rows returned by base.query() + offset = 0 + + while True: + query = f"SELECT _id, `{source_sql_tag}` FROM `{source_table}` LIMIT {sql_limit} OFFSET {offset}" + rows = base.query(query) + + for row in rows: + identifier = row.get(source_sql_tag) + if identifier is not None: + id_map.setdefault(str(identifier), row["_id"]) - return filtered_rows + if len(rows) < sql_limit: # last page + break + + offset += sql_limit + + print("... identifiers found in table", source_table, ": ", str(len(id_map))) + id_map_cache[key] = id_map + + return id_map_cache[key] print("funtion loaded!") @@ -100,74 +97,93 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col table_rows = list with rows from table 'table_to_link' called via the API ref_label = name of a selected column used as identfier the current row (e.g. the Org-ID or Pers-ID) column_to_link_name = name of the column in which the links should be created - column_with_string = name of the column that contains the data as string (exporterd with the common data set) + column_with_string = name of the column that contains the data as string (exporterd with the common data set) source_table = name of the data, that should be linked source_sql_tag = name of the column in the source table, used as identifier (e.g. the Org-ID or Pers-ID) """ print("... restoring links in table", table_to_link) + # stop, if a column is missing (e.g. a typo), as missing cells would clear all links: + column_names = [column["name"] for column in base.list_columns(table_to_link)] + for column_name in (column_to_link_name, column_with_string): + if column_name not in column_names: + raise ValueError(f"column '{column_name}' not found in table '{table_to_link}'") + # get the id of the column in the target table, in which the links should be created: column_link_id = base.get_column_link_id(table_to_link, column_to_link_name) print("Column to be linked:", column_to_link_name) print("Column ID is:", column_link_id) + # get the mapping 'identifier -> row_id' for the source table: + id_map = get_id_map(source_table, source_sql_tag) + + rows_unchanged = 0 + # for current_row in table_rows[0:1]: # selected rows for testing for current_row in table_rows: # all rows # Note: from the CD, we expect a list exported as comma separated list and need to restore the list # 1.) get the string for the member(s) = elements of the list: - members_string = current_row[column_with_string] + members_string = current_row.get(column_with_string) # print(members_string) - # check for empty fields (Note, if empty, seatable will return "None") + # empty fields (seatable will return "None" or omit the column) clear the links: if members_string is None: - print("no input data found, skipping row:", current_row["_id"]) - pass - - else: - # print("updating linked records for row:", current_row[ref_label], "with row_id", current_row["_id"]) - # split the string to get a list: - members_id_list = members_string.split(", ") - # print("members found:", members_id_list) - - # 2. use SQL query to get the row_id's of the row that should be linked: - members_row_id_list = [] - - for member in members_id_list: - # print(member) - query = f"SELECT _id, `{source_sql_tag}` FROM `{source_table}` WHERE `{source_sql_tag}` = '{member}'" - rows = base.query(query) # should is a list with one dictionary - rows_dic = rows[0] - member_row_id = rows_dic['_id'] - # print(member_row_id) - members_row_id_list.append(member_row_id) - - # print("row_id's of the members:", members_row_id_list) - - # 3.) get row_id for the line in which the link should be created: - current_row_id = current_row["_id"] - - # 4.) create linked records: - try: - response = base.update_link( - link_id = column_link_id, # id of the column in which the link should be created - table_name = table_to_link, # Name of the table in which the link should be created - other_table_name = source_table, # Name of the table that contains the data that should be linked - row_id = current_row_id, # '_id' of the row in which the link should be updated - other_rows_ids = members_row_id_list # '_id' of the row that that shouöd be linked - ) - - # check response (response will be "True", if the the link could be created or False if not): - if response: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"], response) - else: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed with response:", response) - - except Exception as e: - # Ensure to catch any specific exceptions if the SDK specifies them: - print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(response) + "\n") - - # time dealy to avoid api overflow: - # time.sleep(upload_delay_time) + members_string = "" + + # split the string to get a list (ignore whitespace and empty elements): + members_id_list = [m.strip() for m in members_string.split(",") if m.strip()] + # print("members found:", members_id_list) + + # 2. look up the row_id's of the rows that should be linked: + members_row_id_list = [] + + for member in members_id_list: + # print(member) + member_row_id = id_map.get(member) + if member_row_id is None: + print("Warning: no row with", source_sql_tag, "=", member, "found in table", source_table, + "for row:", current_row[ref_label], "with row_id", current_row["_id"]) + continue + # print(member_row_id) + members_row_id_list.append(member_row_id) + + # print("row_id's of the members:", members_row_id_list) + + # 3.) skip the row, if the current links already match (creating links is expensive): + current_links = current_row.get(column_to_link_name) or [] + current_row_id_set = {link["row_id"] for link in current_links} + + if current_row_id_set == set(members_row_id_list): + rows_unchanged += 1 + continue + + # 4.) get row_id for the line in which the link should be created: + current_row_id = current_row["_id"] + + # 5.) create linked records: + try: + response = base.update_link( + link_id = column_link_id, # id of the column in which the link should be created + table_name = table_to_link, # Name of the table in which the link should be created + other_table_name = source_table, # Name of the table that contains the data that should be linked + row_id = current_row_id, # '_id' of the row in which the link should be updated + other_rows_ids = members_row_id_list # '_id' of the row that that shouöd be linked + ) + + # check response (response will be "True", if the the link could be created or False if not): + if response: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"], response) + else: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed with response:", response) + + except Exception as e: + # Ensure to catch any specific exceptions if the SDK specifies them: + print("Updating column for row:", current_row[ref_label], "with row_id", current_row["_id"] + " failed! check: " + str(e) + "\n") + + # time dealy to avoid api overflow: + # time.sleep(upload_delay_time) + + print("... rows with unchanged links:", rows_unchanged) # specify for testing: #table_to_link = "CD_OrgEinheit_Mutter" @@ -185,105 +201,89 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col # get data from the the table 'orgeinh_table' (Note: rows will be dictionaries): print("... creating linked records in table:", orgeinh_table) -orgeinh_rows_raw = base.list_rows(orgeinh_table) -print("... rows found in table", orgeinh_table, ": ", str(len(orgeinh_rows_raw))) - -task_status = "go" +orgeinh_rows = [] # final list -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - orgeinh_rows = check_rows_for_last_change(orgeinh_rows_raw) +limit = 1000 # API limit for downloading rows using base.list_rows() +start = 0 - # check, if some rows are left: - if orgeinh_rows: - print("number of rows to update within the time limit:", len(orgeinh_rows)) +while True: + block_of_rows = base.list_rows(orgeinh_table, start=start, limit=limit) + orgeinh_rows.extend(block_of_rows) - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" + if len(block_of_rows) < limit: # last page + break -else: - print("no time check, using all rows!") - orgeinh_rows = orgeinh_rows_raw + start += limit -print("task_status is:", task_status) +print("... rows found in table", orgeinh_table, ": ", str(len(orgeinh_rows))) # for debugging: # for i in orgeinh_rows[0:1]: # print(i) # 1.) Update links for group leaders/Leitung: -if task_status == "go": - - print("restoring links for PIs ...") +print("restoring links for PIs ...") - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Leitung(Link)" - column_with_string = "Leitung(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Leitung(Link)" +column_with_string = "Leitung(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 2.) Update links for secretaries/Sekretariate: -if task_status == "go": +print("restoring links for secretaries ...") - print("restoring links for secretaries ...") +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Sek(Link)" +column_with_string = "Sekrertariat(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Sek(Link)" - column_with_string = "Sekrertariat(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" - - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 3.) Update links for directors: -if task_status == "go": - - print("restoring links for directors ...") +print("restoring links for directors ...") - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "Dir(Link)" - column_with_string = "Direktor(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "Dir(Link)" +column_with_string = "Direktor(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # 4.) Update links for HRAdmins: -if task_status == "go": - - print("restoring links for HR members ...") +print("restoring links for HR members ...") - table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - table_rows = orgeinh_rows - ref_label = "Org-ID" - column_to_link_name = "SaBeHR(Link)" - column_with_string = "HRAdmin(Exp)" - source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" - source_sql_tag = "Pers-ID" +table_to_link = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +table_rows = orgeinh_rows +ref_label = "Org-ID" +column_to_link_name = "SaBeHR(Link)" +column_with_string = "HRAdmin(Exp)" +source_table = staff_table # should refer to CD "CD_Stammdaten_Mutter" +source_sql_tag = "Pers-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # get data from the the table 'staff_table' (Note: might be more than 1000 entries, thus we loop print("... creating linked records in table:", staff_table) -staff_rows_raw = [] # final list +staff_rows = [] # final list limit = 1000 # API limit for downloading rows using base.list_rows() start = 0 while True: block_of_rows = base.list_rows(staff_table, view_name = staff_table_view, start=start, limit=limit) - staff_rows_raw.extend(block_of_rows) + staff_rows.extend(block_of_rows) if len(block_of_rows) < limit: # last row break @@ -291,63 +291,35 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col start += limit # for debugging: -# for i in staff_rows_raw[0:1]: -# print(i["Pers-ID"], i['_id'], i["LastChanged_in_CD"]) - -print("... rows found in table", staff_table, ": ", str(len(staff_rows_raw))) - -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -task_status = "go" - -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - staff_rows = check_rows_for_last_change(staff_rows_raw) +# for i in staff_rows[0:1]: +# print(i["Pers-ID"], i['_id'], i["OrgEinh(Link)"], i["OrgEinhReal(ID)"]) - # check, if some rows are left: - if staff_rows: - print("number of rows to update within the time limit:", len(staff_rows)) - - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" - -else: - print("no time check, using all rows!") - staff_rows = staff_rows_raw - -print("task_status is:", task_status) - -# for debugging -#for i in staff_rows[0:1]: -# # print(i["Pers-ID"], i['_id'], i["LastChanged_in_CD"]) -# print(i["OrgEinh(link)"], i["OrgEinhReal(ID)"]) +print("... rows found in table", staff_table, ": ", str(len(staff_rows))) # 5.) Update links for OrgEinh in staff_table "Stammdaten": -if task_status == "go": - - print("restoring links for people to the assigned groups ...") +print("restoring links for people to the assigned groups ...") - table_to_link = staff_table # should refer to CD "CD_Stammdaten_Mutter" - table_rows = staff_rows - ref_label = "Pers-ID" - column_to_link_name = "OrgEinh(Link)" - column_with_string = "OrgEinhReal(ID)" # or: "OrgEinh(CD String)" - source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" - source_sql_tag = "Org-ID" # or: "OrgEinhName" +table_to_link = staff_table # should refer to CD "CD_Stammdaten_Mutter" +table_rows = staff_rows +ref_label = "Pers-ID" +column_to_link_name = "OrgEinh(Link)" +column_with_string = "OrgEinhReal(ID)" # or: "OrgEinh(CD String)" +source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter" +source_sql_tag = "Org-ID" # or: "OrgEinhName" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) # get data from the the table 'room_table' (Note: we have more than 1000 entries and need to loop): print("... creating linked records in table:", room_table) -room_rows_raw = [] # final list +room_rows = [] # final list limit = 1000 # API limit for downloading rows using base.list_rows() start = 0 while True: block_of_rows = base.list_rows(room_table, view_name = "Default View", start=start, limit=limit) - room_rows_raw.extend(block_of_rows) + room_rows.extend(block_of_rows) if len(block_of_rows) < limit: # last page break @@ -355,45 +327,22 @@ def restore_links(table_to_link, table_rows, ref_label, column_to_link_name, col start += limit # for debugging: -#for i in room_rows_raw[0:1]: -# print(i["R-ID"], i['_id'], i["LastChanged_in_CD"]) - -print("... rows found in table", room_table, ": ", str(len(room_rows_raw))) +#for i in room_rows[0:1]: +# print(i["R-ID"], i['_id']) -# filter raw data for time stamp, use only rows that have been changed within the time cut-off: -task_status = "go" - -if time_check == "yes": - print("... filtering for rows with a '_mtime' within the time cut_off limit of:", cut_off) - room_rows = check_rows_for_last_change(room_rows_raw) - - # check, if some rows are left: - if room_rows: - print("number of rows to update within the time limit:", len(room_rows)) - - else: - print("Warning: ... no rows to update, check time limit!") - task_status = "stop" - -else: - print("no time check, using all rows!") - room_rows = room_rows_raw - -print("task_status is:", task_status) +print("... rows found in table", room_table, ": ", str(len(room_rows))) # 6.) Update links for groups in room_table "CD_Raumdaten_Mutter": -if task_status == "go": - - print("restoring links for rooms to the assigned groups ...") +print("restoring links for rooms to the assigned groups ...") - table_to_link = room_table # should refer to CD "CD_Raumdaten_Mutter" - table_rows = room_rows - ref_label = "R-ID" - column_to_link_name = "OrgEinh(Link)" - column_with_string = "OrgEinhSeatableID(Exp)" - source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter""CD_OrgEinheit_Mutter" - source_sql_tag = "Org-ID" +table_to_link = room_table # should refer to CD "CD_Raumdaten_Mutter" +table_rows = room_rows +ref_label = "R-ID" +column_to_link_name = "OrgEinh(Link)" +column_with_string = "OrgEinhSeatableID(Exp)" +source_table = orgeinh_table # should refer to CD "CD_OrgEinheit_Mutter""CD_OrgEinheit_Mutter" +source_sql_tag = "Org-ID" - restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) +restore_links(table_to_link, table_rows, ref_label, column_to_link_name, column_with_string, source_table, source_sql_tag) print("... done!")