From 96863d40f2f22185d0e86d4ab0661f84c20b8140 Mon Sep 17 00:00:00 2001 From: crugas Date: Fri, 7 May 2021 11:17:36 -0400 Subject: [PATCH] Fixed merging issue with UGA_Remote --- as_xtf_GUI.py | 109 +++++++++++++++++--------------------------------- xtf_upload.py | 8 +--- 2 files changed, 38 insertions(+), 79 deletions(-) diff --git a/as_xtf_GUI.py b/as_xtf_GUI.py index 1ab7f72..96e8fb3 100644 --- a/as_xtf_GUI.py +++ b/as_xtf_GUI.py @@ -56,8 +56,8 @@ def run_gui(defaults): # For XTF Users Only rid_box_len = 36 if xtf_version is True: - xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path, xtf_lazy_path, close_program_xtf \ - = get_xtf_log(defaults, login=True) + xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path, close_program_xtf = \ + get_xtf_log(defaults, login=True) if close_program_xtf is True: sys.exit() xtf_login_menu_button = 'Change XTF Login Credentials' @@ -66,8 +66,7 @@ def run_gui(defaults): else: xtf_login_menu_button = '!Change XTF Login Credentials' xtf_opt_button = '!Change XTF Options' - xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path, xtf_lazy_path = "", "", "", \ - "", "", "" + xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path = "", "", "", "", "" cleanup_defaults = ["_ADD_EADID_", "_DEL_NOTES_", "_CLN_EXTENTS_", "_ADD_CERTAIN_", "_ADD_LABEL_", "_DEL_LANGTRAIL_", "_DEL_CONTAIN_", "_ADD_PHYSLOC_", "_DEL_ATIDS_", "_CNT_XLINKS_", "_DEL_NMSPCS_", "_DEL_ALLNS_"] @@ -436,10 +435,9 @@ def run_gui(defaults): get_aspace_log(defaults, xtf_checkbox=False, as_un=as_username, as_pw=as_password, as_ap=as_api, as_client=client, as_repos=repositories, xtf_ver=xtf_version) if event_simple == 'Change XTF Login Credentials': - xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path, xtf_lazy_path, \ - close_program_xtf = get_xtf_log(defaults, login=False, xtf_un=xtf_username, xtf_pw=xtf_password, - xtf_ht=xtf_hostname, xtf_rp=xtf_remote_path, xtf_ip=xtf_indexer_path, - xtf_lp=xtf_lazy_path) + xtf_username, xtf_password, xtf_hostname, xtf_remote_path, xtf_indexer_path, close_program_xtf = \ + get_xtf_log(defaults, login=False, xtf_un=xtf_username, xtf_pw=xtf_password, xtf_ht=xtf_hostname, + xtf_rp=xtf_remote_path, xtf_ip=xtf_indexer_path) # ------------------- HELP ------------------- if event_simple == "About": window_about_active = True @@ -490,8 +488,8 @@ def run_gui(defaults): if event_upl == "_UPLOAD_TO_XTF_": xtfup_thread = threading.Thread(target=upload_files_xtf, args=(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, - xtf_indexer_path, xtf_lazy_path, - values_upl, window_simple,)) + xtf_indexer_path, values_upl, + window_simple,)) xtfup_thread.start() window_simple[f'{"_UPLOAD_"}'].update(disabled=True) window_simple[f'{"_INDEX_"}'].update(disabled=True) @@ -502,7 +500,7 @@ def run_gui(defaults): window_del_active = True print("Getting remote files, this may take a second...", flush=True, end="") remote_files = get_remote_files(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, - xtf_indexer_path, xtf_lazy_path, window_simple) + xtf_indexer_path, window_simple) print("Done") delete_options_layout = [[sg.Button(" Delete from XTF ", key="_DELETE_XTF_", disabled=False), sg.Text(" " * 62)], @@ -524,8 +522,8 @@ def run_gui(defaults): if event_del == "_DELETE_XTF_": xtfup_thread = threading.Thread(target=delete_files_xtf, args=(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, - xtf_indexer_path, xtf_lazy_path, - values_del, window_simple,)) + xtf_indexer_path, values_del, + window_simple,)) xtfup_thread.start() window_simple[f'{"_UPLOAD_"}'].update(disabled=True) window_simple[f'{"_INDEX_"}'].update(disabled=True) @@ -534,8 +532,7 @@ def run_gui(defaults): window_del_active = False if event_simple == "_INDEX_": xtfind_thread = threading.Thread(target=index_xtf, args=(defaults, xtf_hostname, xtf_username, xtf_password, - xtf_remote_path, xtf_indexer_path, - xtf_lazy_path, window_simple,)) + xtf_remote_path, xtf_indexer_path, window_simple,)) xtfind_thread.start() window_simple[f'{"_UPLOAD_"}'].update(disabled=True) window_simple[f'{"_INDEX_"}'].update(disabled=True) @@ -663,7 +660,7 @@ def get_aspace_log(defaults, xtf_checkbox, as_un=None, as_pw=None, as_ap=None, a return as_username, as_password, as_api, close_program, client, asp_version, repositories, xtf_version -def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf_rp=None, xtf_ip=None, xtf_lp=None): +def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf_rp=None, xtf_ip=None): """ Gets a user's XTF credentials. @@ -678,7 +675,6 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf xtf_ht (object, optional): the host URL for the XTF instance xtf_rp (object, optional): the path (folder) where a user wants their data to be stored on the XTF host xtf_ip (object, optional): the path (file) where the website indexer is located - xtf_lp (object, optional): the path (folder) where xml.lazy files are stored - for permissions updates Returns: xtf_username (str): user's XTF username @@ -686,7 +682,6 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf xtf_host (str): the host URL for the XTF instance xtf_remote_path (str): the path (folder) where a user wants their data to be stored on the XTF host xtf_indexer_path (str): the path (file) where the website indexer is located - xtf_lazy_path (str): the path (folder) where the xml.lazy files are stored - used to update permissions close_program (bool): if a user exits the popup, this will return true and end run_gui() """ xtf_username = xtf_un @@ -694,7 +689,6 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf xtf_host = xtf_ht xtf_remote_path = xtf_rp xtf_indexer_path = xtf_ip - xtf_lazy_path = xtf_lp if login is True: save_button_xtf = " Save and Continue " else: @@ -707,14 +701,12 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf [sg.Text("XTF password:", font=("Roboto", 11))], [sg.Text("XTF Hostname:", font=("Roboto", 11))], [sg.Text("XTF Remote Path:", font=("Roboto", 11))], - [sg.Text("XTF Indexer Path:", font=("Roboto", 11))], - [sg.Text("XTF Lazy Index Path:", font=("Roboto", 11))]] + [sg.Text("XTF Indexer Path:", font=("Roboto", 11))]] xtflog_col2 = [[sg.InputText(focus=True, key="_XTF_UNAME_")], [sg.InputText(password_char='*', key="_XTF_PWORD_")], [sg.InputText(defaults["xtf_default"]["xtf_host"], key="_XTF_HOSTNAME_")], [sg.InputText(defaults["xtf_default"]["xtf_remote_path"], key="_XTF_REMPATH_")], - [sg.InputText(defaults["xtf_default"]["xtf_indexer_path"], key="_XTF_INDPATH_")], - [sg.InputText(defaults["xtf_default"]["xtf_lazyindex_path"], key="_XTF_LAZYPATH_")]] + [sg.InputText(defaults["xtf_default"]["xtf_indexer_path"], key="_XTF_INDPATH_")]] layout_xtflog = [ [sg.Column(xtflog_col1), sg.Column(xtflog_col2)], [sg.Button(save_button_xtf, bind_return_key=True, key="_SAVE_CLOSE_LOGIN_")] @@ -726,7 +718,7 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf try: remote = xup.RemoteClient(values_xlog["_XTF_HOSTNAME_"], values_xlog["_XTF_UNAME_"], values_xlog["_XTF_PWORD_"], values_xlog["_XTF_REMPATH_"], - values_xlog["_XTF_INDPATH_"], values_xlog["_XTF_LAZYPATH_"]) + values_xlog["_XTF_INDPATH_"]) remote.client = remote.connect_remote() if remote.scp is None: raise Exception(remote.client) @@ -736,13 +728,11 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf xtf_host = values_xlog["_XTF_HOSTNAME_"] xtf_remote_path = values_xlog["_XTF_REMPATH_"] xtf_indexer_path = values_xlog["_XTF_INDPATH_"] - xtf_lazy_path = values_xlog["_XTF_LAZYPATH_"] with open("defaults.json", "w") as defaults_xtf: defaults["xtf_default"]["xtf_host"] = values_xlog["_XTF_HOSTNAME_"] defaults["xtf_default"]["xtf_remote_path"] = values_xlog["_XTF_REMPATH_"] defaults["xtf_default"]["xtf_indexer_path"] = values_xlog["_XTF_INDPATH_"] - defaults["xtf_default"]["xtf_lazyindex_path"] = values_xlog["_XTF_LAZYPATH_"] json.dump(defaults, defaults_xtf) defaults_xtf.close() window_xtflog_active = False @@ -759,7 +749,7 @@ def get_xtf_log(defaults, login=True, xtf_un=None, xtf_pw=None, xtf_ht=None, xtf close_program = True break window_xtfcred.close() - return xtf_username, xtf_password, xtf_host, xtf_remote_path, xtf_indexer_path, xtf_lazy_path, close_program + return xtf_username, xtf_password, xtf_host, xtf_remote_path, xtf_indexer_path, close_program def get_eads(input_ids, defaults, cleanup_options, repositories, client, values_simple, gui_window): @@ -1278,8 +1268,8 @@ def get_contlabels(input_ids, defaults, repositories, client, values_simple, gui gui_window.write_event_value('-CONTLABEL_THREAD-', (threading.current_thread().name,)) -def upload_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, - xtf_lazy_path, values_upl, gui_window): +def upload_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, values_upl, + gui_window): """ Uploads files to XTF. @@ -1290,33 +1280,27 @@ def upload_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_rem xtf_password (str): user's XTF password xtf_remote_path (str): the path (folder) where a user wants their data to be stored on the XTF host xtf_index_path (str): the path (file) where the textIndexer for XTF is - used to run the index - xtf_lazy_path (str): the path (folder) where the xml.lazy files are stored - used to update permissions values_upl (dict): the GUI values a user chose when selecting files to upload to XTF gui_window (PySimpleGUI object): the GUI window used by PySimpleGUI. Used to return an event Returns: None """ - remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path) + remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path) print("Uploading files...") xtf_files = fetch_local_files(defaults["xtf_default"]["xtf_local_path"], values_upl["_SELECT_FILES_"]) upload_output = remote.bulk_upload(xtf_files) print(upload_output) - for file in xtf_files: - update_permissions = remote.execute_commands(['/bin/chmod 664 {}/{}'.format(defaults["xtf_default"]["xtf_remote_path"], - Path(file).name)]) - print(update_permissions) if defaults["xtf_default"]["_REINDEX_AUTO_"] is True: - index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path, - gui_window, xtf_files) + index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, gui_window) else: print("-" * 135) remote.disconnect() gui_window.write_event_value('-XTFUP_THREAD-', (threading.current_thread().name,)) -def delete_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path, - values_del, gui_window): +def delete_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, values_del, + gui_window): """ Delete files from XTF. @@ -1327,14 +1311,13 @@ def delete_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_rem xtf_password (str): user's XTF password xtf_remote_path (str): the path (folder) where a user wants their data to be stored on the XTF host xtf_index_path (str): the path (file) where the textIndexer for XTF is - used to run the index - xtf_lazy_path (str): the path (folder) where the xml.lazy files are stored - used to update permissions values_del (dict): the GUI values a user chose when selecting files to upload to XTF gui_window (PySimpleGUI object): the GUI window used by PySimpleGUI. Used to return an event Returns: None """ - remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path) + remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path) print("Deleting files...") xtf_files = [str(defaults["xtf_default"]["xtf_remote_path"] + "/" + str(file)) for file in values_del["_SELECT_FILES_"]] @@ -1345,16 +1328,14 @@ def delete_files_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_rem print(cmds_output) print("-" * 135) if defaults["xtf_default"]["_REINDEX_AUTO_"] is True: - index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, - xtf_lazy_path, gui_window) + index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, gui_window) remote.disconnect() except Exception as e: print("An error occurred: " + str(e)) gui_window.write_event_value('-XTFDEL_THREAD-', (threading.current_thread().name,)) -def index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path, - gui_window, xtf_files=None): +def index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, gui_window): """ Runs a re-index of all changed or new files in XTF. It is not a clean re-index. @@ -1365,42 +1346,26 @@ def index_xtf(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_pat xtf_password (str): user's XTF password xtf_remote_path (str): the path (folder) where a user wants their data to be stored on the XTF host xtf_index_path (str): the path (file) where the textIndexer for XTF is - used to run the index - xtf_lazy_path (str): the path (folder) where the lazy files are generated from an index, used to set permissions gui_window (PySimpleGUI object): the GUI window used by PySimpleGUI. Used to return an event - xtf_files (list, optional): the list of file paths of files that were uploaded Returns: None """ print("Beginning Re-Index, this may take awhile...") - remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path) - if xtf_files is None: - try: - cmds_output = remote.execute_commands( - ['{} -index default'.format(defaults["xtf_default"]["xtf_indexer_path"]), - '/bin/chmod 664 {}/*'.format(defaults["xtf_default"]["xtf_lazyindex_path"])]) - print(cmds_output) - print("-" * 135) - except Exception as e: - print("An error occurred: " + str(e)) - else: - try: - commands = ['{} -index default'.format(defaults["xtf_default"]["xtf_indexer_path"])] - for file in xtf_files: - lazyfile = Path(file).name + ".lazy" - commands.append('/bin/chmod 664 {}/{}'.format(defaults["xtf_default"]["xtf_lazyindex_path"], lazyfile)) - cmds_output = remote.execute_commands(commands) - print(cmds_output) - print("-" * 135) - except Exception as e: - print("An error occurred: " + str(e)) - remote.disconnect() + remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path) + try: + cmds_output = remote.execute_commands( + ['{} -index default'.format(defaults["xtf_default"]["xtf_indexer_path"])]) + print(cmds_output) + print("-" * 135) + remote.disconnect() + except Exception as e: + print("An error occurred: " + str(e)) gui_window.write_event_value('-XTFIND_THREAD-', (threading.current_thread().name,)) -def get_remote_files(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path, - gui_window): - remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path) +def get_remote_files(defaults, xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, gui_window): + remote = xup.RemoteClient(xtf_hostname, xtf_username, xtf_password, xtf_remote_path, xtf_index_path) remote_files = sort_list(remote.execute_commands( ['ls {}'.format(defaults["xtf_default"]["xtf_remote_path"])]).splitlines()) gui_window.write_event_value('-XTFGET_THREAD-', (threading.current_thread().name,)) diff --git a/xtf_upload.py b/xtf_upload.py index c39cdcb..62b3d2d 100644 --- a/xtf_upload.py +++ b/xtf_upload.py @@ -21,15 +21,13 @@ class RemoteClient: - def __init__(self, xtf_host, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_lazy_path, - xtf_ssh_key=None): + def __init__(self, xtf_host, xtf_username, xtf_password, xtf_remote_path, xtf_index_path, xtf_ssh_key=None): self.host = xtf_host self.user = xtf_username self.password = xtf_password self.ssh_key_filepath = xtf_ssh_key self.remote_path = xtf_remote_path self.index_path = xtf_index_path - self.lazy_path = xtf_lazy_path self.client = None self.scp = None self.__upload_ssh_key() @@ -75,10 +73,6 @@ def connect_remote(self): sftp.stat(self.index_path) except FileNotFoundError: raise AuthenticationException("Index path does not exist") - try: - sftp.stat(self.lazy_path) - except FileNotFoundError: - raise AuthenticationException("Lazy index path does not exist") return self.client except AuthenticationException as error: logger.info('Authentication failed: did you enter the correct username and password?')