From aafe356c27f502108c8a446a2aa419231668cbdf Mon Sep 17 00:00:00 2001 From: Ratul Hasan <34002411+RaSan147@users.noreply.github.com> Date: Mon, 6 Jan 2025 02:05:53 +0600 Subject: [PATCH] fixed 7z, improved security --- CHANGELOG.MD | 8 + VERSION | 2 +- dev_src/_list_maker.py | 24 +- dev_src/_zipfly_manager.py | 9 +- dev_src/clone.py | 16 +- dev_src/pyroDB.py | 657 ++++++++++++++++++++++++++++++---- dev_src/pyroboxCore.py | 290 +++++++++++---- dev_src/server.py | 3 +- setup.cfg | 4 +- src/_list_maker.py | 24 +- src/_zipfly_manager.py | 8 +- src/pyroDB.py | 709 ++++++++++++++++++++++++++++++++----- src/pyroboxCore.py | 290 +++++++++++---- 13 files changed, 1738 insertions(+), 306 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 533837a..f063765 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,3 +1,11 @@ +# Version 0.9.8 + ## HOTFIX: + * **Core** updated core with python 3.12 http.server guidelines (escaping control characters) + * **Server** Now using better temp directory. + * **Server** `?folder_data` json used to escape character causing `&` -> `& amp;` in folder name + * **Server** Improved Zfunction and better logging + * **Server** Fixed 7z issue since 7z subprocess took ns time to start (but main thread assumed it was done) + # Version 0.9.7 ## HOTFIX: * Fixed requests requirement in the list diff --git a/VERSION b/VERSION index c81aa44..e3e1807 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.7 +0.9.8 diff --git a/dev_src/_list_maker.py b/dev_src/_list_maker.py index ad00017..5beb7de 100644 --- a/dev_src/_list_maker.py +++ b/dev_src/_list_maker.py @@ -215,7 +215,7 @@ def list_directory_html(self:SH, path, user:User, cookie:Union[SimpleCookie, str -def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=None): +def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=None, escape_html=False): """ Helper to produce a directory listing (absent index.html). @@ -223,6 +223,10 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non error). In either case, the headers are sent, making the interface the same as for send_head(). + path: path to the directory + user: User object + cookie: cookie object + escape_html: whether to escape html or not (default: False, since it will be sent as json) """ try: dir_list = scansort(os.scandir(path)) @@ -232,7 +236,10 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non "No permission to list directory") return None - displaypath = self.get_displaypath(self.url_path) + displaypath = self.get_displaypath( + url_path=self.url_path, + escape_html=escape_html + ) title = get_titles(displaypath) @@ -252,6 +259,9 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non #fullname = os.path.join(path, name) name = file.name displayname = linkname = name + + if escape_html: + displayname = html.escape(displayname, quote=False) size=0 # Append / for directories or @ for symbolic links _is_dir_ = True @@ -266,22 +276,22 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non __, ext = posixpath.splitext(name) if ext=='.html': r_li.append('h'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) elif self.guess_type(linkname).startswith('video/'): r_li.append('v'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) elif self.guess_type(linkname).startswith('image/'): r_li.append('i'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) else: r_li.append('f'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) if _is_dir_: r_li.append('d' + urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) s_li.append(size) diff --git a/dev_src/_zipfly_manager.py b/dev_src/_zipfly_manager.py index 7747178..ced93a2 100644 --- a/dev_src/_zipfly_manager.py +++ b/dev_src/_zipfly_manager.py @@ -17,6 +17,10 @@ from collections import OrderedDict import re +from logging import Logger + +logger = Logger(__name__) + from _fs_utils import get_dir_m_time, _get_tree_path_n_size, humanbytes @@ -302,6 +306,9 @@ def process_thread(): t = threading.Thread(target=process_thread) t.start() + time.sleep(0.2) + + while self.running_process: if self.running_process.stdout is None: break @@ -499,7 +506,7 @@ def zip7z_handler(self, path, zid, source_size, zfile_name, err): try: for progress in zip7z_obj.make_zip_with_7z_generator(path, zfile_name): - print([progress]) + logger.info(f"Zipping {[path]}: {[progress]}") self.zip_in_progress[zid] = int(progress[:-1]) except Exception as e: traceback.print_exc() diff --git a/dev_src/clone.py b/dev_src/clone.py index 34a07bf..0f9245e 100644 --- a/dev_src/clone.py +++ b/dev_src/clone.py @@ -247,7 +247,7 @@ def main(): check_exist = "date+" o = None - while not o: + while o not in ["date", "date+", "size", ""]: o = input("""Check exist? [Default: date+] date: download if remote file is update time same as local file date+: download if remote file is newer or same as local file (ignored if local file is newer) @@ -255,13 +255,17 @@ def main(): >>> """) if o in ["date", "date+", "size"]: check_exist = o - else: - o = None + if o == "": + check_exist = "date+" delete_extras = False - o = input("Delete extras? (y/n) [Default: n]: ") - if o.lower() == "y": - delete_extras = True + + o = None + while o not in ["y", "n", ""]: + o = input("Delete extras? (y/n) [Default: n]: ") + if o.lower() == "y": + delete_extras = True + cloner = Cloner() diff --git a/dev_src/pyroDB.py b/dev_src/pyroDB.py index 91e5197..a9a407e 100644 --- a/dev_src/pyroDB.py +++ b/dev_src/pyroDB.py @@ -38,11 +38,13 @@ import io +import json import os import signal import atexit import shutil from collections.abc import Iterable +import sys import time import random from tempfile import NamedTemporaryFile @@ -66,6 +68,13 @@ TABLE = False try: + # Check if msgpack is installed + + # Check if GIL is enabled (Now available in msgpack-1.1.0-cp313-cp313t-win_amd64.whl) + if not getattr(sys, '_is_gil_enabled', lambda: True)(): + os.environ["MSGPACK_PUREPYTHON"] = "1" + # msgpack is not thread safe (yet) + import msgpack # pip install msgpack SAVE_LOAD = True except ImportError: @@ -108,6 +117,8 @@ def __init__(self, location="", auto_dump=True, sig=True): if sig: self.set_sigterm_handler() + self._autodumpdb() + def __getitem__(self, item): """ Syntax sugar for get() @@ -576,10 +587,10 @@ def _int_to_alpha(n): return string class PickleTable(dict): - def __init__(self, file_path="", *args, **kwargs): + def __init__(self, filepath="", *args, **kwargs): """ args: - - filename: path to the db file (default: `""` or in-memory db) + - filepath: path to the db file (default: `""` or in-memory db) - auto_dump: auto dump on change (default: `True`) - sig: Add signal handler for graceful shutdown (default: `True`) """ @@ -590,7 +601,7 @@ def __init__(self, file_path="", *args, **kwargs): self.busy = False - self._pk = PickleDB(file_path, *args, **kwargs) + self._pk = PickleDB(location=filepath, *args, **kwargs) # make the super dict = self._pk.db @@ -710,7 +721,7 @@ def __str__(self, limit:int=None): x += "\t|\t".join(self.column_names_func(rescan=False)) for i in range(min(self.height, limit)): x += "\n" - x += "\t|\t".join(str(self.row(i).values())) + x += "\t|\t".join([str(cell) if cell is not None else '' for cell in self.row(i).values()]) else: x = tabulate( @@ -1384,7 +1395,7 @@ def _add_row(self, row:Union[dict, "_PickleTRow"], position:int="last", rescan=T return self.row_obj_by_id(row_id) - def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True) -> "_PickleTRow": + def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True, rescan=True) -> "_PickleTRow": """ @ locked - row: row must be a dict|_PickleTRow containing column names and values @@ -1399,7 +1410,7 @@ def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True) -> " ``` """ - row_obj = self.lock(self._add_row)(row=row,position=position) + row_obj = self.lock(self._add_row)(row=row,position=position, rescan=rescan) self.auto_dump(AD=AD) @@ -1426,7 +1437,7 @@ def insert_row(self, row:Union[dict, "_PickleTRow"], position:int="last", AD=Tru - def add_row_as_list(self, row:list, position:int="last", AD=True) -> "_PickleTRow": + def add_row_as_list(self, row:list, position:int="last", AD=True, rescan=True) -> "_PickleTRow": """ @ locked - row: row must be a list containing values. (order must match with column names) @@ -1440,7 +1451,7 @@ def add_row_as_list(self, row:list, position:int="last", AD=True) -> "_PickleTRo db.add_row_as_list(["John", 25]) """ - row_obj = self.lock(self._add_row)(row={k:v for k, v in zip(self.column_names, row)}, position=position) + row_obj = self.lock(self._add_row)(row={k:v for k, v in zip(self.column_names, row)}, position=position, rescan=rescan) self.auto_dump(AD=AD) @@ -1578,24 +1589,141 @@ def auto_dump(self, AD=True): """ self._pk._autodumpdb(AD=AD) + def to_json(self, filepath=None, indent=4, format=list) -> str: + """ + Write the table to a json file + - filepath: path to the file (if None, use current filepath.json) (if in memory and not provided, uses "table.json") + - indent: indentation level (default: 4 spaces) + - format: format of the json file (default: list) [options: list|dict] + - list: list of rows [{col1: val1, col2: val2, ...}, ...] + - dict: dict of columns {col1: [val1, val2, ...], col2: [val1, val2, ...], ...} + + - return: path to the file + """ + if filepath is None: + # check filepath + path = self._pk.location + if not path: + path = "table.json" + else: + path = os.path.splitext(path)[0] + ".json" + else: + path = filepath + + + # write to file + with open(path, "w", encoding='utf8') as f: + if format == dict or format == 'dict': + json.dump(self.__db__(), f, indent=indent) + elif format == list or format == 'list': + json.dump(list(self.rows()), f, indent=indent) + else: + raise AttributeError("Invalid format. [expected: list|dict] [got:", format, "]") + + return os.path.realpath(path) + + def to_json_str(self, indent=4) -> str: + """ + Return the table as a json string + """ + return json.dumps(self.__db__(), indent=indent) + + def load_json(self, filepath=None, iostream=None, json_str=None, ignore_new_headers=False, on_file_not_found='error', AD=True): + """ + Load a json file to the table + - WILL OVERWRITE THE EXISTING DATA (To append, make a new table and extend) + - filepath: path to the file + - ignore_new_headers: ignore new headers|columns if found `[when header=True]` (default: `False` and will add new headers) + - on_file_not_found: action to take if the file is not found (default: `'error'`) [options: `error`|`ignore`|`warn`|`no_warning`] + * if `error`, raise FileNotFoundError + * if `ignore`, ignore the operation (no warning, **no clearing**) + * if `no_warning`, ignore the operation, but **clears db** + * if `warn`, print warning and ignore the operation, but **clears db** + """ + + # if more than one source is provided + sources = [i for i in [filepath, iostream, json_str] if i] + if len(sources) > 1: + raise AttributeError(f"Only one source is allowed. Got: {len(sources)}") + + # if no source is provided + if not sources: + raise AttributeError("No source provided") + + if json_str: + # load it as io stream + iostream = io.StringIO(json_str) + + if not ((filepath and os.path.isfile(filepath)) or (iostream and isinstance(iostream, io.IOBase))): + if on_file_not_found == 'error': + raise FileNotFoundError(f"File not found: {filepath}") + + elif on_file_not_found == 'ignore': + return + else: + self.clear(AD=AD) + if on_file_not_found == 'warn': + print(f"File not found: {filepath}. Cleared the table.") + if on_file_not_found == 'no_warning': + pass + + return + + self.clear(AD=False) + + def load_as_io(f): + return json.load(f) + + if iostream: + data = load_as_io(iostream) + else: + with open(filepath, "r", encoding='utf8') as f: + data = load_as_io(f) + + # print(data) + + if not data: + return + + existing_columns = self.column_names_func(rescan=False) + + if isinstance(data, dict): + # column names are keys + self.add(table=data, + add_extra_columns=ignore_new_headers, + AD=AD + ) + + elif isinstance(data, list): + # per row + for row in data: + if not ignore_new_headers: + for col in row: + if col not in existing_columns: + self.add_column(col, exist_ok=True, AD=False, rescan=False) + + self._add_row(row, rescan=False) + + self.auto_dump(AD=AD) + - def to_csv(self, filename=None, write_header=True) -> str: + def to_csv(self, filepath=None, write_header=True) -> str: """ Write the table to a csv file - - filename: path to the file (if None, use current filename.csv) (if in memory and not provided, uses "table.csv") + - filepath: path to the file (if None, use current filepath.csv) (if in memory and not provided, uses "table.csv") - write_header: write column names as header (1st row) (default: `True`) - return: path to the file """ - if filename is None: - # check filename + if filepath is None: + # check filepath path = self._pk.location if not path: path = "table.csv" else: path = os.path.splitext(path)[0] + ".csv" else: - path = filename + path = filepath with open(path, "wb") as f: f.write(b'') @@ -1613,6 +1741,8 @@ def to_csv(self, filename=None, write_header=True) -> str: return os.path.realpath(path) + + def to_csv_str(self, write_header=True) -> str: """ Return the table as a csv string @@ -1629,16 +1759,19 @@ def to_csv_str(self, write_header=True) -> str: return output.getvalue() - def load_csv(self, filename, header=True, ignore_none=False, ignore_new_headers=False, on_file_not_found='error', AD=True): + def load_csv(self, filepath=None, iostream=None, csv_str=None, + header=True, ignore_none=False, ignore_new_headers=False, on_file_not_found='error', AD=True): """ Load a csv file to the table - WILL OVERWRITE THE EXISTING DATA (To append, make a new table and extend) + - filepath: path to the file + - iostream: io stream object (use either filepath or iostream) - header: * if True, the first row will be considered as column names * if False, the columns will be named as "Unnamed-1", "Unnamed-2", ... * if "auto", the columns will be named as "A", "B", "C", ..., "Z", "AA", "AB", ... - ignore_none: ignore the None rows - - ignore_new_headers: ignore new headers if found `[when header=True]` (default: `False`) + - ignore_new_headers: ignore new headers if found `[when header=True]` (default: `False` and will add new headers) - on_file_not_found: action to take if the file is not found (default: `'error'`) [options: `error`|`ignore`|`warn`|`no_warning`] * if `error`, raise FileNotFoundError * if `ignore`, ignore the operation (no warning, **no clearing**) @@ -1648,26 +1781,36 @@ def load_csv(self, filename, header=True, ignore_none=False, ignore_new_headers= columns_names = self.column_names def add_row(row, columns): - if ignore_none and all(v is None for v in row): + if ignore_none and all((v is None or v == '') for v in row): return new_row = {k: v for k, v in zip(columns, row)} - - # print(new_row) - self.add_row(new_row, AD=False) + self.add_row(new_row, rescan=False) + # if more than one source is provided + sources = [i for i in [filepath, iostream, csv_str] if i] + if len(sources) > 1: + raise AttributeError(f"Only one source is allowed. Got: {len(sources)}") + if not sources: + raise AttributeError("No source provided") - if not os.path.exists(filename): + if csv_str: + # load it as io stream + iostream = io.StringIO(csv_str) + + + if not ((filepath and os.path.isfile(filepath)) or (iostream and isinstance(iostream, io.IOBase))): if on_file_not_found == 'error': - raise FileNotFoundError(f"File not found: {filename}") + raise FileNotFoundError(f"File not found: {filepath}") + elif on_file_not_found == 'ignore': return else: self.clear(AD=AD) if on_file_not_found == 'warn': - print(f"File not found: {filename}") + print(f"File not found: {filepath}. Cleared the table.") if on_file_not_found == 'no_warning': pass @@ -1675,12 +1818,15 @@ def add_row(row, columns): self.clear(AD=False) - with open(filename, 'r', encoding='utf8') as f: + + def load_as_io(f): reader = csv.reader(f) if header is True: columns = next(reader) updated_columns = [] n = len(self.column_names_func(rescan=False)) + + for col in columns: if (col is None or col == ""): while f"Unnamed-{n}" in columns_names: @@ -1688,13 +1834,16 @@ def add_row(row, columns): col = f"Unnamed-{n}" n += 1 - if col in columns_names or col in updated_columns: + if not (col in columns_names): if ignore_new_headers: continue + if col in updated_columns: + continue updated_columns.append(col) columns = updated_columns + # print(updated_columns) self.add_column(updated_columns, exist_ok=True, AD=False, rescan=False) @@ -1713,7 +1862,7 @@ def add_row(row, columns): self.add_column(new_columns, exist_ok=True, AD=False, rescan=False) columns = new_columns - add_row(row, columns) + add_row(row, columns) # add the first row else: # count the columns, and name them as "Unnamed-1", "Unnamed-2", ... row = next(reader) @@ -1731,7 +1880,7 @@ def add_row(row, columns): self.add_column(new_columns, exist_ok=True, AD=False, rescan=False) columns = new_columns - add_row(row, columns) + add_row(row, columns) # add the first row @@ -1740,10 +1889,17 @@ def add_row(row, columns): for row in reader: add_row(row, columns) + if filepath: + with open(filepath, 'r', encoding='utf8') as f: + load_as_io(f) + elif iostream: + load_as_io(iostream) + + self.auto_dump(AD=AD) - def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): + def extend(self, other: "PickleTable", add_extra_columns=None, AD=True, rescan=True): """ Extend the table with another table - other: `PickleTable` object @@ -1760,6 +1916,8 @@ def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): if not isinstance(other, type(self)): raise TypeError("Unsupported operand type(s) for +: 'PickleTable' and '{}'".format(type(other).__name__)) + self.rescan(rescan=rescan) + keys = other.column_names this_keys = self.column_names @@ -1775,7 +1933,7 @@ def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): for row in other: - self._add_row({k: row[k] for k in keys}) + self._add_row({k: row[k] for k in keys}, rescan=False) self.auto_dump(AD=AD) @@ -1824,8 +1982,6 @@ def add(self, table:Union["PickleTable", dict], add_extra_columns=None, AD=True) self.auto_dump(AD=AD) - - class _PickleTCell: def __init__(self, source:PickleTable, column, row_id:int, CC): self.source = source @@ -2395,6 +2551,7 @@ def apply(self, func=None, row_func=False, copy=False, AD=True): + if __name__ == "__main__": import string @@ -2473,7 +2630,7 @@ def test(): dt = time.perf_counter() tb.dump() tt = time.perf_counter() - print(f"โฑ๏ธ DUMP time: {tt-dt}s") + print(f"โฑ๏ธ [IN-MEMORY] DUMP time: {tt-dt}s") print("="*50) @@ -2491,16 +2648,10 @@ def test(): # for cell in cells: # print(cell.row_obj()) - print(tabulate(cells, headers="keys", tablefmt="simple_grid")) - - - print("="*50) - print("\n\n๐Ÿ“ TEST convert to CSV") - st = time.perf_counter() - tb.to_csv(f"test{st}.csv") - et = time.perf_counter() - print(f"โฑ๏ธ Convert to csv test in {et - st}s") - os.remove(f"test{st}.csv") + if TABLE: + print(tabulate(cells, headers="keys", tablefmt="simple_grid")) + else: + print(cells, sep="\n") print("="*50) @@ -2743,6 +2894,37 @@ def _update_table(tb:PickleTable): tb.add_column("x", exist_ok=True) # bring back the column print("="*50) + + + print("="*50) + print("\n\n๐Ÿ“ TEST convert to CSV") + + CSV_tb = PickleTable() + CSV_tb.add_column("x", exist_ok=True) + CSV_tb.add_column("Y", exist_ok=True) + + CSV_tb.add_row({"x":1, "Y":2}) + CSV_tb.add_row({"x":3, "Y":None}) + CSV_tb.add_row({"x":"5,5", "Y":6}) + + st = time.perf_counter() + CSV_tb.to_csv(f"test{st}.csv") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to csv test in {et - st}s") + + with open(f"test{st}.csv", "r") as f: + text = f.read() + + if text != """x,Y +1,2 +3, +"5,5",6 +""": + raise Exception("โŒ TEST [CONVERT TO CSV]: Failed (Data Mismatch)") + os.remove(f"test{st}.csv") + + print("="*50) + print("\n\n๐Ÿ“ Load csv test") tb.clear() @@ -2751,26 +2933,137 @@ def _update_table(tb:PickleTable): print(tb) print(tb.column_names) + csv_test_data = """x,Y\n1,2\n3,4\n5,6\n""" + csv_test_iostream = io.StringIO(csv_test_data) + # make a csv file with open("test.csv", "w") as f: - f.write("x,Y\n1,2\n3,4\n5,6\n") + f.write(csv_test_data) - print("\t๐Ÿ“ Normal Load") + print("\t๐Ÿ“ Normal Load [from File ๐Ÿ“„ ]") try: tb.load_csv("test.csv") except Exception as e: - print("โŒ TEST [LOAD CSV]: Failed (Other error)") + print("โŒ TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Failed (Other error)") raise e if tb.height != 3 or tb.column("x") != ['1', '3', '5']: print(tb) print(tb.column("x"), tb.column('x')==[1, 3, 5]) print(tb.height, tb.height==3) - raise Exception("โŒ TEST [LOAD CSV]: Failed") + raise Exception("โŒ TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Failed") + + print("\tโœ… TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Passed") + + print("\t๐Ÿ“ Normal Load [from CSV string ๐Ÿงต ]") + tb.clear() + + tb.load_csv(csv_str=csv_test_data) + + if tb.height != 3 or tb.column("x") != ['1', '3', '5']: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD CSV][Normal Load][from CSV string ๐Ÿงต ]: Failed") + + print("\tโœ… TEST [LOAD CSV][Normal Load][from CSV string ๐Ÿงต ]: Passed") + + print("\t๐Ÿ“ Normal Load [From IOstream ๐Ÿชก ]") + tb.clear() + + tb.load_csv(iostream=csv_test_iostream) + + if tb.height != 3 or tb.column("x") != ['1', '3', '5']: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD CSV][Normal Load][From IOstream ๐Ÿชก ]: Failed") + + print("\tโœ… TEST [LOAD CSV][Normal Load][From IOstream ๐Ÿชก ]: Passed") + + print("\t๐Ÿ“ Multiple source [filepath+csv_str]['error']") + tb.clear() + # 1st error mode on multiple source + try: + tb.load_csv(filepath="test.csv", csv_str=csv_test_data) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + + print("\t๐Ÿ“ Multiple source [filepath+iostream]['error']") + # 2nd error mode on multiple source + tb.clear() + try: + tb.load_csv(filepath="test.csv", iostream=csv_test_iostream) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [iostream+csv_str]['error']") + # 3rd error mode on multiple source + tb.clear() + try: + tb.load_csv(iostream=csv_test_iostream, csv_str=csv_test_data) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [filepath+csv_str+iostream]['error']") + # 4th error mode on multiple source + tb.clear() + try: + tb.load_csv(filepath="test.csv", csv_str=csv_test_data, iostream=csv_test_iostream) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ No source []['error']") + # 5th error mode on no source + tb.clear() + try: + tb.load_csv() + raise Exception("โŒ TEST [LOAD CSV][No source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][No source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][No source]: Failed (Other error)") + raise e + + + + + - print("\tโœ… TEST [LOAD CSV][Normal Load]: Passed") print("\t๐Ÿ“ NoFile Load ['error']") # 1st error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + try: tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="error") raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='error']: Failed") @@ -2778,25 +3071,15 @@ def _update_table(tb:PickleTable): print("\tโœ… TEST [LOAD CSV][NoFile Load]['error'][on_file_not_found='error']: Passed") print("\tError message = ", e) - print("\t๐Ÿ“ NoFile Load ['ignore']") - # 3rd error mode on file not found - try: - tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="ignore") - if tb.height != 3 or tb.column("x") != ['1', '3', '5']: - raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed") - print("\tโœ… TEST [LOAD CSV][NoFile Load]['ignore'][on_file_not_found='ignore']: Passed") - - except FileNotFoundError as e: - print("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (File not found)") - raise e - except Exception as e: - print("\tโŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (Other error)") - raise e - print("\t๐Ÿ“ NoFile Load ['warn']") - # 2nd error mode on file not found + # 2nd error mode on file not found [DB is cleared] + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + try: tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="warn") if tb.height: # must be 0 @@ -2811,9 +3094,36 @@ def _update_table(tb:PickleTable): print("\tโŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='warn']: Failed (Other error)") raise e + + print("\t๐Ÿ“ NoFile Load ['ignore']") + # 3rd error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + + try: + tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="ignore") + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb.columns()) + raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed") + print("\tโœ… TEST [LOAD CSV][NoFile Load]['ignore'][on_file_not_found='ignore']: Passed") + + except FileNotFoundError as e: + print("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (File not found)") + raise e + except Exception as e: + print("\tโŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (Other error)") + raise e + print("\t๐Ÿ“ NoFile Load ['no_warning']") # 4th error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + try: tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="no_warning") if tb.height: # must be 0 @@ -2837,6 +3147,8 @@ def _update_table(tb:PickleTable): tb.clear() tb.load_csv("test.csv", header=True) + + # in CSV every value is string if tb.height != 3 or tb.column("x") != ['1', '3', '5']: raise Exception("โŒ TEST [LOAD CSV][Header Load][header=True]: Failed") @@ -2878,13 +3190,222 @@ def _update_table(tb:PickleTable): os.remove("test.csv") + ############################################### print("="*50) + JSON_tb = PickleTable() + JSON_tb.add_column("x", exist_ok=True) + JSON_tb.add_column("Y", exist_ok=True) + + JSON_tb.add_row({"x":1, "Y":2}) + JSON_tb.add_row({"x":3, "Y":None}) + JSON_tb.add_row({"x":"5", "Y":6}) + + print("\n\n๐Ÿ“ TEST convert to JSON [dict format]") + + st = time.perf_counter() + JSON_tb.to_json(f"test{st}.json", format="dict") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to JSON test in {et - st}s") + + with open(f"test{st}.json") as f: + text = f.read() + + if text != """{ + "x": [ + 1, + 3, + "5" + ], + "Y": [ + 2, + null, + 6 + ] +}""": + raise Exception("โŒ TEST [CONVERT TO JSON]: Failed (Data Mismatch)") + os.remove(f"test{st}.json") + + print("="*50) + + print("\n\n๐Ÿ“ TEST convert to JSON [list format]") + + st = time.perf_counter() + JSON_tb.to_json(f"test{st}.json", format="list") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to JSON test in {et - st}s") + + with open(f"test{st}.json") as f: + text = f.read() + + if text != """[ + { + "x": 1, + "Y": 2 + }, + { + "x": 3, + "Y": null + }, + { + "x": "5", + "Y": 6 + } +]""": + raise Exception("โŒ TEST [CONVERT TO JSON]: Failed (Data Mismatch)") + + os.remove(f"test{st}.json") + + print("="*50) + + + print("\n\n๐Ÿ“ Load JSON (dict) test") + + tb.clear() + + print("Initial table") + print(tb) + print(tb.column_names) + + json_dict_test_data = """{ + "x": [1, 3, 5], + "Y": [2, 4, 6] + }""" + + json_list_test_data = """[ + {"x":1, "Y":2}, + {"x":3, "Y":4}, + {"x":5, "Y":6} + ]""" + + json_dict_test_iostream = io.StringIO(json_dict_test_data) + json_file = "test.json" + # make a json file + with open(json_file, "w") as f: + f.write(json_dict_test_data) + + print("\t๐Ÿ“ Normal Load [from JSON dict ๐Ÿ“„ ]") + try: + tb.load_json(json_file) + except Exception as e: + print("โŒ TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Failed (Other error)") + raise e + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Failed") + + print("\tโœ… TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Passed") + + print("\t๐Ÿ“ Normal Load [from JSON string ๐Ÿงต ]") + tb.clear() + + tb.load_json(json_str=json_dict_test_data) + + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][from JSON string ๐Ÿงต ]: Failed") + + print("\tโœ… TEST [LOAD JSON][Normal Load][from JSON string ๐Ÿงต ]: Passed") + + print("\t๐Ÿ“ Normal Load [From IOstream ๐Ÿชก ]") + tb.clear() + + tb.load_json(iostream=json_dict_test_iostream) + + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][From IOstream ๐Ÿชก ]: Failed") + + print("\tโœ… TEST [LOAD JSON][Normal Load][From IOstream ๐Ÿชก ]: Passed") + + print("\t๐Ÿ“ Multiple source [filepath+json_str]['error']") + # 1st error mode on multiple source + tb.clear() + + try: + tb.load_json(filepath=json_file, json_str=json_dict_test_data) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [filepath+iostream]['error']") + # 2nd error mode on multiple source + tb.clear() + try: + tb.load_json(filepath=json_file, iostream=json_dict_test_iostream) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [iostream+json_str]['error']") + # 3rd error mode on multiple source + tb.clear() + try: + tb.load_json(iostream=json_dict_test_iostream, json_str=json_dict_test_data) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [filepath+json_str+iostream]['error']") + # 4th error mode on multiple source + tb.clear() + + try: + tb.load_json(filepath=json_file, json_str=json_dict_test_data, iostream=json_dict_test_iostream) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ No source []['error']") + # 5th error mode on no source + tb.clear() + + try: + tb.load_json() + raise Exception("โŒ TEST [LOAD JSON][No source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][No source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][No source]: Failed (Other error)") + raise e + if os.path.exists(json_file): + os.remove(json_file) + if os.path.exists("test.csv"): + os.remove("test.csv") + print([tb.location]) @@ -2893,12 +3414,20 @@ def _update_table(tb:PickleTable): -# if __name__ == "__main__": + + + + + + + +if __name__ == "__main__": for i in range(1): try: os.remove("__test.pdb") except: pass test() - os.remove("__test.pdb") + + # os.remove("__test.pdb") print("\n\n\n" + "# "*25 + "\n") diff --git a/dev_src/pyroboxCore.py b/dev_src/pyroboxCore.py index f00f404..ce7ed20 100644 --- a/dev_src/pyroboxCore.py +++ b/dev_src/pyroboxCore.py @@ -4,6 +4,7 @@ import random import base64 import re +import itertools from http import HTTPStatus from http.cookies import SimpleCookie from functools import partial @@ -306,6 +307,14 @@ def __init__(self, caller, store_return=False): self.caller = caller + def done(self): + """returns True if all tasks are done""" + return self.queue.empty() and not self.BUSY + + def outputs(self): + """returns all the return values""" + return [self.returner.get() for i in range(self.returner.qsize())] + def next(self): """ check if any item in queje and call, if already running or queue empty, returns """ if self.queue.empty() or self.BUSY: @@ -322,13 +331,15 @@ def next(self): if not self.queue.empty(): # will make the loop continue running - return True + self.next() def update(self, *args, **kwargs): - """ Uses xprint and parse string""" + """ + Adds a task to the queue + """ self.queue.put((args, kwargs)) - while self.next() is True: + if self.next() is True: # use while instead of recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion.... error pass @@ -570,6 +581,10 @@ def parse_request(self): # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError + if any(not component.isdigit() for component in version_number): + raise ValueError("non digit in http version") + if any(len(component) > 10 for component in version_number): + raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( @@ -608,6 +623,20 @@ def parse_request(self): if self.path.startswith('//'): self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # The path should not contain any raw control characters; these + # are not allowed in URLs and could be used to bypass security + # filters. If a control character is found, send a 400 error + # response. This includes all characters below 0x20 except for + # horizontal tab, line feed, and carriage return, and all characters + self.path = self.safe_for_terminal(self.path) + if '\x00' in self.path: + self.send_error( + HTTPStatus.BAD_REQUEST, + message="Illegal null character in path") + return False + + + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, @@ -729,7 +758,8 @@ def handle_one_request(self): try: method() except Exception: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) if config.log_extra: logger.info('-'*w + f' {self.req_hash} ' + '-'*w + '\n' + @@ -910,7 +940,8 @@ def flush_headers(self): try: raise RuntimeError("Headers already flushed") except RuntimeError: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) return if hasattr(self, '_headers_buffer'): self.wfile.write(b"".join(self._headers_buffer)) @@ -959,6 +990,34 @@ def _log_writer(self, message): f.write( (f"#{self.req_hash} by [{self.address_string()}] at [{self.log_date_time_string()}]|=> {message}\n")) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table_removal = str.maketrans( + {c: None for c in itertools.chain(range(0x20), range(0x7f, 0xa0))}) + + __allowed_control_chars = [ + '\t', '\n' # , '\r' no need to keep carriage return + ] + _control_char_table[ord('\\')] = r'\\' + for c in __allowed_control_chars: + _control_char_table[ord(c)] = c + _control_char_table_removal[ord(c)] = c + + + + + + def safe_for_terminal(self, text: str, remove_control=False): + """Replace control characters in text with escaped hex.""" + + + if not isinstance(text, str): + return text + if remove_control: + return text.translate(self._control_char_table_removal) + return text.translate(self._control_char_table) + def log_message(self, *args, error=False, warning=False, debug=False, write=True, **kwargs): """Log an arbitrary message. @@ -968,7 +1027,20 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + + try: + self._log_message(*args, error=error, warning=warning, debug=debug, write=write, **kwargs) + except Exception: + # SUB FUNCTION to avoid recursion error + ERROR = traceback.format_exc() + print(*args, {"error": error, "warning": warning, "debug": debug, "write": write}, **kwargs, sep='\n') + logger.error(ERROR) + + def _log_message(self, *args, error=False, warning=False, debug=False, write=True, **kwargs): if not args: return @@ -977,6 +1049,9 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True message = sep.join(map(str, args)) + end + # Replace control characters in message with hex escapes + message = self.safe_for_terminal(message) + message = f"# {self.req_hash} by [{self.address_string()}] at [{self.log_date_time_string()}]|=> {message}\n" if error: @@ -997,7 +1072,8 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True try: self.Zlog_writer.update(message) except Exception: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) def version_string(self): """Return the server software version string.""" @@ -1090,6 +1166,13 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): handlers = { 'HEAD': [], 'POST': [], + # 'GET': [], + 'OPTIONS': [], + 'PUT': [], + 'DELETE': [], + 'PATCH': [], + 'CONNECT': [], + 'TRACE': [] } def __init__(self, *args, directory=None, **kwargs): @@ -1158,9 +1241,51 @@ def decorator(func): return func return decorator + @staticmethod + def on_GET(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when GET request is received''' + self = __class__ + + return self.on_req(method='GET', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_POST(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when POST request is received''' + self = __class__ + + return self.on_req(method='POST', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_HEAD(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when HEAD request is received''' + self = __class__ + + return self.on_req(method='HEAD', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_OPTIONS(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when OPTIONS request is received''' + self = __class__ + + return self.on_req(method='OPTIONS', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_DELETE(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when DELETE request is received''' + self = __class__ + + return self.on_req(method='DELETE', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_PUT(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when PUT request is received''' + self = __class__ + + return self.on_req(method='PUT', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + @staticmethod - def alt_directory(dir, method='', url='', hasQ=(), QV={}, fragent='', url_regex=''): + def alt_directory(dir, method='', url='', hasQ=(), QV={}, fragent='', url_regex='', cache_control="", cookie:Union[SimpleCookie, str]=None): """ alternative directory handler (only handles GET and HEAD request for files) """ @@ -1173,12 +1298,17 @@ def alt_dir_function(self: Type[__class__], *args, **kwargs): """ file = self.url_path.split("/")[-1] + requested_file = tools.xpath(dir, file) - if not os.path.exists(tools.xpath(dir, file)): + if not os.path.exists(requested_file): self.send_error(HTTPStatus.NOT_FOUND, "File not found") + self.log_info(f'File not found: {requested_file}') return None - return self.send_file(tools.xpath(dir, file)) + return self.send_file( + requested_file, + cache_control=cache_control, + cookie=cookie) def test_req(self, url='', hasQ=(), QV={}, fragent='', url_regex=''): @@ -1221,7 +1351,9 @@ def do_HEAD(self): try: resp = self.send_head() except Exception as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) + self.send_error(500, str(e)) return finally: @@ -1242,10 +1374,13 @@ def do_POST(self): for case, func in self.handlers['POST']: if self.test_req(*case): try: + self.log_info(f'[POST] -> [{func.__name__}] -> {self.url_path}') + resp = func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) except PostError: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) # break if error is raised and send BAD_REQUEST (at end of loop) break @@ -1253,7 +1388,7 @@ def do_POST(self): try: self.copyfile(resp, self.wfile) except (ConnectionAbortedError, ConnectionResetError, BrokenPipeError) as e: - logger.info(tools.text_box( + self.log_info(tools.text_box( e.__class__.__name__, e, "\nby ", [self.address_string()])) finally: resp.close() @@ -1262,14 +1397,57 @@ def do_POST(self): return self.send_error(HTTPStatus.BAD_REQUEST, "Invalid request.") except (ConnectionAbortedError, ConnectionResetError, BrokenPipeError) as e: - logger.info(tools.text_box(e.__class__.__name__, + self.log_info(tools.text_box(e.__class__.__name__, e, "\nby ", [self.address_string()])) return except Exception as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) + self.send_error(500, str(e)) return + + def send_head(self): + """Common code for GET and HEAD commands. + + This sends the response code and MIME headers. + + Return value is either a file object (which has to be copied + to the outputfile by the caller unless the command was HEAD, + and must be closed by the caller under all circumstances), or + None, in which case the caller has nothing further to do. + + """ + + if 'Range' not in self.headers: + self.range = None, None + first, last = 0, 0 + + else: + try: + self.range = parse_byte_range(self.headers['Range']) + first, last = self.range + self.use_range = True + except ValueError as e: + self.send_error(400, message='Invalid byte range') + return None + + path = self.translate_path(self.path) + # DIRECTORY DONT CONTAIN SLASH / AT END + + url_path, query, fragment = self.url_path, self.query, self.fragment + + spathsplit = self.url_path.split("/") + + # GET WILL Also BE HANDLED BY HEAD + for case, func in self.handlers['HEAD']: + if self.test_req(*case): + return func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) + + return self.send_error(HTTPStatus.NOT_FOUND, message="File not found") + + def redirect(self, location, cookie:Union[SimpleCookie, str]=None): '''redirect to location''' self.log_info("REDIRECT ", location) @@ -1340,12 +1518,12 @@ def send_css(self, msg:Union[str, bytes, Template], code:int=200, content_type=" '''proxy to send_txt''' return self.send_txt(msg, code, content_type) - def send_json(self, obj:Union[object, str, bytes], code=200, cookie:Union[SimpleCookie, str]=None): + def send_json(self, obj:Union[object, str, bytes], code=200, cookie:Union[SimpleCookie, str]=None, cache_control=""): """send object as json obj: json-able object or json.dumps() string""" if not isinstance(obj, str): obj = json.dumps(obj, indent=1) - file = self.return_txt(obj, code, content_type="application/json", cookie=cookie) + file = self.return_txt(obj, code, content_type="application/json", cookie=cookie, cache_control=cache_control) if self.command == "HEAD": return # to avoid sending file on get request self.copyfile(file, self.wfile) @@ -1468,13 +1646,19 @@ def return_file(self, path, filename=None, download=False, cache_control="", coo except OSError: self.send_error(HTTPStatus.NOT_FOUND, message="File not found", cookie=cookie) + self.log_info(f'File not found: {path}') + return None except Exception: - traceback.print_exc() + ERR_LOG = traceback.format_exc() + self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, message="Internal Server Error", cookie=cookie) + self.log_error(ERR_LOG) - # if f and not f.closed(): f.close() - raise + if file and not file.closed: + file.close() + + return None def send_file(self, path, filename=None, download=False, cache_control='', cookie:Union[SimpleCookie, str]=None): '''sends the head and file to client''' @@ -1489,46 +1673,7 @@ def send_file(self, path, filename=None, download=False, cache_control='', cooki file.close() - def send_head(self): - """Common code for GET and HEAD commands. - - This sends the response code and MIME headers. - - Return value is either a file object (which has to be copied - to the outputfile by the caller unless the command was HEAD, - and must be closed by the caller under all circumstances), or - None, in which case the caller has nothing further to do. - - """ - - if 'Range' not in self.headers: - self.range = None, None - first, last = 0, 0 - - else: - try: - self.range = parse_byte_range(self.headers['Range']) - first, last = self.range - self.use_range = True - except ValueError as e: - self.send_error(400, message='Invalid byte range') - return None - - path = self.translate_path(self.path) - # DIRECTORY DONT CONTAIN SLASH / AT END - - url_path, query, fragment = self.url_path, self.query, self.fragment - - spathsplit = self.url_path.split("/") - - # GET WILL Also BE HANDLED BY HEAD - for case, func in self.handlers['HEAD']: - if self.test_req(*case): - return func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) - - return self.send_error(HTTPStatus.NOT_FOUND, message="File not found") - - def get_displaypath(self, url_path): + def get_displaypath(self, url_path, escape_html=True): """ Helper to produce a display path for the directory listing. """ @@ -1538,7 +1683,9 @@ def get_displaypath(self, url_path): url_path, errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(url_path) - displaypath = html.escape(displaypath, quote=False) + + if escape_html: + displaypath = html.escape(displaypath, quote=False) return displaypath @@ -1627,7 +1774,8 @@ def copyfile(self, source, outputfile): source.read(1) source.seek(0) except OSError as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) raise e if not self.range: @@ -1721,7 +1869,21 @@ def __contains__(self, key): class DealPostData: - """do_login + """ + Deal with POST data + + init with SimpleHTTPRequestHandler + + * get: get next line + * get_content: get all content + * get_json: get json data + * skip: skip next line + * start: start reading + * check_size_limit: check if content size is within limit + + """ + + __sample_post_req__ = """do_post #get starting boundary 0: b'------WebKitFormBoundary7RGDIyjMpWhLXcZa\r\n' @@ -1839,6 +2001,7 @@ def get_json(self, max_size=-1): """ if not self.content_type == "application/json": + self.req.log_info(f"Are you sure you called `x=DealPostData(); x.start()` ?") raise PostError("Content-Type is not application/json") line = self.get_content(max_size=max_size) @@ -2290,7 +2453,7 @@ def __init__(self, port=0, directory="", bind="", arg_parse=True, handler=Simple handler=handler ) - def run(self, poll_interval=0.1): + def run(self, poll_interval=0.1, on_exit=lambda: None): try: self.httpd.serve_forever(poll_interval=poll_interval) except KeyboardInterrupt: @@ -2305,6 +2468,7 @@ def run(self, poll_interval=0.1): finally: self.stop() + on_exit() def stop(self): diff --git a/dev_src/server.py b/dev_src/server.py index c987987..214a2ab 100644 --- a/dev_src/server.py +++ b/dev_src/server.py @@ -31,7 +31,6 @@ from pyroboxCore import config as CoreConfig, logger, DealPostData as DPD, runner as pyroboxRunner, tools, reload_server, __version__ as pyroboxCore_version from tools import xpath, EXT, make_dir, os_scan_walk, is_file, Text_Box - from pyrobox_ServerHost import ServerConfig, ServerHost as SH @@ -1050,7 +1049,7 @@ def get_folder_data(self: SH, *args, **kwargs): }) - data = list_directory(self, os_path, user, cookie=cookie) + data = list_directory(self, os_path, user, cookie=cookie, escape_html=False) if data: return self.send_json(data, cookie=cookie) diff --git a/setup.cfg b/setup.cfg index 04d09ae..8659408 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyrobox -version = 0.9.7 +version = 0.9.8 author = Rasan author_email= wwwqweasd147@gmail.com description = Personal DropBox for Private Network @@ -21,6 +21,8 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 License :: OSI Approved :: MIT License Operating System :: OS Independent Topic :: Internet :: File Transfer Protocol (FTP) diff --git a/src/_list_maker.py b/src/_list_maker.py index 5d8e540..990b3f9 100644 --- a/src/_list_maker.py +++ b/src/_list_maker.py @@ -215,7 +215,7 @@ def list_directory_html(self:SH, path, user:User, cookie:Union[SimpleCookie, str -def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=None): +def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=None, escape_html=False): """ Helper to produce a directory listing (absent index.html). @@ -223,6 +223,10 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non error). In either case, the headers are sent, making the interface the same as for send_head(). + path: path to the directory + user: User object + cookie: cookie object + escape_html: whether to escape html or not (default: False, since it will be sent as json) """ try: dir_list = scansort(os.scandir(path)) @@ -232,7 +236,10 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non "No permission to list directory") return None - displaypath = self.get_displaypath(self.url_path) + displaypath = self.get_displaypath( + url_path=self.url_path, + escape_html=escape_html + ) title = get_titles(displaypath) @@ -252,6 +259,9 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non #fullname = os.path.join(path, name) name = file.name displayname = linkname = name + + if escape_html: + displayname = html.escape(displayname, quote=False) size=0 # Append / for directories or @ for symbolic links _is_dir_ = True @@ -266,22 +276,22 @@ def list_directory(self:SH, path, user:User, cookie:Union[SimpleCookie, str]=Non __, ext = posixpath.splitext(name) if ext=='.html': r_li.append('h'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) elif self.guess_type(linkname).startswith('video/'): r_li.append('v'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) elif self.guess_type(linkname).startswith('image/'): r_li.append('i'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) else: r_li.append('f'+ urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) if _is_dir_: r_li.append('d' + urllib.parse.quote(linkname, errors='surrogatepass')) - f_li.append(html.escape(displayname, quote=False)) + f_li.append(displayname) s_li.append(size) diff --git a/src/_zipfly_manager.py b/src/_zipfly_manager.py index d4bb7ff..99b26bb 100644 --- a/src/_zipfly_manager.py +++ b/src/_zipfly_manager.py @@ -17,6 +17,9 @@ from collections import OrderedDict import re +from logging import Logger + +logger = Logger(__name__) from ._fs_utils import get_dir_m_time, _get_tree_path_n_size, humanbytes @@ -302,6 +305,9 @@ def process_thread(): t = threading.Thread(target=process_thread) t.start() + time.sleep(0.2) + + while self.running_process: if self.running_process.stdout is None: break @@ -499,7 +505,7 @@ def zip7z_handler(self, path, zid, source_size, zfile_name, err): try: for progress in zip7z_obj.make_zip_with_7z_generator(path, zfile_name): - print([progress]) + logger.info(f"Zipping {[path]}: {[progress]}") self.zip_in_progress[zid] = int(progress[:-1]) except Exception as e: traceback.print_exc() diff --git a/src/pyroDB.py b/src/pyroDB.py index 59d885c..a9a407e 100644 --- a/src/pyroDB.py +++ b/src/pyroDB.py @@ -38,11 +38,13 @@ import io +import json import os import signal import atexit import shutil from collections.abc import Iterable +import sys import time import random from tempfile import NamedTemporaryFile @@ -66,6 +68,13 @@ TABLE = False try: + # Check if msgpack is installed + + # Check if GIL is enabled (Now available in msgpack-1.1.0-cp313-cp313t-win_amd64.whl) + if not getattr(sys, '_is_gil_enabled', lambda: True)(): + os.environ["MSGPACK_PUREPYTHON"] = "1" + # msgpack is not thread safe (yet) + import msgpack # pip install msgpack SAVE_LOAD = True except ImportError: @@ -108,6 +117,8 @@ def __init__(self, location="", auto_dump=True, sig=True): if sig: self.set_sigterm_handler() + self._autodumpdb() + def __getitem__(self, item): """ Syntax sugar for get() @@ -576,10 +587,10 @@ def _int_to_alpha(n): return string class PickleTable(dict): - def __init__(self, file_path="", *args, **kwargs): + def __init__(self, filepath="", *args, **kwargs): """ args: - - filename: path to the db file (default: `""` or in-memory db) + - filepath: path to the db file (default: `""` or in-memory db) - auto_dump: auto dump on change (default: `True`) - sig: Add signal handler for graceful shutdown (default: `True`) """ @@ -590,7 +601,7 @@ def __init__(self, file_path="", *args, **kwargs): self.busy = False - self._pk = PickleDB(file_path, *args, **kwargs) + self._pk = PickleDB(location=filepath, *args, **kwargs) # make the super dict = self._pk.db @@ -674,7 +685,7 @@ def unlink(self): def delete_file(self): """ - Delete the file from the disk + Delete the file from the disk """ self._pk.delete_file() @@ -710,13 +721,13 @@ def __str__(self, limit:int=None): x += "\t|\t".join(self.column_names_func(rescan=False)) for i in range(min(self.height, limit)): x += "\n" - x += "\t|\t".join(str(self.row(i).values())) + x += "\t|\t".join([str(cell) if cell is not None else '' for cell in self.row(i).values()]) - else: + else: x = tabulate( - # self[:min(self.height, limit)], + # self[:min(self.height, limit)], self.rows(start=0, end=min(self.height, limit)), - headers="keys", + headers="keys", tablefmt= "simple_grid", #"orgtbl", maxcolwidths=60 @@ -783,7 +794,7 @@ def column(self, name, rescan=True) -> list: """ self.rescan(rescan=rescan) return self._pk.db[name].copy() - + def get_column(self, name) -> list: """ Return the list pointer to the column (unsafe) @@ -1018,7 +1029,7 @@ def search_iter(self, kw, column=None , row=None, full_match=False, return_obj=T - return_obj: return cell object instead of value (default: `True`) - return: cell object - ie: + ie: ```python for cell in db.search_iter("abc"): print(cell.value) @@ -1099,7 +1110,7 @@ def search(self, kw, column=None , row=None, full_match=False, return_obj=True, - return_row: return row object instead of cell object (default: `False`) - return: cell object - ie: + ie: ```python for cell in db.search("abc"): print(cell.value) @@ -1142,7 +1153,7 @@ def find_1st(self, kw, column=None , row=None, full_match=False, return_obj=True for cell in self.search_iter(kw, column=column , row=row, full_match=full_match, return_obj=return_obj, rescan=rescan): return cell - + def find_1st_row(self, kw, column=None , row=None, full_match=False, return_obj=True, rescan=True) -> Union["_PickleTRow", None]: """ search a keyword in a cell/row/column/entire sheet and return the 1st matched row object @@ -1335,7 +1346,7 @@ def copy(self, location=None, auto_dump=True, sig=True) -> "PickleTable": - sig: Add signal handler for graceful shutdown (default: `True`) - return: new PickleTable object """ - + new = PickleTable(location, auto_dump=auto_dump, sig=sig) new.add_column(*self.column_names) new._pk.db = datacopy.deepcopy(self.__db__()) @@ -1384,7 +1395,7 @@ def _add_row(self, row:Union[dict, "_PickleTRow"], position:int="last", rescan=T return self.row_obj_by_id(row_id) - def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True) -> "_PickleTRow": + def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True, rescan=True) -> "_PickleTRow": """ @ locked - row: row must be a dict|_PickleTRow containing column names and values @@ -1399,7 +1410,7 @@ def add_row(self, row:Union[dict, "_PickleTRow"], position="last", AD=True) -> " ``` """ - row_obj = self.lock(self._add_row)(row=row,position=position) + row_obj = self.lock(self._add_row)(row=row,position=position, rescan=rescan) self.auto_dump(AD=AD) @@ -1426,7 +1437,7 @@ def insert_row(self, row:Union[dict, "_PickleTRow"], position:int="last", AD=Tru - def add_row_as_list(self, row:list, position:int="last", AD=True) -> "_PickleTRow": + def add_row_as_list(self, row:list, position:int="last", AD=True, rescan=True) -> "_PickleTRow": """ @ locked - row: row must be a list containing values. (order must match with column names) @@ -1440,7 +1451,7 @@ def add_row_as_list(self, row:list, position:int="last", AD=True) -> "_PickleTRo db.add_row_as_list(["John", 25]) """ - row_obj = self.lock(self._add_row)(row={k:v for k, v in zip(self.column_names, row)}, position=position) + row_obj = self.lock(self._add_row)(row={k:v for k, v in zip(self.column_names, row)}, position=position, rescan=rescan) self.auto_dump(AD=AD) @@ -1578,24 +1589,141 @@ def auto_dump(self, AD=True): """ self._pk._autodumpdb(AD=AD) + def to_json(self, filepath=None, indent=4, format=list) -> str: + """ + Write the table to a json file + - filepath: path to the file (if None, use current filepath.json) (if in memory and not provided, uses "table.json") + - indent: indentation level (default: 4 spaces) + - format: format of the json file (default: list) [options: list|dict] + - list: list of rows [{col1: val1, col2: val2, ...}, ...] + - dict: dict of columns {col1: [val1, val2, ...], col2: [val1, val2, ...], ...} + + - return: path to the file + """ + if filepath is None: + # check filepath + path = self._pk.location + if not path: + path = "table.json" + else: + path = os.path.splitext(path)[0] + ".json" + else: + path = filepath + + + # write to file + with open(path, "w", encoding='utf8') as f: + if format == dict or format == 'dict': + json.dump(self.__db__(), f, indent=indent) + elif format == list or format == 'list': + json.dump(list(self.rows()), f, indent=indent) + else: + raise AttributeError("Invalid format. [expected: list|dict] [got:", format, "]") + + return os.path.realpath(path) + + def to_json_str(self, indent=4) -> str: + """ + Return the table as a json string + """ + return json.dumps(self.__db__(), indent=indent) - def to_csv(self, filename=None, write_header=True) -> str: + def load_json(self, filepath=None, iostream=None, json_str=None, ignore_new_headers=False, on_file_not_found='error', AD=True): + """ + Load a json file to the table + - WILL OVERWRITE THE EXISTING DATA (To append, make a new table and extend) + - filepath: path to the file + - ignore_new_headers: ignore new headers|columns if found `[when header=True]` (default: `False` and will add new headers) + - on_file_not_found: action to take if the file is not found (default: `'error'`) [options: `error`|`ignore`|`warn`|`no_warning`] + * if `error`, raise FileNotFoundError + * if `ignore`, ignore the operation (no warning, **no clearing**) + * if `no_warning`, ignore the operation, but **clears db** + * if `warn`, print warning and ignore the operation, but **clears db** + """ + + # if more than one source is provided + sources = [i for i in [filepath, iostream, json_str] if i] + if len(sources) > 1: + raise AttributeError(f"Only one source is allowed. Got: {len(sources)}") + + # if no source is provided + if not sources: + raise AttributeError("No source provided") + + if json_str: + # load it as io stream + iostream = io.StringIO(json_str) + + if not ((filepath and os.path.isfile(filepath)) or (iostream and isinstance(iostream, io.IOBase))): + if on_file_not_found == 'error': + raise FileNotFoundError(f"File not found: {filepath}") + + elif on_file_not_found == 'ignore': + return + else: + self.clear(AD=AD) + if on_file_not_found == 'warn': + print(f"File not found: {filepath}. Cleared the table.") + if on_file_not_found == 'no_warning': + pass + + return + + self.clear(AD=False) + + def load_as_io(f): + return json.load(f) + + if iostream: + data = load_as_io(iostream) + else: + with open(filepath, "r", encoding='utf8') as f: + data = load_as_io(f) + + # print(data) + + if not data: + return + + existing_columns = self.column_names_func(rescan=False) + + if isinstance(data, dict): + # column names are keys + self.add(table=data, + add_extra_columns=ignore_new_headers, + AD=AD + ) + + elif isinstance(data, list): + # per row + for row in data: + if not ignore_new_headers: + for col in row: + if col not in existing_columns: + self.add_column(col, exist_ok=True, AD=False, rescan=False) + + self._add_row(row, rescan=False) + + self.auto_dump(AD=AD) + + + def to_csv(self, filepath=None, write_header=True) -> str: """ Write the table to a csv file - - filename: path to the file (if None, use current filename.csv) (if in memory and not provided, uses "table.csv") + - filepath: path to the file (if None, use current filepath.csv) (if in memory and not provided, uses "table.csv") - write_header: write column names as header (1st row) (default: `True`) - return: path to the file """ - if filename is None: - # check filename + if filepath is None: + # check filepath path = self._pk.location if not path: path = "table.csv" else: path = os.path.splitext(path)[0] + ".csv" else: - path = filename + path = filepath with open(path, "wb") as f: f.write(b'') @@ -1613,6 +1741,8 @@ def to_csv(self, filename=None, write_header=True) -> str: return os.path.realpath(path) + + def to_csv_str(self, write_header=True) -> str: """ Return the table as a csv string @@ -1629,16 +1759,19 @@ def to_csv_str(self, write_header=True) -> str: return output.getvalue() - def load_csv(self, filename, header=True, ignore_none=False, ignore_new_headers=False, on_file_not_found='error', AD=True): + def load_csv(self, filepath=None, iostream=None, csv_str=None, + header=True, ignore_none=False, ignore_new_headers=False, on_file_not_found='error', AD=True): """ Load a csv file to the table - WILL OVERWRITE THE EXISTING DATA (To append, make a new table and extend) - - header: + - filepath: path to the file + - iostream: io stream object (use either filepath or iostream) + - header: * if True, the first row will be considered as column names * if False, the columns will be named as "Unnamed-1", "Unnamed-2", ... * if "auto", the columns will be named as "A", "B", "C", ..., "Z", "AA", "AB", ... - ignore_none: ignore the None rows - - ignore_new_headers: ignore new headers if found `[when header=True]` (default: `False`) + - ignore_new_headers: ignore new headers if found `[when header=True]` (default: `False` and will add new headers) - on_file_not_found: action to take if the file is not found (default: `'error'`) [options: `error`|`ignore`|`warn`|`no_warning`] * if `error`, raise FileNotFoundError * if `ignore`, ignore the operation (no warning, **no clearing**) @@ -1648,26 +1781,36 @@ def load_csv(self, filename, header=True, ignore_none=False, ignore_new_headers= columns_names = self.column_names def add_row(row, columns): - if ignore_none and all(v is None for v in row): + if ignore_none and all((v is None or v == '') for v in row): return new_row = {k: v for k, v in zip(columns, row)} + + self.add_row(new_row, rescan=False) - # print(new_row) + # if more than one source is provided + sources = [i for i in [filepath, iostream, csv_str] if i] + if len(sources) > 1: + raise AttributeError(f"Only one source is allowed. Got: {len(sources)}") - self.add_row(new_row, AD=False) + if not sources: + raise AttributeError("No source provided") + if csv_str: + # load it as io stream + iostream = io.StringIO(csv_str) - if not os.path.exists(filename): + if not ((filepath and os.path.isfile(filepath)) or (iostream and isinstance(iostream, io.IOBase))): if on_file_not_found == 'error': - raise FileNotFoundError(f"File not found: {filename}") + raise FileNotFoundError(f"File not found: {filepath}") + elif on_file_not_found == 'ignore': return else: self.clear(AD=AD) if on_file_not_found == 'warn': - print(f"File not found: {filename}") + print(f"File not found: {filepath}. Cleared the table.") if on_file_not_found == 'no_warning': pass @@ -1675,12 +1818,15 @@ def add_row(row, columns): self.clear(AD=False) - with open(filename, 'r', encoding='utf8') as f: + + def load_as_io(f): reader = csv.reader(f) if header is True: columns = next(reader) updated_columns = [] n = len(self.column_names_func(rescan=False)) + + for col in columns: if (col is None or col == ""): while f"Unnamed-{n}" in columns_names: @@ -1688,13 +1834,16 @@ def add_row(row, columns): col = f"Unnamed-{n}" n += 1 - if col in columns_names or col in updated_columns: + if not (col in columns_names): if ignore_new_headers: continue + if col in updated_columns: + continue updated_columns.append(col) columns = updated_columns + # print(updated_columns) self.add_column(updated_columns, exist_ok=True, AD=False, rescan=False) @@ -1712,8 +1861,8 @@ def add_row(row, columns): self.add_column(new_columns, exist_ok=True, AD=False, rescan=False) columns = new_columns - - add_row(row, columns) + + add_row(row, columns) # add the first row else: # count the columns, and name them as "Unnamed-1", "Unnamed-2", ... row = next(reader) @@ -1731,7 +1880,7 @@ def add_row(row, columns): self.add_column(new_columns, exist_ok=True, AD=False, rescan=False) columns = new_columns - add_row(row, columns) + add_row(row, columns) # add the first row @@ -1740,10 +1889,17 @@ def add_row(row, columns): for row in reader: add_row(row, columns) - self.auto_dump(AD=AD) + if filepath: + with open(filepath, 'r', encoding='utf8') as f: + load_as_io(f) + elif iostream: + load_as_io(iostream) - def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): + self.auto_dump(AD=AD) + + + def extend(self, other: "PickleTable", add_extra_columns=None, AD=True, rescan=True): """ Extend the table with another table - other: `PickleTable` object @@ -1760,9 +1916,11 @@ def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): if not isinstance(other, type(self)): raise TypeError("Unsupported operand type(s) for +: 'PickleTable' and '{}'".format(type(other).__name__)) + self.rescan(rescan=rescan) + keys = other.column_names this_keys = self.column_names - + if add_extra_columns: self.add_column(*keys, exist_ok=True, AD=False) else: @@ -1775,7 +1933,7 @@ def extend(self, other: "PickleTable", add_extra_columns=None, AD=True): for row in other: - self._add_row({k: row[k] for k in keys}) + self._add_row({k: row[k] for k in keys}, rescan=False) self.auto_dump(AD=AD) @@ -1820,12 +1978,10 @@ def add(self, table:Union["PickleTable", dict], add_extra_columns=None, AD=True) else: self.extend(table, ) - + self.auto_dump(AD=AD) - - class _PickleTCell: def __init__(self, source:PickleTable, column, row_id:int, CC): self.source = source @@ -1911,7 +2067,7 @@ def row_index(self): @property def row(self): """ - returns a COPY row dict of the cell + returns a COPY row dict of the cell """ return self.source.row_by_id(self.id) @@ -1949,7 +2105,7 @@ def __init__(self, source:PickleTable, uid, CC): self.id = uid self.CC = CC self.deleted = False - + def is_deleted(self): """ return True if the row is deleted, else False @@ -2028,7 +2184,7 @@ def del_item(self, name, AD=True): self.source.raise_source(self.CC) self.source.set_cell_by_id(name, self.id, None, AD=AD) - + def __delitem__(self, name): # Auto dump @@ -2126,7 +2282,7 @@ def __eq__(self, other): def __ne__(self, other): self.raise_deleted() return not self.__eq__(other) - + class _PickleTColumn(list): @@ -2345,7 +2501,7 @@ def __str__(self): def __repr__(self): self.raise_deleted() - + return repr(self.source.get_column(self.name)) def del_column(self): @@ -2395,6 +2551,7 @@ def apply(self, func=None, row_func=False, copy=False, AD=True): + if __name__ == "__main__": import string @@ -2473,7 +2630,7 @@ def test(): dt = time.perf_counter() tb.dump() tt = time.perf_counter() - print(f"โฑ๏ธ DUMP time: {tt-dt}s") + print(f"โฑ๏ธ [IN-MEMORY] DUMP time: {tt-dt}s") print("="*50) @@ -2491,16 +2648,10 @@ def test(): # for cell in cells: # print(cell.row_obj()) - print(tabulate(cells, headers="keys", tablefmt="simple_grid")) - - - print("="*50) - print("\n\n๐Ÿ“ TEST convert to CSV") - st = time.perf_counter() - tb.to_csv(f"test{st}.csv") - et = time.perf_counter() - print(f"โฑ๏ธ Convert to csv test in {et - st}s") - os.remove(f"test{st}.csv") + if TABLE: + print(tabulate(cells, headers="keys", tablefmt="simple_grid")) + else: + print(cells, sep="\n") print("="*50) @@ -2559,7 +2710,7 @@ def test(): except Exception as e: print("โŒ TEST [ADD] (Raise Exception Invalid type): Failed") raise e - + try: tb.add({"x":[1,2,3], "Y":[4,5,6,7], "Z":[1,2,3]}) @@ -2571,7 +2722,7 @@ def test(): print("โŒ TEST [ADD dict] (Raise Exception extra column) (F): Failed") raise e - + try: tb.add({"x":[1,2,3], "Y":[4,5,6,7]}, add_extra_columns=True) print("โœ… TEST [ADD dict] (Add extra column): Passed") @@ -2604,21 +2755,21 @@ def _update_table(tb:PickleTable): print("\n\n๐Ÿ“ Sort test 1 (sort by x)") - + _update_table(tb) tb.sort("x") if tb.column("x") != [1, 3, 5, 7]: raise Exception("โŒ TEST [SORT] (sort by x): Failed") - + print("โœ… TEST [SORT] (sort by x): Passed") print("="*50) print("\n\n๐Ÿ“ Sort test 2 (sort by Y reverse)") _update_table(tb) - + tb.sort("Y", reverse=True) if tb.column("Y") != [8, 6, 4, 2]: @@ -2636,7 +2787,7 @@ def _update_table(tb:PickleTable): tb.add_column("x+Y", exist_ok=True) - + print("๐Ÿ“ APPLYING COLUMN FUNCTION with row_func=True") try: tb["x+Y"].apply(func=lambda x: x["x"]+x["Y"], row_func=True) @@ -2649,7 +2800,7 @@ def _update_table(tb:PickleTable): raise Exception("โŒ TEST [SORT] (sort by key function) {x+y}: Failed") print("โœ… TEST [SORT] (sort by key function) {x+y}: Passed") - + print("="*50) print("\n\n๐Ÿ“ Sort test 4 (sort by key function) {x+y} (copy)") @@ -2685,7 +2836,7 @@ def _update_table(tb:PickleTable): print("="*50) - + print("\n\n๐Ÿ“ Remove duplicates test [selective column]") tb.clear() @@ -2743,6 +2894,37 @@ def _update_table(tb:PickleTable): tb.add_column("x", exist_ok=True) # bring back the column print("="*50) + + + print("="*50) + print("\n\n๐Ÿ“ TEST convert to CSV") + + CSV_tb = PickleTable() + CSV_tb.add_column("x", exist_ok=True) + CSV_tb.add_column("Y", exist_ok=True) + + CSV_tb.add_row({"x":1, "Y":2}) + CSV_tb.add_row({"x":3, "Y":None}) + CSV_tb.add_row({"x":"5,5", "Y":6}) + + st = time.perf_counter() + CSV_tb.to_csv(f"test{st}.csv") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to csv test in {et - st}s") + + with open(f"test{st}.csv", "r") as f: + text = f.read() + + if text != """x,Y +1,2 +3, +"5,5",6 +""": + raise Exception("โŒ TEST [CONVERT TO CSV]: Failed (Data Mismatch)") + os.remove(f"test{st}.csv") + + print("="*50) + print("\n\n๐Ÿ“ Load csv test") tb.clear() @@ -2751,52 +2933,153 @@ def _update_table(tb:PickleTable): print(tb) print(tb.column_names) + csv_test_data = """x,Y\n1,2\n3,4\n5,6\n""" + csv_test_iostream = io.StringIO(csv_test_data) + # make a csv file with open("test.csv", "w") as f: - f.write("x,Y\n1,2\n3,4\n5,6\n") + f.write(csv_test_data) - print("\t๐Ÿ“ Normal Load") + print("\t๐Ÿ“ Normal Load [from File ๐Ÿ“„ ]") try: tb.load_csv("test.csv") except Exception as e: - print("โŒ TEST [LOAD CSV]: Failed (Other error)") + print("โŒ TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Failed (Other error)") raise e if tb.height != 3 or tb.column("x") != ['1', '3', '5']: print(tb) print(tb.column("x"), tb.column('x')==[1, 3, 5]) print(tb.height, tb.height==3) - raise Exception("โŒ TEST [LOAD CSV]: Failed") + raise Exception("โŒ TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Failed") - print("\tโœ… TEST [LOAD CSV][Normal Load]: Passed") + print("\tโœ… TEST [LOAD CSV][Normal Load][from File ๐Ÿ“„ ]: Passed") - print("\t๐Ÿ“ NoFile Load ['error']") - # 1st error mode on file not found + print("\t๐Ÿ“ Normal Load [from CSV string ๐Ÿงต ]") + tb.clear() + + tb.load_csv(csv_str=csv_test_data) + + if tb.height != 3 or tb.column("x") != ['1', '3', '5']: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD CSV][Normal Load][from CSV string ๐Ÿงต ]: Failed") + + print("\tโœ… TEST [LOAD CSV][Normal Load][from CSV string ๐Ÿงต ]: Passed") + + print("\t๐Ÿ“ Normal Load [From IOstream ๐Ÿชก ]") + tb.clear() + + tb.load_csv(iostream=csv_test_iostream) + + if tb.height != 3 or tb.column("x") != ['1', '3', '5']: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD CSV][Normal Load][From IOstream ๐Ÿชก ]: Failed") + + print("\tโœ… TEST [LOAD CSV][Normal Load][From IOstream ๐Ÿชก ]: Passed") + + print("\t๐Ÿ“ Multiple source [filepath+csv_str]['error']") + tb.clear() + # 1st error mode on multiple source try: - tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="error") - raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='error']: Failed") - except FileNotFoundError as e: - print("\tโœ… TEST [LOAD CSV][NoFile Load]['error'][on_file_not_found='error']: Passed") + tb.load_csv(filepath="test.csv", csv_str=csv_test_data) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") print("\tError message = ", e) - print("\t๐Ÿ“ NoFile Load ['ignore']") - # 3rd error mode on file not found + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + + print("\t๐Ÿ“ Multiple source [filepath+iostream]['error']") + # 2nd error mode on multiple source + tb.clear() try: - tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="ignore") - if tb.height != 3 or tb.column("x") != ['1', '3', '5']: - raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed") - print("\tโœ… TEST [LOAD CSV][NoFile Load]['ignore'][on_file_not_found='ignore']: Passed") + tb.load_csv(filepath="test.csv", iostream=csv_test_iostream) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) - except FileNotFoundError as e: - print("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (File not found)") + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") raise e + + print("\t๐Ÿ“ Multiple source [iostream+csv_str]['error']") + # 3rd error mode on multiple source + tb.clear() + try: + tb.load_csv(iostream=csv_test_iostream, csv_str=csv_test_data) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + except Exception as e: - print("\tโŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (Other error)") + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [filepath+csv_str+iostream]['error']") + # 4th error mode on multiple source + tb.clear() + try: + tb.load_csv(filepath="test.csv", csv_str=csv_test_data, iostream=csv_test_iostream) + raise Exception("โŒ TEST [LOAD CSV][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ No source []['error']") + # 5th error mode on no source + tb.clear() + try: + tb.load_csv() + raise Exception("โŒ TEST [LOAD CSV][No source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD CSV][No source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD CSV][No source]: Failed (Other error)") raise e + + + + + print("\t๐Ÿ“ NoFile Load ['error']") + # 1st error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + + try: + tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="error") + raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='error']: Failed") + except FileNotFoundError as e: + print("\tโœ… TEST [LOAD CSV][NoFile Load]['error'][on_file_not_found='error']: Passed") + print("\tError message = ", e) + + + print("\t๐Ÿ“ NoFile Load ['warn']") - # 2nd error mode on file not found + # 2nd error mode on file not found [DB is cleared] + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + try: tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="warn") if tb.height: # must be 0 @@ -2812,8 +3095,35 @@ def _update_table(tb:PickleTable): raise e + print("\t๐Ÿ“ NoFile Load ['ignore']") + # 3rd error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + + try: + tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="ignore") + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb.columns()) + raise Exception("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed") + print("\tโœ… TEST [LOAD CSV][NoFile Load]['ignore'][on_file_not_found='ignore']: Passed") + + except FileNotFoundError as e: + print("โŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (File not found)") + raise e + except Exception as e: + print("\tโŒ TEST [LOAD CSV][NoFile Load][on_file_not_found='ignore']: Failed (Other error)") + raise e + + print("\t๐Ÿ“ NoFile Load ['no_warning']") # 4th error mode on file not found + tb.clear() + tb.add_row({"x":1, "Y":2}) + tb.add_row({"x":3, "Y":4}) + tb.add_row({"x":5, "Y":6}) + try: tb.load_csv(f"test-{random.random()}.csv", on_file_not_found="no_warning") if tb.height: # must be 0 @@ -2837,6 +3147,8 @@ def _update_table(tb:PickleTable): tb.clear() tb.load_csv("test.csv", header=True) + + # in CSV every value is string if tb.height != 3 or tb.column("x") != ['1', '3', '5']: raise Exception("โŒ TEST [LOAD CSV][Header Load][header=True]: Failed") @@ -2878,27 +3190,244 @@ def _update_table(tb:PickleTable): os.remove("test.csv") + ############################################### print("="*50) + JSON_tb = PickleTable() + JSON_tb.add_column("x", exist_ok=True) + JSON_tb.add_column("Y", exist_ok=True) + + JSON_tb.add_row({"x":1, "Y":2}) + JSON_tb.add_row({"x":3, "Y":None}) + JSON_tb.add_row({"x":"5", "Y":6}) + + print("\n\n๐Ÿ“ TEST convert to JSON [dict format]") + + st = time.perf_counter() + JSON_tb.to_json(f"test{st}.json", format="dict") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to JSON test in {et - st}s") + + with open(f"test{st}.json") as f: + text = f.read() + + if text != """{ + "x": [ + 1, + 3, + "5" + ], + "Y": [ + 2, + null, + 6 + ] +}""": + raise Exception("โŒ TEST [CONVERT TO JSON]: Failed (Data Mismatch)") + os.remove(f"test{st}.json") + + print("="*50) + + print("\n\n๐Ÿ“ TEST convert to JSON [list format]") + + st = time.perf_counter() + JSON_tb.to_json(f"test{st}.json", format="list") + et = time.perf_counter() + print(f"โฑ๏ธ Convert to JSON test in {et - st}s") + + with open(f"test{st}.json") as f: + text = f.read() + + if text != """[ + { + "x": 1, + "Y": 2 + }, + { + "x": 3, + "Y": null + }, + { + "x": "5", + "Y": 6 + } +]""": + raise Exception("โŒ TEST [CONVERT TO JSON]: Failed (Data Mismatch)") + + os.remove(f"test{st}.json") + + print("="*50) + + + print("\n\n๐Ÿ“ Load JSON (dict) test") + + tb.clear() + + print("Initial table") + print(tb) + print(tb.column_names) + + json_dict_test_data = """{ + "x": [1, 3, 5], + "Y": [2, 4, 6] + }""" + + json_list_test_data = """[ + {"x":1, "Y":2}, + {"x":3, "Y":4}, + {"x":5, "Y":6} + ]""" + + json_dict_test_iostream = io.StringIO(json_dict_test_data) + json_file = "test.json" + # make a json file + with open(json_file, "w") as f: + f.write(json_dict_test_data) + + print("\t๐Ÿ“ Normal Load [from JSON dict ๐Ÿ“„ ]") + try: + tb.load_json(json_file) + except Exception as e: + print("โŒ TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Failed (Other error)") + raise e + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Failed") + print("\tโœ… TEST [LOAD JSON][Normal Load][from JSON dict ๐Ÿ“„ ]: Passed") + + print("\t๐Ÿ“ Normal Load [from JSON string ๐Ÿงต ]") + tb.clear() + + tb.load_json(json_str=json_dict_test_data) + + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][from JSON string ๐Ÿงต ]: Failed") + print("\tโœ… TEST [LOAD JSON][Normal Load][from JSON string ๐Ÿงต ]: Passed") + print("\t๐Ÿ“ Normal Load [From IOstream ๐Ÿชก ]") + tb.clear() + tb.load_json(iostream=json_dict_test_iostream) + if tb.height != 3 or tb.column("x") != [1, 3, 5]: + print(tb) + print(tb.column("x"), tb.column('x')==[1, 3, 5]) + print(tb.height, tb.height==3) + raise Exception("โŒ TEST [LOAD JSON][Normal Load][From IOstream ๐Ÿชก ]: Failed") + print("\tโœ… TEST [LOAD JSON][Normal Load][From IOstream ๐Ÿชก ]: Passed") + print("\t๐Ÿ“ Multiple source [filepath+json_str]['error']") + # 1st error mode on multiple source + tb.clear() + try: + tb.load_json(filepath=json_file, json_str=json_dict_test_data) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + print("\t๐Ÿ“ Multiple source [filepath+iostream]['error']") + # 2nd error mode on multiple source + tb.clear() + try: + tb.load_json(filepath=json_file, iostream=json_dict_test_iostream) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + print("\t๐Ÿ“ Multiple source [iostream+json_str]['error']") + # 3rd error mode on multiple source + tb.clear() + try: + tb.load_json(iostream=json_dict_test_iostream, json_str=json_dict_test_data) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) -# if __name__ == "__main__": + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ Multiple source [filepath+json_str+iostream]['error']") + # 4th error mode on multiple source + tb.clear() + + try: + tb.load_json(filepath=json_file, json_str=json_dict_test_data, iostream=json_dict_test_iostream) + raise Exception("โŒ TEST [LOAD JSON][Multiple source]: Failed") + + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][Multiple source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][Multiple source]: Failed (Other error)") + raise e + + print("\t๐Ÿ“ No source []['error']") + # 5th error mode on no source + tb.clear() + + try: + tb.load_json() + raise Exception("โŒ TEST [LOAD JSON][No source]: Failed") + except AttributeError as e: + print("\tโœ… TEST [LOAD JSON][No source]: Passed") + print("\tError message = ", e) + + except Exception as e: + print("\tโŒ TEST [LOAD JSON][No source]: Failed (Other error)") + raise e + + + if os.path.exists(json_file): + os.remove(json_file) + if os.path.exists("test.csv"): + os.remove("test.csv") + + + print([tb.location]) + + + + + + + + + + + + + + + +if __name__ == "__main__": for i in range(1): try: os.remove("__test.pdb") except: pass test() - os.remove("__test.pdb") + + # os.remove("__test.pdb") print("\n\n\n" + "# "*25 + "\n") diff --git a/src/pyroboxCore.py b/src/pyroboxCore.py index b0ba711..6d51610 100644 --- a/src/pyroboxCore.py +++ b/src/pyroboxCore.py @@ -4,6 +4,7 @@ import random import base64 import re +import itertools from http import HTTPStatus from http.cookies import SimpleCookie from functools import partial @@ -306,6 +307,14 @@ def __init__(self, caller, store_return=False): self.caller = caller + def done(self): + """returns True if all tasks are done""" + return self.queue.empty() and not self.BUSY + + def outputs(self): + """returns all the return values""" + return [self.returner.get() for i in range(self.returner.qsize())] + def next(self): """ check if any item in queje and call, if already running or queue empty, returns """ if self.queue.empty() or self.BUSY: @@ -322,13 +331,15 @@ def next(self): if not self.queue.empty(): # will make the loop continue running - return True + self.next() def update(self, *args, **kwargs): - """ Uses xprint and parse string""" + """ + Adds a task to the queue + """ self.queue.put((args, kwargs)) - while self.next() is True: + if self.next() is True: # use while instead of recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion to avoid recursion.... error pass @@ -570,6 +581,10 @@ def parse_request(self): # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError + if any(not component.isdigit() for component in version_number): + raise ValueError("non digit in http version") + if any(len(component) > 10 for component in version_number): + raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( @@ -608,6 +623,20 @@ def parse_request(self): if self.path.startswith('//'): self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # The path should not contain any raw control characters; these + # are not allowed in URLs and could be used to bypass security + # filters. If a control character is found, send a 400 error + # response. This includes all characters below 0x20 except for + # horizontal tab, line feed, and carriage return, and all characters + self.path = self.safe_for_terminal(self.path) + if '\x00' in self.path: + self.send_error( + HTTPStatus.BAD_REQUEST, + message="Illegal null character in path") + return False + + + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, @@ -729,7 +758,8 @@ def handle_one_request(self): try: method() except Exception: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) if config.log_extra: logger.info('-'*w + f' {self.req_hash} ' + '-'*w + '\n' + @@ -910,7 +940,8 @@ def flush_headers(self): try: raise RuntimeError("Headers already flushed") except RuntimeError: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) return if hasattr(self, '_headers_buffer'): self.wfile.write(b"".join(self._headers_buffer)) @@ -959,6 +990,34 @@ def _log_writer(self, message): f.write( (f"#{self.req_hash} by [{self.address_string()}] at [{self.log_date_time_string()}]|=> {message}\n")) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table_removal = str.maketrans( + {c: None for c in itertools.chain(range(0x20), range(0x7f, 0xa0))}) + + __allowed_control_chars = [ + '\t', '\n' # , '\r' no need to keep carriage return + ] + _control_char_table[ord('\\')] = r'\\' + for c in __allowed_control_chars: + _control_char_table[ord(c)] = c + _control_char_table_removal[ord(c)] = c + + + + + + def safe_for_terminal(self, text: str, remove_control=False): + """Replace control characters in text with escaped hex.""" + + + if not isinstance(text, str): + return text + if remove_control: + return text.translate(self._control_char_table_removal) + return text.translate(self._control_char_table) + def log_message(self, *args, error=False, warning=False, debug=False, write=True, **kwargs): """Log an arbitrary message. @@ -968,7 +1027,20 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + + try: + self._log_message(*args, error=error, warning=warning, debug=debug, write=write, **kwargs) + except Exception: + # SUB FUNCTION to avoid recursion error + ERROR = traceback.format_exc() + print(*args, {"error": error, "warning": warning, "debug": debug, "write": write}, **kwargs, sep='\n') + logger.error(ERROR) + + def _log_message(self, *args, error=False, warning=False, debug=False, write=True, **kwargs): if not args: return @@ -977,6 +1049,9 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True message = sep.join(map(str, args)) + end + # Replace control characters in message with hex escapes + message = self.safe_for_terminal(message) + message = f"# {self.req_hash} by [{self.address_string()}] at [{self.log_date_time_string()}]|=> {message}\n" if error: @@ -997,7 +1072,8 @@ def log_message(self, *args, error=False, warning=False, debug=False, write=True try: self.Zlog_writer.update(message) except Exception: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) def version_string(self): """Return the server software version string.""" @@ -1090,6 +1166,13 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): handlers = { 'HEAD': [], 'POST': [], + # 'GET': [], + 'OPTIONS': [], + 'PUT': [], + 'DELETE': [], + 'PATCH': [], + 'CONNECT': [], + 'TRACE': [] } def __init__(self, *args, directory=None, **kwargs): @@ -1158,9 +1241,51 @@ def decorator(func): return func return decorator + @staticmethod + def on_GET(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when GET request is received''' + self = __class__ + + return self.on_req(method='GET', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_POST(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when POST request is received''' + self = __class__ + + return self.on_req(method='POST', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_HEAD(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when HEAD request is received''' + self = __class__ + + return self.on_req(method='HEAD', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_OPTIONS(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when OPTIONS request is received''' + self = __class__ + + return self.on_req(method='OPTIONS', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_DELETE(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when DELETE request is received''' + self = __class__ + + return self.on_req(method='DELETE', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + + @staticmethod + def on_PUT(url='', hasQ=(), QV={}, fragent='', url_regex=''): + '''called when PUT request is received''' + self = __class__ + + return self.on_req(method='PUT', url=url, hasQ=hasQ, QV=QV, fragent=fragent, url_regex=url_regex) + @staticmethod - def alt_directory(dir, method='', url='', hasQ=(), QV={}, fragent='', url_regex=''): + def alt_directory(dir, method='', url='', hasQ=(), QV={}, fragent='', url_regex='', cache_control="", cookie:Union[SimpleCookie, str]=None): """ alternative directory handler (only handles GET and HEAD request for files) """ @@ -1173,12 +1298,17 @@ def alt_dir_function(self: Type[__class__], *args, **kwargs): """ file = self.url_path.split("/")[-1] + requested_file = tools.xpath(dir, file) - if not os.path.exists(tools.xpath(dir, file)): + if not os.path.exists(requested_file): self.send_error(HTTPStatus.NOT_FOUND, "File not found") + self.log_info(f'File not found: {requested_file}') return None - return self.send_file(tools.xpath(dir, file)) + return self.send_file( + requested_file, + cache_control=cache_control, + cookie=cookie) def test_req(self, url='', hasQ=(), QV={}, fragent='', url_regex=''): @@ -1221,7 +1351,9 @@ def do_HEAD(self): try: resp = self.send_head() except Exception as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) + self.send_error(500, str(e)) return finally: @@ -1242,10 +1374,13 @@ def do_POST(self): for case, func in self.handlers['POST']: if self.test_req(*case): try: + self.log_info(f'[POST] -> [{func.__name__}] -> {self.url_path}') + resp = func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) except PostError: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) # break if error is raised and send BAD_REQUEST (at end of loop) break @@ -1253,7 +1388,7 @@ def do_POST(self): try: self.copyfile(resp, self.wfile) except (ConnectionAbortedError, ConnectionResetError, BrokenPipeError) as e: - logger.info(tools.text_box( + self.log_info(tools.text_box( e.__class__.__name__, e, "\nby ", [self.address_string()])) finally: resp.close() @@ -1262,14 +1397,57 @@ def do_POST(self): return self.send_error(HTTPStatus.BAD_REQUEST, "Invalid request.") except (ConnectionAbortedError, ConnectionResetError, BrokenPipeError) as e: - logger.info(tools.text_box(e.__class__.__name__, + self.log_info(tools.text_box(e.__class__.__name__, e, "\nby ", [self.address_string()])) return except Exception as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) + self.send_error(500, str(e)) return + + def send_head(self): + """Common code for GET and HEAD commands. + + This sends the response code and MIME headers. + + Return value is either a file object (which has to be copied + to the outputfile by the caller unless the command was HEAD, + and must be closed by the caller under all circumstances), or + None, in which case the caller has nothing further to do. + + """ + + if 'Range' not in self.headers: + self.range = None, None + first, last = 0, 0 + + else: + try: + self.range = parse_byte_range(self.headers['Range']) + first, last = self.range + self.use_range = True + except ValueError as e: + self.send_error(400, message='Invalid byte range') + return None + + path = self.translate_path(self.path) + # DIRECTORY DONT CONTAIN SLASH / AT END + + url_path, query, fragment = self.url_path, self.query, self.fragment + + spathsplit = self.url_path.split("/") + + # GET WILL Also BE HANDLED BY HEAD + for case, func in self.handlers['HEAD']: + if self.test_req(*case): + return func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) + + return self.send_error(HTTPStatus.NOT_FOUND, message="File not found") + + def redirect(self, location, cookie:Union[SimpleCookie, str]=None): '''redirect to location''' self.log_info("REDIRECT ", location) @@ -1340,12 +1518,12 @@ def send_css(self, msg:Union[str, bytes, Template], code:int=200, content_type=" '''proxy to send_txt''' return self.send_txt(msg, code, content_type) - def send_json(self, obj:Union[object, str, bytes], code=200, cookie:Union[SimpleCookie, str]=None): + def send_json(self, obj:Union[object, str, bytes], code=200, cookie:Union[SimpleCookie, str]=None, cache_control=""): """send object as json obj: json-able object or json.dumps() string""" if not isinstance(obj, str): obj = json.dumps(obj, indent=1) - file = self.return_txt(obj, code, content_type="application/json", cookie=cookie) + file = self.return_txt(obj, code, content_type="application/json", cookie=cookie, cache_control=cache_control) if self.command == "HEAD": return # to avoid sending file on get request self.copyfile(file, self.wfile) @@ -1468,13 +1646,19 @@ def return_file(self, path, filename=None, download=False, cache_control="", coo except OSError: self.send_error(HTTPStatus.NOT_FOUND, message="File not found", cookie=cookie) + self.log_info(f'File not found: {path}') + return None except Exception: - traceback.print_exc() + ERR_LOG = traceback.format_exc() + self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, message="Internal Server Error", cookie=cookie) + self.log_error(ERR_LOG) - # if f and not f.closed(): f.close() - raise + if file and not file.closed: + file.close() + + return None def send_file(self, path, filename=None, download=False, cache_control='', cookie:Union[SimpleCookie, str]=None): '''sends the head and file to client''' @@ -1489,46 +1673,7 @@ def send_file(self, path, filename=None, download=False, cache_control='', cooki file.close() - def send_head(self): - """Common code for GET and HEAD commands. - - This sends the response code and MIME headers. - - Return value is either a file object (which has to be copied - to the outputfile by the caller unless the command was HEAD, - and must be closed by the caller under all circumstances), or - None, in which case the caller has nothing further to do. - - """ - - if 'Range' not in self.headers: - self.range = None, None - first, last = 0, 0 - - else: - try: - self.range = parse_byte_range(self.headers['Range']) - first, last = self.range - self.use_range = True - except ValueError as e: - self.send_error(400, message='Invalid byte range') - return None - - path = self.translate_path(self.path) - # DIRECTORY DONT CONTAIN SLASH / AT END - - url_path, query, fragment = self.url_path, self.query, self.fragment - - spathsplit = self.url_path.split("/") - - # GET WILL Also BE HANDLED BY HEAD - for case, func in self.handlers['HEAD']: - if self.test_req(*case): - return func(self, url_path=url_path, query=query, fragment=fragment, path=path, spathsplit=spathsplit) - - return self.send_error(HTTPStatus.NOT_FOUND, message="File not found") - - def get_displaypath(self, url_path): + def get_displaypath(self, url_path, escape_html=True): """ Helper to produce a display path for the directory listing. """ @@ -1538,7 +1683,9 @@ def get_displaypath(self, url_path): url_path, errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(url_path) - displaypath = html.escape(displaypath, quote=False) + + if escape_html: + displaypath = html.escape(displaypath, quote=False) return displaypath @@ -1627,7 +1774,8 @@ def copyfile(self, source, outputfile): source.read(1) source.seek(0) except OSError as e: - traceback.print_exc() + ERROR = traceback.format_exc() + self.log_error(ERROR) raise e if not self.range: @@ -1721,7 +1869,21 @@ def __contains__(self, key): class DealPostData: - """do_login + """ + Deal with POST data + + init with SimpleHTTPRequestHandler + + * get: get next line + * get_content: get all content + * get_json: get json data + * skip: skip next line + * start: start reading + * check_size_limit: check if content size is within limit + + """ + + __sample_post_req__ = """do_post #get starting boundary 0: b'------WebKitFormBoundary7RGDIyjMpWhLXcZa\r\n' @@ -1839,6 +2001,7 @@ def get_json(self, max_size=-1): """ if not self.content_type == "application/json": + self.req.log_info(f"Are you sure you called `x=DealPostData(); x.start()` ?") raise PostError("Content-Type is not application/json") line = self.get_content(max_size=max_size) @@ -2290,7 +2453,7 @@ def __init__(self, port=0, directory="", bind="", arg_parse=True, handler=Simple handler=handler ) - def run(self, poll_interval=0.1): + def run(self, poll_interval=0.1, on_exit=lambda: None): try: self.httpd.serve_forever(poll_interval=poll_interval) except KeyboardInterrupt: @@ -2305,6 +2468,7 @@ def run(self, poll_interval=0.1): finally: self.stop() + on_exit() def stop(self):