From 432dd1c5a106cb51428d785e9c870418394f9e20 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 22 Feb 2024 16:42:30 +0100 Subject: [PATCH 01/58] chore: move version into package --- pyproject.toml | 4 ++-- pyreports/__init__.py | 14 ++++++++------ pyreports/cli.py | 6 +----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81338cd..1c92aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,5 +29,5 @@ reports = "pyreports.cli:main" [project.urls] homepage = "https://github.com/MatteoGuadrini/pyreports'" documentation = "https://pyreports.readthedocs.io/en/latest/" -repository = "https://github.com/MatteoGuadrini/pyreports/pyreports.git" -changelog = "https://github.com/MatteoGuadrini/pyreports/blob/master/CHANGES.md" \ No newline at end of file +repository = "https://github.com/MatteoGuadrini/pyreports.git" +changelog = "https://github.com/MatteoGuadrini/pyreports/blob/master/CHANGES.md" diff --git a/pyreports/__init__.py b/pyreports/__init__.py index fb30e21..a2d3ddc 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -20,10 +20,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Build complex pyreports from/to various formats.""" +"""Build complex reports from/to various formats.""" -from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER -from .core import Executor, Report, ReportBook -from .exception import ReportDataError, ReportManagerError -from .datatools import average, most_common, percentage, counter, aggregate, chunks, merge, deduplicate -from .cli import make_manager, get_data, load_config, validate_config +__version__ = '1.6.0' + +from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 +from .core import Executor, Report, ReportBook # noqa: F401 +from .exception import ReportDataError, ReportManagerError # noqa: F401 +from .datatools import average, most_common, percentage, counter, aggregate, chunks, merge, deduplicate # noqa: F401 +from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 diff --git a/pyreports/cli.py b/pyreports/cli.py index 781a70f..2f69cc5 100644 --- a/pyreports/cli.py +++ b/pyreports/cli.py @@ -28,11 +28,7 @@ import yaml import argparse import pyreports - -# endregion - -# region globals -__version__ = '1.5.1' +from pyreports import __version__ # endregion From 341c0c0d621cff1345c0c960da6883310b80a8da Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 23 Feb 2024 11:49:47 +0100 Subject: [PATCH 02/58] chore: remove pymssql support --- pyproject.toml | 2 +- pyreports/io.py | 141 +++++++++++++++++++++++++----------------------- setup.py | 49 +++++++++-------- 3 files changed, 102 insertions(+), 90 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c92aab..fa5e773 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ] -dependencies = ['ldap3', 'pymssql', 'mysql-connector-python', +dependencies = ['ldap3', 'mysql-connector-python', 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] [project.scripts] diff --git a/pyreports/io.py b/pyreports/io.py index ae3ca04..377e068 100644 --- a/pyreports/io.py +++ b/pyreports/io.py @@ -26,7 +26,6 @@ import sqlite3 from typing import Union, List import nosqlapi -import pymssql import mysql.connector as mdb import psycopg2 import tablib @@ -114,6 +113,7 @@ def __iter__(self): class Manager(ABC): """Manager base class""" + pass @@ -128,8 +128,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write('\n'.join(str(line) for row in data for line in row)) + with open(self.file, mode="w") as file: + file.write("\n".join(str(line) for row in data for line in row)) def read(self, **kwargs): """Read with format @@ -139,7 +139,7 @@ def read(self, **kwargs): data = tablib.Dataset(**kwargs) with open(self.file) as file: for line in file: - data.append([line.strip('\n')]) + data.append([line.strip("\n")]) return data @@ -154,10 +154,10 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(*data) - with open(self.file, mode='w') as file: - file.write('\n'.join([' '.join(row).strip('\n') for row in data])) + with open(self.file, mode="w") as file: + file.write("\n".join([" ".join(row).strip("\n") for row in data])) - def read(self, pattern=r'(.*\n|.*$)', **kwargs): + def read(self, pattern=r"(.*\n|.*$)", **kwargs): """Read with format :param pattern: regular expression pattern @@ -186,8 +186,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('csv')) + with open(self.file, mode="w") as file: + file.write(data.export("csv")) def read(self, **kwargs): """Read csv format @@ -209,8 +209,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('json')) + with open(self.file, mode="w") as file: + file.write(data.export("json")) def read(self, **kwargs): """Read json format @@ -232,8 +232,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('yaml')) + with open(self.file, mode="w") as file: + file.write(data.export("yaml")) def read(self, **kwargs): """Read yaml format @@ -255,15 +255,15 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='wb') as file: - file.write(data.export('xlsx')) + with open(self.file, mode="wb") as file: + file.write(data.export("xlsx")) def read(self, **kwargs): """Read xlsx format :return: Dataset object """ - with open(self.file, 'rb') as file: + with open(self.file, "rb") as file: return tablib.import_set(file, **kwargs) @@ -279,18 +279,6 @@ def close(self): self.cursor.close() -class MSSQLConnection(Connection): - """Connection microsoft sql class""" - - def connect(self): - self.connection = pymssql.connect(*self.args, **self.kwargs) - self.cursor = self.connection.cursor() - - def close(self): - self.connection.close() - self.cursor.close() - - class MySQLConnection(Connection): """Connection mysql class""" @@ -323,7 +311,7 @@ def __init__(self, connection: Connection): :param connection: Connection based object """ - self.type = 'sql' + self.type = "sql" self.connector = connection # Connect database self.connector.connect() @@ -410,7 +398,9 @@ def fetchone(self) -> tablib.Dataset: :return: Dataset object """ header = [field[0] for field in self.description] - self.data = tablib.Dataset(list(self.connector.cursor.fetchone()), headers=header) + self.data = tablib.Dataset( + list(self.connector.cursor.fetchone()), headers=header + ) return self.data def fetchmany(self, size=1) -> tablib.Dataset: @@ -453,11 +443,11 @@ class NoSQLManager(Manager, APIManager): def __init__(self, connection: APIConnection, *args, **kwargs): APIManager.__init__(self, connection, *args, **kwargs) - self.type = 'nosql' + self.type = "nosql" @staticmethod def _response_to_dataset( - obj: Union[List[tuple], List[list], dict, nosqlapi.Response] + obj: Union[List[tuple], List[list], dict, nosqlapi.Response], ) -> tablib.Dataset: """Transform receive data into Dataset object""" data = tablib.Dataset() @@ -469,7 +459,9 @@ def _response_to_dataset( if isinstance(obj.data, (list, tuple)): data = tablib.Dataset([row for row in obj.data]) elif isinstance(obj.data, dict): - data = tablib.Dataset([obj.data[key] for key in obj], headers=list(obj.data.keys())) + data = tablib.Dataset( + [obj.data[key] for key in obj], headers=list(obj.data.keys()) + ) else: data.append(obj) @@ -492,7 +484,7 @@ def __init__(self, file: File): :param file: file object """ - self.type = 'file' + self.type = "file" self.data = file def __repr__(self): @@ -539,18 +531,24 @@ def __init__(self, server, username, password, ssl=False, tls=True): :param ssl: disable or enable SSL. Default is False. :param tls: disable or enable TLS. Default is True. """ - self.type = 'ldap' + self.type = "ldap" # Check ssl connection port = 636 if ssl else 389 - self.connector = ldap3.Server(server, - get_info=ldap3.ALL, - port=port, - use_ssl=ssl) + self.connector = ldap3.Server( + server, get_info=ldap3.ALL, port=port, use_ssl=ssl + ) # Check tls connection - self.auto_bind = ldap3.AUTO_BIND_TLS_BEFORE_BIND if tls else ldap3.AUTO_BIND_NONE + self.auto_bind = ( + ldap3.AUTO_BIND_TLS_BEFORE_BIND if tls else ldap3.AUTO_BIND_NONE + ) # Create a bind connection with user and password - self.bind = ldap3.Connection(self.connector, user=f'{username}', password=f'{password}', - auto_bind=self.auto_bind, raise_exceptions=True) + self.bind = ldap3.Connection( + self.connector, + user=f"{username}", + password=f"{password}", + auto_bind=self.auto_bind, + raise_exceptions=True, + ) self.bind.bind() def __repr__(self): @@ -572,8 +570,13 @@ def rebind(self, username, password): """ # Disconnect LDAP server self.bind.unbind() - self.bind = ldap3.Connection(self.connector, user=f'{username}', password=f'{password}', - auto_bind=self.auto_bind, raise_exceptions=True) + self.bind = ldap3.Connection( + self.connector, + user=f"{username}", + password=f"{password}", + auto_bind=self.auto_bind, + raise_exceptions=True, + ) self.bind.bind() def unbind(self): @@ -591,16 +594,20 @@ def query(self, base_search, search_filter, attributes) -> tablib.Dataset: :param attributes: list of returning LDAP attributes :return: Dataset object """ - if self.bind.search(search_base=base_search, search_filter=f'{search_filter}', attributes=attributes, - search_scope=ldap3.SUBTREE): + if self.bind.search( + search_base=base_search, + search_filter=f"{search_filter}", + attributes=attributes, + search_scope=ldap3.SUBTREE, + ): # Build Dataset data = tablib.Dataset() data.headers = attributes for result in self.bind.response: - if result.get('attributes'): + if result.get("attributes"): row = list() for index, _ in enumerate(attributes): - row.append(result.get('attributes').get(attributes[index])) + row.append(result.get("attributes").get(attributes[index])) data.append(row) # Return object return data @@ -611,24 +618,23 @@ def query(self, base_search, search_filter, attributes) -> tablib.Dataset: # region Variables DBTYPE = { - 'sqlite': SQLiteConnection, - 'mssql': MSSQLConnection, - 'mysql': MySQLConnection, - 'postgresql': PostgreSQLConnection + "sqlite": SQLiteConnection, + "mysql": MySQLConnection, + "postgresql": PostgreSQLConnection, } FILETYPE = { - 'file': TextFile, - 'log': LogFile, - 'csv': CsvFile, - 'json': JsonFile, - 'yaml': YamlFile, - 'xlsx': ExcelFile, + "file": TextFile, + "log": LogFile, + "csv": CsvFile, + "json": JsonFile, + "yaml": YamlFile, + "xlsx": ExcelFile, } -READABLE_MANAGER = ('FileManager', 'DatabaseManager', 'LdapManager', 'NoSQLManager') +READABLE_MANAGER = ("FileManager", "DatabaseManager", "LdapManager", "NoSQLManager") -WRITABLE_MANAGER = ('FileManager', 'DatabaseManager', 'NoSQLManager') +WRITABLE_MANAGER = ("FileManager", "DatabaseManager", "NoSQLManager") # endregion @@ -678,8 +684,10 @@ def create_nosql_manager(connection, *args, **kwargs): :return: NoSQLManager """ # Check if connection class is API compliant with nosqlapi - if not hasattr(connection, 'connect'): - raise nosqlapi.ConnectError('the Connection class is not API compliant. See https://nosqlapi.rtfd.io/') + if not hasattr(connection, "connect"): + raise nosqlapi.ConnectError( + "the Connection class is not API compliant. See https://nosqlapi.rtfd.io/" + ) # Create NoSQLManager object return NoSQLManager(connection=connection, *args, **kwargs) @@ -697,17 +705,18 @@ def manager(datatype, *args, **kwargs): return create_database_manager(datatype, *args, **kwargs) elif datatype in FILETYPE: return create_file_manager(datatype, *args, **kwargs) - elif datatype == 'ldap': + elif datatype == "ldap": return create_ldap_manager(*args, **kwargs) - elif datatype == 'nosql': - connection = kwargs.get('connection') or args[0] + elif datatype == "nosql": + connection = kwargs.get("connection") or args[0] nargs = args[1:] try: - kwargs.pop('connection') + kwargs.pop("connection") except KeyError: pass return create_nosql_manager(connection, *nargs, **kwargs) else: raise ValueError(f"data type {datatype} doesn't exists!") + # endregion diff --git a/setup.py b/setup.py index 264b2fd..9e25aca 100644 --- a/setup.py +++ b/setup.py @@ -1,39 +1,42 @@ from setuptools import setup -__version__ = '1.6.0' -__author__ = 'Matteo Guadrini' -__email__ = 'matteo.guadrini@hotmail.it' -__homepage__ = 'https://github.com/MatteoGuadrini/pyreports' +__version__ = "1.6.0" +__author__ = "Matteo Guadrini" +__email__ = "matteo.guadrini@hotmail.it" +__homepage__ = "https://github.com/MatteoGuadrini/pyreports" with open("README.md") as rme, open("CHANGES.md") as ch: long_description = rme.read() + "\n" + ch.read() setup( - name='pyreports', + name="pyreports", version=__version__, - packages=['pyreports'], + packages=["pyreports"], url=__homepage__, - license='GNU General Public License v3.0', + license="GNU General Public License v3.0", author=__author__, author_email=__email__, - keywords='pyreports reports report csv yaml export excel database ldap dataset file executor book', - maintainer='Matteo Guadrini', - maintainer_email='matteo.guadrini@hotmail.it', - install_requires=['ldap3', 'pymssql', 'mysql-connector-python', - 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'], - description='pyreports is a python library that allows you to create complex report from various sources', + keywords="pyreports reports report csv yaml export excel database ldap dataset file executor book", + maintainer="Matteo Guadrini", + maintainer_email="matteo.guadrini@hotmail.it", + install_requires=[ + "ldap3", + "mysql-connector-python", + "psycopg2-binary", + "tablib", + "tablib[all]", + "nosqlapi", + "pyyaml", + ], + description="pyreports is a python library that allows you to create complex report from various sources", long_description=long_description, long_description_content_type="text/markdown", classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Operating System :: OS Independent", - ], - entry_points={ - 'console_scripts': [ - 'reports = pyreports.cli:main' - ] - }, - python_requires='>=3.7' + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: OS Independent", + ], + entry_points={"console_scripts": ["reports = pyreports.cli:main"]}, + python_requires=">=3.7", ) From 7166a3edb214b165e19be948e3ed8fac84e06303 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 24 Feb 2024 12:09:33 +0100 Subject: [PATCH 03/58] refactor: format code with ruff --- pyproject.toml | 6 +- pyreports/__init__.py | 13 +++- pyreports/cli.py | 161 ++++++++++++++++++++++++----------------- pyreports/core.py | 126 ++++++++++++++++++++++---------- pyreports/datatools.py | 21 +++--- 5 files changed, 206 insertions(+), 121 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fa5e773..c8ed923 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,9 @@ build-backend = "setuptools.build_meta" name = "pyreports" version = "1.6.0" readme = "README.md" -license = {text = "GNU General Public License v3.0"} +license = { text = "GNU General Public License v3.0" } keywords = ['pyreports', 'reports', 'report', 'csv', 'yaml', 'export', - 'excel', 'database', 'ldap', 'dataset', 'file', 'executor', 'book'] + 'excel', 'database', 'ldap', 'dataset', 'file', 'executor', 'book'] authors = [{ name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }] maintainers = [ { name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }, @@ -21,7 +21,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = ['ldap3', 'mysql-connector-python', - 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] + 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] [project.scripts] reports = "pyreports.cli:main" diff --git a/pyreports/__init__.py b/pyreports/__init__.py index a2d3ddc..ecb70ab 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -22,10 +22,19 @@ """Build complex reports from/to various formats.""" -__version__ = '1.6.0' +__version__ = "1.6.0" from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 from .core import Executor, Report, ReportBook # noqa: F401 from .exception import ReportDataError, ReportManagerError # noqa: F401 -from .datatools import average, most_common, percentage, counter, aggregate, chunks, merge, deduplicate # noqa: F401 +from .datatools import ( + average, # noqa: F401 + most_common, # noqa: F401 + percentage, # noqa: F401 + counter, # noqa: F401 + aggregate, # noqa: F401 + chunks, # noqa: F401 + merge, # noqa: F401 + deduplicate, # noqa: F401 +) from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 diff --git a/pyreports/cli.py b/pyreports/cli.py index 2f69cc5..427003e 100644 --- a/pyreports/cli.py +++ b/pyreports/cli.py @@ -33,28 +33,38 @@ # endregion + # region functions def get_args(): """Get command-line arguments""" parser = argparse.ArgumentParser( - description='pyreports command line interface (CLI)', + description="pyreports command line interface (CLI)", formatter_class=argparse.ArgumentDefaultsHelpFormatter, - epilog='Full docs here: https://pyreports.readthedocs.io/en/latest/dev/cli.html' + epilog="Full docs here: https://pyreports.readthedocs.io/en/latest/dev/cli.html", ) - parser.add_argument('config', - metavar='CONFIG_FILE', - default=sys.stdin, - type=argparse.FileType('rt', encoding="utf-8"), - help='YAML configuration file') - parser.add_argument('-e', '--exclude', - help='Excluded title report list', - nargs=argparse.ZERO_OR_MORE, - default=[], - metavar='TITLE') - parser.add_argument('-v', '--verbose', help='Enable verbose mode', action='store_true') - parser.add_argument('-V', '--version', help='Print version', action='version', version=__version__) + parser.add_argument( + "config", + metavar="CONFIG_FILE", + default=sys.stdin, + type=argparse.FileType("rt", encoding="utf-8"), + help="YAML configuration file", + ) + parser.add_argument( + "-e", + "--exclude", + help="Excluded title report list", + nargs=argparse.ZERO_OR_MORE, + default=[], + metavar="TITLE", + ) + parser.add_argument( + "-v", "--verbose", help="Enable verbose mode", action="store_true" + ) + parser.add_argument( + "-V", "--version", help="Print version", action="version", version=__version__ + ) args = parser.parse_args() filename = args.config.name @@ -63,7 +73,7 @@ def get_args(): try: args.config = load_config(args.config) except yaml.YAMLError as err: - parser.error(f'file {filename} is not a valid YAML file: \n{err}') + parser.error(f"file {filename} is not a valid YAML file: \n{err}") # Validate config file try: @@ -71,7 +81,7 @@ def get_args(): except yaml.YAMLError as err: parser.error(str(err)) - print_verbose(f'parsed YAML file {filename}', verbose=args.verbose) + print_verbose(f"parsed YAML file {filename}", verbose=args.verbose) return args @@ -93,10 +103,10 @@ def validate_config(config): :return: None """ try: - reports = config['reports'] + reports = config["reports"] if reports is None or not isinstance(reports, list): raise yaml.YAMLError('"reports" section have not "report" list sections') - datas = all([bool(report.get('report').get('input')) for report in reports]) + datas = all([bool(report.get("report").get("input")) for report in reports]) if not datas: raise yaml.YAMLError( 'one of "report" section does not have "input" section' @@ -104,7 +114,9 @@ def validate_config(config): except KeyError as err: raise yaml.YAMLError(f'there is no "{err}" section') except AttributeError: - raise yaml.YAMLError('correctly indents the "report" section or "report" section not exists') + raise yaml.YAMLError( + 'correctly indents the "report" section or "report" section not exists' + ) def make_manager(input_config): @@ -114,12 +126,16 @@ def make_manager(input_config): :return: manager """ - type_ = input_config.get('manager') + type_ = input_config.get("manager") if type_ in pyreports.io.FILETYPE: - manager = pyreports.manager(input_config.get('manager'), input_config.get('filename', ())) + manager = pyreports.manager( + input_config.get("manager"), input_config.get("filename", ()) + ) else: - manager = pyreports.manager(input_config.get('manager'), **input_config.get('source', {})) + manager = pyreports.manager( + input_config.get("manager"), **input_config.get("source", {}) + ) return manager @@ -135,7 +151,7 @@ def get_data(manager, params=None): params = () data = None # FileManager - if manager.type == 'file': + if manager.type == "file": if params and isinstance(params, (list, tuple)): data = manager.read(*params) elif params and isinstance(params, dict): @@ -143,7 +159,7 @@ def get_data(manager, params=None): else: data = manager.read() # DatabaseManager - elif manager.type == 'sql': + elif manager.type == "sql": if params and isinstance(params, (list, tuple)): manager.execute(*params) data = manager.fetchall() @@ -151,13 +167,13 @@ def get_data(manager, params=None): manager.execute(**params) data = manager.fetchall() # LdapManager - elif manager.type == 'ldap': + elif manager.type == "ldap": if params and isinstance(params, (list, tuple)): data = manager.query(*params) elif params and isinstance(params, dict): data = manager.query(**params) # NosqlManager - elif manager.type == 'nosql': + elif manager.type == "nosql": if params and isinstance(params, (list, tuple)): data = manager.find(*params) elif params and isinstance(params, dict): @@ -174,7 +190,7 @@ def print_verbose(*messages, verbose=False): :return: None """ if verbose: - print('info:', *messages) + print("info:", *messages) def main(): @@ -184,71 +200,84 @@ def main(): args = get_args() # Take reports config = args.config - reports = config.get('reports', ()) + reports = config.get("reports", ()) - print_verbose(f'found {len(config.get("reports", ()))} report(s)', - verbose=args.verbose) + print_verbose( + f'found {len(config.get("reports", ()))} report(s)', verbose=args.verbose + ) # Build the data and report for report in reports: # Check if report isn't in excluded list - if args.exclude and report.get('report').get('title') in args.exclude: - print_verbose(f'exclude report "{report.get("report").get("title")}"', - verbose=args.verbose) + if args.exclude and report.get("report").get("title") in args.exclude: + print_verbose( + f'exclude report "{report.get("report").get("title")}"', + verbose=args.verbose, + ) continue # Make a manager object - input_ = report.get('report').get('input') - print_verbose(f'make an input manager of type {input_.get("manager")}', - verbose=args.verbose) + input_ = report.get("report").get("input") + print_verbose( + f'make an input manager of type {input_.get("manager")}', + verbose=args.verbose, + ) manager = make_manager(input_) # Get data - print_verbose(f'get data from manager {manager}', verbose=args.verbose) + print_verbose(f"get data from manager {manager}", verbose=args.verbose) try: # Make a report object - data = get_data(manager, input_.get('params')) - if 'map' in report.get('report'): - exec(report.get('report').get('map')) - map_func = globals().get('map_func') + data = get_data(manager, input_.get("params")) + if "map" in report.get("report"): + exec(report.get("report").get("map")) + map_func = globals().get("map_func") report_ = pyreports.Report( input_data=data, - title=report.get('report').get('title'), - filters=report.get('report').get('filters'), + title=report.get("report").get("title"), + filters=report.get("report").get("filters"), map_func=map_func, - negation=report.get('report').get('negation', False), - column=report.get('report').get('column'), - count=report.get('report').get('count', False), - output=make_manager(report.get('report').get('output')) if 'output' in report.get('report') else None + negation=report.get("report").get("negation", False), + column=report.get("report").get("column"), + count=report.get("report").get("count", False), + output=make_manager(report.get("report").get("output")) + if "output" in report.get("report") + else None, ) print_verbose(f'created report "{report_.title}"', verbose=args.verbose) except Exception as err: pyreports.Report(tablib.Dataset()) - exit(f'error: {err}') + exit(f"error: {err}") # Check output if report_.output: # Check if export or send report - if report.get('report').get('mail'): - print_verbose(f'send report to {report.get("report").get("mail").get("to")}', verbose=args.verbose) - mail_settings = report.get('report').get('mail') + if report.get("report").get("mail"): + print_verbose( + f'send report to {report.get("report").get("mail").get("to")}', + verbose=args.verbose, + ) + mail_settings = report.get("report").get("mail") report_.send( - server=mail_settings.get('server'), - _from=mail_settings.get('from'), - to=mail_settings.get('to'), - cc=mail_settings.get('cc'), - bcc=mail_settings.get('bcc'), - subject=mail_settings.get('subject'), - body=mail_settings.get('body'), - auth=tuple(mail_settings.get('auth')) if 'auth' in mail_settings else None, - _ssl=bool(mail_settings.get('ssl')), - headers=mail_settings.get('headers') + server=mail_settings.get("server"), + _from=mail_settings.get("from"), + to=mail_settings.get("to"), + cc=mail_settings.get("cc"), + bcc=mail_settings.get("bcc"), + subject=mail_settings.get("subject"), + body=mail_settings.get("body"), + auth=tuple(mail_settings.get("auth")) + if "auth" in mail_settings + else None, + _ssl=bool(mail_settings.get("ssl")), + headers=mail_settings.get("headers"), ) else: - print_verbose(f'export report to {report_.output}', - verbose=args.verbose) + print_verbose( + f"export report to {report_.output}", verbose=args.verbose + ) report_.export() else: # Print report in stdout - print_verbose('print report to stdout', verbose=args.verbose) - title = report.get('report').get('title') + print_verbose("print report to stdout", verbose=args.verbose) + title = report.get("report").get("title") report_.exec() print(f"{title}\n{'=' * len(title)}\n") print(report_) @@ -257,7 +286,7 @@ def main(): # endregion # region main -if __name__ == '__main__': +if __name__ == "__main__": main() # endregion diff --git a/pyreports/core.py b/pyreports/core.py index 70f9f03..5d83d5c 100644 --- a/pyreports/core.py +++ b/pyreports/core.py @@ -39,6 +39,7 @@ # region Classes + class Executor: """Executor receives, processes, transforms and writes data""" @@ -253,15 +254,17 @@ def clone(self): class Report: """Report represents the workflow for generating a report""" - def __init__(self, - input_data: tablib.Dataset, - title=None, - filters=None, - map_func=None, - negation=False, - column=None, - count=False, - output: Manager = None): + def __init__( + self, + input_data: tablib.Dataset, + title=None, + filters=None, + map_func=None, + negation=False, + column=None, + count=False, + output: Manager = None, + ): """Create Report object :param input_data: Dataset object @@ -277,7 +280,7 @@ def __init__(self, if isinstance(input_data, tablib.Dataset): self.data = input_data else: - raise ReportDataError('Only Dataset object is allowed for input') + raise ReportDataError("Only Dataset object is allowed for input") # Set other arguments self.title = title self.filter = filters @@ -288,10 +291,12 @@ def __init__(self, if isinstance(output, Manager) or output is None: if output: if output.__class__.__name__ not in WRITABLE_MANAGER: - raise ReportManagerError(f'Only {WRITABLE_MANAGER} object is allowed for output') + raise ReportManagerError( + f"Only {WRITABLE_MANAGER} object is allowed for output" + ) self.output = output else: - raise ReportManagerError('Only Manager object is allowed for output') + raise ReportManagerError("Only Manager object is allowed for output") # Data for report self.report = None @@ -346,7 +351,7 @@ def _print_data(self): :return: data and count """ if self.count: - out = f'{self.report}\nrows: {int(self.count)}' + out = f"{self.report}\nrows: {int(self.count)}" return out else: return self.report @@ -380,22 +385,21 @@ def export(self): # Process data before export self.exec() if self.output is not None: - if self.output.type == 'file': + if self.output.type == "file": self.output.write(self.report) - elif self.output.type == 'sql': + elif self.output.type == "sql": if not self.report.headers: raise ReportDataError("Dataset object doesn't have a header") - table_name = self.title.replace(' ', '_').lower() - fields = ','.join([ - f'{field} VARCHAR(255)' - for field in self.report.headers - ]) + table_name = self.title.replace(" ", "_").lower() + fields = ",".join( + [f"{field} VARCHAR(255)" for field in self.report.headers] + ) # Create table with header self.output.execute( f"CREATE TABLE IF NOT EXISTS {table_name} ({fields});" ) # Insert data into table - table_header = ','.join(field for field in self.report.headers) + table_header = ",".join(field for field in self.report.headers) for row in self.report: row_table = [f"'{element}'" for element in row] self.output.execute( @@ -407,7 +411,19 @@ def export(self): else: print(self) - def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth=None, _ssl=True, headers=None): + def send( + self, + server, + _from, + to, + cc=None, + bcc=None, + subject=None, + body="", + auth=None, + _ssl=True, + headers=None, + ): """Send saved report to email :param server: server SMTP @@ -423,7 +439,9 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth :return: None """ if not self.output: - raise ReportDataError('if you want send a mail with a report in attachment, must be specified output') + raise ReportDataError( + "if you want send a mail with a report in attachment, must be specified output" + ) # Prepare mail header message = MIMEMultipart("alternative") @@ -441,7 +459,7 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth elif isinstance(headers, (tuple, list)): message.add_header(*headers) else: - raise ValueError('headers must be tuple or List[tuple]') + raise ValueError("headers must be tuple or List[tuple]") # Prepare body part = MIMEText(body, "html") @@ -450,18 +468,22 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth # Prepare attachment self.export() attach_file_name = self.output.data.file - attach_file = open(attach_file_name, 'rb') - payload = MIMEBase('application', 'octate-stream') + attach_file = open(attach_file_name, "rb") + payload = MIMEBase("application", "octate-stream") payload.set_payload(attach_file.read()) encoders.encode_base64(payload) - payload.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attach_file_name)) + payload.add_header( + "Content-Disposition", + "attachment", + filename=os.path.basename(attach_file_name), + ) message.attach(payload) # Prepare SMTP connection if _ssl: port = smtplib.SMTP_SSL_PORT protocol = smtplib.SMTP_SSL - kwargs = {'context': ssl.create_default_context()} + kwargs = {"context": ssl.create_default_context()} else: port = smtplib.SMTP_PORT protocol = smtplib.SMTP @@ -469,7 +491,11 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth with protocol(server, port, **kwargs) as srv: if auth: srv.login(*auth) - srv.sendmail(_from, [receiver for receiver in (to, cc, bcc) if receiver], message.as_string()) + srv.sendmail( + _from, + [receiver for receiver in (to, cc, bcc) if receiver], + message.as_string(), + ) class ReportBook: @@ -495,7 +521,7 @@ def __add__(self, other): :return: ReportBook """ if not isinstance(other, ReportBook): - raise ReportDataError('you can only add ReportBook object') + raise ReportDataError("you can only add ReportBook object") self.reports.extend(other.reports) return self @@ -518,9 +544,8 @@ def __str__(self): :return: string """ - output = f'ReportBook {self.title if self.title else None}\n' - output += '\n'.join('\t' + str(report.title) - for report in self.reports) + output = f"ReportBook {self.title if self.title else None}\n" + output += "\n".join("\t" + str(report.title) for report in self.reports) return output def __repr__(self): @@ -561,7 +586,7 @@ def add(self, report: Report): :return: None """ if not isinstance(report, Report): - raise ReportDataError('you can only add Report object') + raise ReportDataError("you can only add Report object") self.reports.append(report) def remove(self, index: int = None): @@ -588,13 +613,25 @@ def export(self, output=None): # Prepare book for export book = tablib.Databook(tuple([report.report for report in self])) # Save Excel WorkBook - with open(output, 'wb') as f: - f.write(book.export('xlsx')) + with open(output, "wb") as f: + f.write(book.export("xlsx")) else: for report in self: report.export() - def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth=None, _ssl=True, headers=None): + def send( + self, + server, + _from, + to, + cc=None, + bcc=None, + subject=None, + body="", + auth=None, + _ssl=True, + headers=None, + ): """Send saved report to email :param server: server SMTP @@ -610,7 +647,18 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth :return: None """ for report in self: - report.send(server, _from, to, cc=cc, bcc=bcc, subject=subject, body=body, auth=auth, - _ssl=_ssl, headers=headers) + report.send( + server, + _from, + to, + cc=cc, + bcc=bcc, + subject=subject, + body=body, + auth=auth, + _ssl=_ssl, + headers=headers, + ) + # endregion diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 48b4a16..0b80a7b 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -30,6 +30,7 @@ # endregion + # region Functions def _select_column(data, column): """Select Dataset column @@ -40,7 +41,7 @@ def _select_column(data, column): """ # Check if dataset have a column if not data.headers: - raise ReportDataError('dataset object must have the headers') + raise ReportDataError("dataset object must have the headers") # Select column if isinstance(column, int): return data.get_col(column) @@ -60,7 +61,7 @@ def average(data, column): data = _select_column(data, column) # Check if all item is integer or float if not all(isinstance(item, (int, float)) for item in data): - raise ReportDataError('the column contains only int or float') + raise ReportDataError("the column contains only int or float") # Calculate average return float(sum(data) / len(data)) @@ -87,10 +88,7 @@ def percentage(data, filter_): :return: float """ # Filtering data... - data_filtered = [item - for row in data - for item in row - if filter_ == item] + data_filtered = [item for row in data for item in row if filter_ == item] quotient = len(data_filtered) / len(data) return quotient * 100 @@ -128,14 +126,14 @@ def aggregate(*columns, fill_empty: bool = False, fill_value=None): list_.append(fill_value() if callable(fill_value) else fill_value) else: if max_len != len(list_): - raise InvalidDimensions('the columns are not the same length') + raise InvalidDimensions("the columns are not the same length") max_len = len(list_) # Aggregate columns for column in columns: new_data.append_col(column) return new_data else: - raise ReportDataError('you can aggregate two or more columns') + raise ReportDataError("you can aggregate two or more columns") def merge(*datasets): @@ -151,11 +149,11 @@ def merge(*datasets): length_row = len(datasets[0][0]) for data in datasets: if length_row != len(data[0]): - raise InvalidDimensions('the row are not the same length') + raise InvalidDimensions("the row are not the same length") new_data.extend(data) return new_data else: - raise ReportDataError('you can merge two or more dataset object') + raise ReportDataError("you can merge two or more dataset object") def chunks(data, length): @@ -167,7 +165,7 @@ def chunks(data, length): :return: generator """ for i in range(0, len(data), length): - yield data[i:i + length] + yield data[i : i + length] def deduplicate(data): @@ -178,4 +176,5 @@ def deduplicate(data): """ return Dataset(*list(dict.fromkeys(data))) + # endregion From 0eed8650dafe158920d5670219b4337c292d6fd2 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 25 Feb 2024 12:49:40 +0100 Subject: [PATCH 04/58] chore: add DataAdapters class --- pyreports/datatools.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 0b80a7b..7afe702 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -31,6 +31,21 @@ # endregion +# region Classes +class DataAdapters: + """Data adapters class""" + + def __init__(self, input_data: Dataset): + # Discard all objects that are not Datasets + if isinstance(input_data, Dataset): + self.data = input_data + else: + raise ReportDataError("only Dataset object is allowed for input") + + +# endregion + + # region Functions def _select_column(data, column): """Select Dataset column @@ -164,8 +179,8 @@ def chunks(data, length): :param length: n-sized chunks :return: generator """ - for i in range(0, len(data), length): - yield data[i : i + length] + for idx in range(0, len(data), length): + yield data[idx : idx + length] def deduplicate(data): From 6fd15ac889c2167bc0d63d9b62606e94718c1dc1 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 26 Feb 2024 14:52:36 +0100 Subject: [PATCH 05/58] chore: add aggregate into DataAdapters class --- pyreports/datatools.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 7afe702..5811dfe 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -42,6 +42,18 @@ def __init__(self, input_data: Dataset): else: raise ReportDataError("only Dataset object is allowed for input") + def aggregate(self, *columns, fill_value=None): + """ + Aggregate in the current Dataset other columns + + :param columns: columns added + :param fill_value: fill value for empty field if "fill_empty" argument is specified + :return: None + """ + local_columns = [self.data[col] for col in self.data.headers] + local_columns.extend(columns) + self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) + # endregion From 2f8199ed5c76d00e8910829f03f9ce54c1c51ff3 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 27 Feb 2024 14:52:27 +0100 Subject: [PATCH 06/58] fix: indent report yaml file --- index.html | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/index.html b/index.html index 9641e58..33eed5e 100644 --- a/index.html +++ b/index.html @@ -245,24 +245,24 @@

Shell CLI


 $ cat car.yml
 reports:
-- report:
-  title: 'Red ford machine'
-  input:
-    manager: 'mysql'
-    source:
-    # Connection parameters of my mysql database
-      host: 'mysql1.local'
-      database: 'cars'
-      user: 'admin'
-      password: 'dba0000'
-    params:
-      query: 'SELECT * FROM cars WHERE brand = %s AND color = %s'
-      params: ['ford', 'red']
-  # Filter km
-  filters: [40000, 45000]
-  output:
-    manager: 'csv'
-    filename: '/tmp/car_csv.csv'
+  - report:
+    title: 'Red ford machine'
+    input:
+      manager: 'mysql'
+      source:
+      # Connection parameters of my mysql database
+        host: 'mysql1.local'
+        database: 'cars'
+        user: 'admin'
+        password: 'dba0000'
+      params:
+        query: 'SELECT * FROM cars WHERE brand = %s AND color = %s'
+        params: ['ford', 'red']
+    # Filter km
+    filters: [40000, 45000]
+    output:
+      manager: 'csv'
+      filename: '/tmp/car_csv.csv'
 
 $ report car.yaml
From d881473bcf8575e980de0c1dd44e4e426ddbed15 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 28 Feb 2024 11:06:42 +0100 Subject: [PATCH 07/58] fix: change version of cython --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84abb3c..f25c7de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,8 +8,7 @@ jobs: - checkout - run: sudo apt update && sudo apt install -y freetds-dev libssl-dev python-dev freetds-bin - run: sudo pip install --upgrade pip - - run: sudo pip install cython - - run: sudo pip install pymssql + - run: sudo pip install "Cython<3" - run: sudo pip install numpy - run: sudo pip install pandas - run: sudo python setup.py install From ed4374054b57ea5c6fe0d9a10339e6bdce27a60a Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 29 Feb 2024 10:31:38 +0100 Subject: [PATCH 08/58] fix: delete pymssql function test --- tests/test_db.py | 102 +++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 62 deletions(-) diff --git a/tests/test_db.py b/tests/test_db.py index 5acb974..50b53df 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -5,7 +5,6 @@ class TestDBConnection(unittest.TestCase): - def test_connection(self): # pyreports.io.Connection object self.assertRaises(TypeError, pyreports.io.Connection) @@ -13,86 +12,65 @@ def test_connection(self): def test_sqllite_connection(self): # Simulate pyreports.io.SQLliteConnection object conn = MagicMock() - with patch(target='sqlite3.connect') as mock: + with patch(target="sqlite3.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.database = 'mydb.db' - self.assertEqual(conn.connection.database, 'mydb.db') + conn.connection.database = "mydb.db" + self.assertEqual(conn.connection.database, "mydb.db") # Test close conn.cursor.close() def test_mysql_connection(self): # Simulate pyreports.io.MySQLConnection object conn = MagicMock() - with patch(target='mysql.connector.connect') as mock: + with patch(target="mysql.connector.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'mysqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "mysqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 3306 - self.assertEqual(conn.connection.host, 'mysqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') + self.assertEqual(conn.connection.host, "mysqldb.local") + self.assertEqual(conn.connection.database, "mydb") + self.assertEqual(conn.connection.username, "username") + self.assertEqual(conn.connection.password, "password") self.assertEqual(conn.connection.port, 3306) # Test close conn.cursor.close() - def test_mssqldb_connection(self): - # Simulate pyreports.io.MSSQLConnection object - conn = MagicMock() - with patch(target='pymssql.connect') as mock: - # Test connect - conn.connection = mock.return_value - conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'mssqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' - conn.connection.port = 1433 - self.assertEqual(conn.connection.host, 'mssqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') - self.assertEqual(conn.connection.port, 1433) - # Test close - conn.cursor.close() - def test_postgresqldb_connection(self): # Simulate pyreports.io.PostgreSQLConnection object conn = MagicMock() - with patch(target='psycopg2.connect') as mock: + with patch(target="psycopg2.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'postgresqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "postgresqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 5432 - self.assertEqual(conn.connection.host, 'postgresqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') + self.assertEqual(conn.connection.host, "postgresqldb.local") + self.assertEqual(conn.connection.database, "mydb") + self.assertEqual(conn.connection.username, "username") + self.assertEqual(conn.connection.password, "password") self.assertEqual(conn.connection.port, 5432) # Test close conn.cursor.close() class TestDBManager(unittest.TestCase): - conn = MagicMock() - with patch(target='psycopg2.connect') as mock: + with patch(target="psycopg2.connect") as mock: conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'postgresqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "postgresqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 5432 def test_db_manager(self): @@ -102,25 +80,24 @@ def test_db_manager(self): # Test reconnect db_manager.reconnect() # Test SELECT query - db_manager.execute('SELECT * from test') + db_manager.execute("SELECT * from test") data = db_manager.fetchall() self.assertIsInstance(data, Dataset) # Test store procedure - db_manager.callproc('myproc') + db_manager.callproc("myproc") data = db_manager.fetchone() self.assertIsInstance(data, Dataset) class TestNoSQLManager(unittest.TestCase): - conn = MagicMock() - with patch(target='nosqlapi.Connection') as mock: + with patch(target="nosqlapi.Connection") as mock: conn.connection = mock.return_value conn.session = conn.connection.connect() - conn.connection.host = 'mongodb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "mongodb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 27017 def test_nosql_manager(self): @@ -128,7 +105,7 @@ def test_nosql_manager(self): nosql_manager = pyreports.io.NoSQLManager(connection=self.conn) self.assertIsInstance(nosql_manager, pyreports.io.NoSQLManager) # Test get data - data = nosql_manager.get('doc1') + data = nosql_manager.get("doc1") self.assertIsInstance(data, Dataset) # Test find data data = nosql_manager.find({"name": "Arthur"}) @@ -136,11 +113,10 @@ def test_nosql_manager(self): class TestLDAPManager(unittest.TestCase): - conn = MagicMock() - with patch(target='ldap3.Server') as mock: + with patch(target="ldap3.Server") as mock: conn.connector = mock.return_value - with patch(target='ldap3.Connection') as mock: + with patch(target="ldap3.Connection") as mock: conn.bind = mock.return_value def test_bind(self): @@ -148,8 +124,10 @@ def test_bind(self): self.conn.bind.unbind() def test_query(self): - self.conn.bind.search('OU=test,DC=test,DC=local', 'objectCategory=person', ['name', 'sn', 'phone']) + self.conn.bind.search( + "OU=test,DC=test,DC=local", "objectCategory=person", ["name", "sn", "phone"] + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 31a600b4c081207f8e3d1af53be22fb45d550690 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 1 Mar 2024 09:43:18 +0100 Subject: [PATCH 09/58] docs: add readthedocs yaml configuration --- .readthedocs.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..94ecf89 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file From da841924c62250a92b60ad703dfeafef1059afa2 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 2 Mar 2024 10:19:27 +0100 Subject: [PATCH 10/58] docs: removed pymssql --- docs/requirements.txt | 1 - docs/source/conf.py | 23 ++++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3b7dffb..0f96a07 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,4 @@ ldap3 -pymssql mysql-connector-python psycopg2-binary tablib diff --git a/docs/source/conf.py b/docs/source/conf.py index 2177fab..0d19b94 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,28 +15,27 @@ # sys.path.insert(0, os.path.abspath('.')) import os import sys -sys.path.insert(0, os.path.abspath('../..')) -import __info__ +sys.path.insert(0, os.path.abspath("../..")) # -- Project information ----------------------------------------------------- -project = 'pyreports' -copyright = '2021, Matteo Guadrini' -author = __info__.__author__ +project = "pyreports" +copyright = "2024, Matteo Guadrini" +author = "Matteo Guadrini" # The full version, including alpha/beta/rc tags -release = __info__.__version__ +release = "1.6.0" # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -53,11 +52,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] -html_theme_options = { - 'logo_only': False -} +html_theme_options = {"logo_only": False} html_logo = "_static/pyreports.svg" -master_doc = 'index' +master_doc = "index" From 6a694732010b400e7febecf9ece07fd351ea2e62 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 3 Mar 2024 10:20:47 +0100 Subject: [PATCH 11/58] docs: removed pymssql...again --- docs/source/example.rst | 2 +- docs/source/managers.rst | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/example.rst b/docs/source/example.rst index 345f3ef..1e9da63 100644 --- a/docs/source/example.rst +++ b/docs/source/example.rst @@ -99,7 +99,7 @@ In this example, we will take two different inputs, and combine them to export a # Config Unix application file: this is a FileManager object config_file = pyreports.manager('yaml', '/home/myapp.yml') # Console admin: this is a DatabaseManager object - mydb = pyreports.manager('mssql', server='mssql1.local', database='admins', user='sa', password='sa0000') + mydb = pyreports.manager('mysql', server='mysql1.local', database='admins', user='sa', password='sa0000') # Get data admin_app = config_file.read() # return Dataset object: three column (name, shell, login) mydb.execute('SELECT * FROM console_admins') diff --git a/docs/source/managers.rst b/docs/source/managers.rst index a2667f0..4e98fa4 100644 --- a/docs/source/managers.rst +++ b/docs/source/managers.rst @@ -16,7 +16,6 @@ Each type of manager is managed by micro types; Below is the complete list: #. Database #. sqlite (SQLite) - #. mssql (Microsoft SQL) #. mysql (MySQL or MariaDB) #. postgresql (PostgreSQL or EnterpriseDB) #. File @@ -41,7 +40,6 @@ Each type of manager is managed by micro types; Below is the complete list: # DatabaseManager object sqlite_db = pyreports.manager('sqlite', database='/tmp/mydb.db') - mssql_db = pyreports.manager('mssql', server='mssql1.local', database='test', user='dba', password='dba0000') mysql_db = pyreports.manager('mysql', host='mysql1.local', database='test', user='dba', password='dba0000') postgresql_db = pyreports.manager('postgresql', host='postgresql1.local', database='test', user='dba', password='dba0000') From 1574c902016f7c4316459d81be7f8303c0a7baf1 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 4 Mar 2024 10:21:31 +0100 Subject: [PATCH 12/58] chore: add merge method on DataAdapters class --- pyreports/datatools.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 5811dfe..72864e6 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -43,8 +43,7 @@ def __init__(self, input_data: Dataset): raise ReportDataError("only Dataset object is allowed for input") def aggregate(self, *columns, fill_value=None): - """ - Aggregate in the current Dataset other columns + """Aggregate in the current Dataset other columns :param columns: columns added :param fill_value: fill value for empty field if "fill_empty" argument is specified @@ -54,6 +53,16 @@ def aggregate(self, *columns, fill_value=None): local_columns.extend(columns) self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) + def merge(self, *datasets): + """Merge in the current Dataset other Dataset objects + + :param datasets: datasets that will merge + :return: None + """ + datasets = list(datasets) + datasets += self.data + self.data = merge(*datasets) + # endregion From 39a7e54e8fdcf28339aaab309811f1500dd44d05 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 5 Mar 2024 11:08:24 +0100 Subject: [PATCH 13/58] chore: add DataAdapters class for import directly from package name --- pyreports/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyreports/__init__.py b/pyreports/__init__.py index ecb70ab..a326c40 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -36,5 +36,6 @@ chunks, # noqa: F401 merge, # noqa: F401 deduplicate, # noqa: F401 + DataAdapters, # noqa: F401 ) from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 From ccf0d0a5c2d43b83270a8b092f68a8ffc6fb22db Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 6 Mar 2024 09:06:51 +0100 Subject: [PATCH 14/58] chore: add test_data_adapters function --- tests/test_data.py | 73 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/tests/test_data.py b/tests/test_data.py index cab12fd..8c6769c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -7,14 +7,16 @@ class TestDataTools(unittest.TestCase): - data = Dataset(*[('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.headers = ['name', 'surname', 'age'] + data = Dataset( + *[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42), ("Ford", "Prefect", 42)] + ) + data.headers = ["name", "surname", "age"] def test_average(self): - self.assertEqual(int(pyreports.average(self.data, 'age')), 39) + self.assertEqual(int(pyreports.average(self.data, "age")), 39) def test_most_common(self): - self.assertEqual(pyreports.most_common(self.data, 'age'), 42) + self.assertEqual(pyreports.most_common(self.data, "age"), 42) def test_percentage(self): self.assertEqual(int(pyreports.percentage(self.data, 42)), 66) @@ -28,36 +30,63 @@ def test_aggregate(self): names = self.data.get_col(0) surnames = self.data.get_col(1) ages = self.data.get_col(2) - self.assertEqual(pyreports.aggregate(names, surnames, ages)[0], ('Matteo', 'Guadrini', 35)) - ages = ['Name', 'Surname'] - self.assertRaises(tablib.InvalidDimensions, pyreports.aggregate, names, - surnames, ages) + self.assertEqual( + pyreports.aggregate(names, surnames, ages)[0], ("Matteo", "Guadrini", 35) + ) + ages = ["Name", "Surname"] + self.assertRaises( + tablib.InvalidDimensions, pyreports.aggregate, names, surnames, ages + ) self.assertRaises(pyreports.ReportDataError, pyreports.aggregate, names) def test_aggregate_fill_empty(self): names = self.data.get_col(0) surnames = self.data.get_col(1) - ages = ['Name', 'Surname'] - self.assertEqual(pyreports.aggregate(names, surnames, ages, - fill_empty=True)[2], - ('Ford', 'Prefect', None)) + ages = ["Name", "Surname"] + self.assertEqual( + pyreports.aggregate(names, surnames, ages, fill_empty=True)[2], + ("Ford", "Prefect", None), + ) def test_chunks(self): - data = Dataset(*[('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.extend([('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.headers = ['name', 'surname', 'age'] - self.assertEqual(list(pyreports.chunks(data, 4))[0][0], ('Matteo', 'Guadrini', 35)) + data = Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.extend( + [ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.headers = ["name", "surname", "age"] + self.assertEqual( + list(pyreports.chunks(data, 4))[0][0], ("Matteo", "Guadrini", 35) + ) def test_merge(self): - self.assertEqual(pyreports.merge(self.data, self.data)[3], - ('Matteo', 'Guadrini', 35)) + self.assertEqual( + pyreports.merge(self.data, self.data)[3], ("Matteo", "Guadrini", 35) + ) def test_deduplication(self): - data = Dataset(*[('Matteo', 'Guadrini', 35), - ('Arthur', 'Dent', 42), - ('Matteo', 'Guadrini', 35)]) + data = Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) self.assertEqual(len(pyreports.deduplicate(data)), 2) + def test_data_adapters(self): + data = pyreports.DataAdapters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataAdapters) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() From 221508d58127c7bdcb7eb47daeb6ab9268cab903 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 7 Mar 2024 10:07:09 +0100 Subject: [PATCH 15/58] fix: aggregation also without header --- pyreports/datatools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 72864e6..7b7962a 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -49,7 +49,9 @@ def aggregate(self, *columns, fill_value=None): :param fill_value: fill value for empty field if "fill_empty" argument is specified :return: None """ - local_columns = [self.data[col] for col in self.data.headers] + if not self.data: + raise ReportDataError("dataset is empty") + local_columns = [self.data.get_col(col) for col in range(self.data.width)] local_columns.extend(columns) self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) From 3fdac9b6ec832057cdac371444971defc36c6316 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 8 Mar 2024 09:04:07 +0100 Subject: [PATCH 16/58] chore: add test_data_adapters_aggregate function --- tests/test_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 8c6769c..de1c899 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -87,6 +87,18 @@ def test_data_adapters(self): data = pyreports.DataAdapters(Dataset(*[("Matteo", "Guadrini", 35)])) self.assertIsInstance(data, pyreports.DataAdapters) + def test_data_adapters_aggregate(self): + names = self.data.get_col(0) + surnames = self.data.get_col(1) + ages = self.data.get_col(2) + data = pyreports.DataAdapters(Dataset()) + self.assertRaises( + pyreports.ReportDataError, data.aggregate, names, surnames, ages + ) + data = pyreports.DataAdapters(Dataset(*[("Heart",)])) + data.aggregate(names, surnames, ages) + self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) + if __name__ == "__main__": unittest.main() From 9f06af91bde5754ee658a42456356921dab6d3a0 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 9 Mar 2024 16:32:38 +0100 Subject: [PATCH 17/58] fix: add check id Datasets are empties --- pyreports/datatools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 7b7962a..b524756 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -62,7 +62,10 @@ def merge(self, *datasets): :return: None """ datasets = list(datasets) - datasets += self.data + datasets.append(self.data) + # Check if all Datasets are not empty + if not all([data for data in datasets]): + raise ReportDataError("one or more Datasets are empty") self.data = merge(*datasets) From 9b46736ede30033ae21b4118ca6eec11370b0f9b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 10 Mar 2024 16:56:03 +0100 Subject: [PATCH 18/58] chore: add test_data_adapters_merge function --- tests/test_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index de1c899..64624c7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -99,6 +99,12 @@ def test_data_adapters_aggregate(self): data.aggregate(names, surnames, ages) self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) + def test_data_adapters_merge(self): + data = pyreports.DataAdapters(Dataset()) + self.assertRaises(pyreports.ReportDataError, data.merge, self.data) + data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) + data.merge(self.data) + if __name__ == "__main__": unittest.main() From 570fb7c3f85e27486c8efb7140899219d4a3a2a2 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 11 Mar 2024 17:07:47 +0100 Subject: [PATCH 19/58] chore: add counter method into DataAdapters class --- pyreports/datatools.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index b524756..a84151f 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -63,11 +63,18 @@ def merge(self, *datasets): """ datasets = list(datasets) datasets.append(self.data) - # Check if all Datasets are not empty + # Check if all Datasets are not empties if not all([data for data in datasets]): - raise ReportDataError("one or more Datasets are empty") + raise ReportDataError("one or more Datasets are empties") self.data = merge(*datasets) + def counter(self): + """Count value into the rows + + :return: Counter + """ + return Counter((item for row in self.data for item in row)) + # endregion From 75182fead4d320ce56c0f72dfb21d471dc1d5b60 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 12 Mar 2024 11:14:24 +0100 Subject: [PATCH 20/58] chore: add test_data_adapters_counter function --- tests/test_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 64624c7..186decd 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -105,6 +105,12 @@ def test_data_adapters_merge(self): data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) data.merge(self.data) + def test_data_adapters_counter(self): + data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) + data.merge(self.data) + counter = data.counter() + self.assertEqual(counter["Arthur"], 2) + if __name__ == "__main__": unittest.main() From bf27f3bb35272809b9d8b82f56a68d50b907365a Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 13 Mar 2024 10:41:05 +0100 Subject: [PATCH 21/58] chore: add chunks method into DataAdapters class --- pyreports/datatools.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index a84151f..f841909 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -75,6 +75,16 @@ def counter(self): """ return Counter((item for row in self.data for item in row)) + def chunks(self, length): + """ + Yield successive n-sized chunks from Dataset + + :param length: n-sized chunks + :return: generator + """ + for idx in range(0, len(self.data), length): + yield self.data[idx : idx + length] + # endregion From c70daa6bf01dfcdf92ce5ec757e894bfc3456d60 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 14 Mar 2024 10:25:31 +0100 Subject: [PATCH 22/58] chore: add test_adapters_chunks function --- tests/test_data.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 186decd..b7af6af 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -100,6 +100,7 @@ def test_data_adapters_aggregate(self): self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) def test_data_adapters_merge(self): + data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) data = pyreports.DataAdapters(Dataset()) self.assertRaises(pyreports.ReportDataError, data.merge, self.data) data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) @@ -111,6 +112,26 @@ def test_data_adapters_counter(self): counter = data.counter() self.assertEqual(counter["Arthur"], 2) + def test_adapters_chunks(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.extend( + [ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.data.headers = ["name", "surname", "age"] + self.assertEqual(list(data.chunks(4))[0][0], ("Matteo", "Guadrini", 35)) + if __name__ == "__main__": unittest.main() From 068fa3db9a4f5fe4baeb25a31fea705d9cba081b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 15 Mar 2024 09:35:36 +0100 Subject: [PATCH 23/58] chore: add deduplicate method on DataAdapters class --- pyreports/datatools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index f841909..98fe3af 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -85,6 +85,13 @@ def chunks(self, length): for idx in range(0, len(self.data), length): yield self.data[idx : idx + length] + def deduplicate(self): + """Remove duplicated rows + + :return: Dataset + """ + self.data = Dataset(*list(dict.fromkeys(self.data))) + # endregion From c32e49f6215ee31c9515ce2868e6b8cca51212ff Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 17 Mar 2024 10:29:24 +0100 Subject: [PATCH 24/58] fix: add iter call into deduplicate functions --- pyreports/datatools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 98fe3af..651582c 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -88,9 +88,9 @@ def chunks(self, length): def deduplicate(self): """Remove duplicated rows - :return: Dataset + :return: None """ - self.data = Dataset(*list(dict.fromkeys(self.data))) + self.data = Dataset(*list(dict.fromkeys(iter(self.data)))) # endregion @@ -239,7 +239,7 @@ def deduplicate(data): :param data: Dataset object :return: Dataset """ - return Dataset(*list(dict.fromkeys(data))) + return Dataset(*list(dict.fromkeys(iter(data)))) # endregion From 1303d1a928f54a864f5858f39c4c6ee57464016c Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 18 Mar 2024 10:30:23 +0100 Subject: [PATCH 25/58] chore: add test_data_adapters_deduplicate function --- tests/test_data.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_data.py b/tests/test_data.py index b7af6af..8c5373b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -100,7 +100,6 @@ def test_data_adapters_aggregate(self): self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) def test_data_adapters_merge(self): - data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) data = pyreports.DataAdapters(Dataset()) self.assertRaises(pyreports.ReportDataError, data.merge, self.data) data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) @@ -132,6 +131,19 @@ def test_adapters_chunks(self): data.data.headers = ["name", "surname", "age"] self.assertEqual(list(data.chunks(4))[0][0], ("Matteo", "Guadrini", 35)) + def test_data_adapters_deduplicate(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) + ) + data.deduplicate() + self.assertEqual(len(data.data), 2) + if __name__ == "__main__": unittest.main() From b81849bf1f2510556513d8676bdb93a5dd0838a7 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 19 Mar 2024 13:09:20 +0100 Subject: [PATCH 26/58] chore: add iter method into DataAdapters class --- pyreports/datatools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 651582c..19d4877 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -92,6 +92,9 @@ def deduplicate(self): """ self.data = Dataset(*list(dict.fromkeys(iter(self.data)))) + def __iter__(self): + return (row for row in self.data) + # endregion From ea0e8eb2206bec02997280e22971aa18975caed9 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 20 Mar 2024 09:02:08 +0100 Subject: [PATCH 27/58] chore: add test_data_adapters_iter function --- tests/test_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 8c5373b..c4f27cf 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -144,6 +144,18 @@ def test_data_adapters_deduplicate(self): data.deduplicate() self.assertEqual(len(data.data), 2) + def test_data_adapters_iter(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) + ) + self.assertEqual(list(iter(data.data))[1], ("Arthur", "Dent", 42)) + if __name__ == "__main__": unittest.main() From 1dfe12b1cdd841108192c38ebb5603b072b08a50 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 21 Mar 2024 09:24:00 +0100 Subject: [PATCH 28/58] chore: add getitem method into DataAdapters class --- pyreports/datatools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 19d4877..1741d4c 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -95,6 +95,9 @@ def deduplicate(self): def __iter__(self): return (row for row in self.data) + def __getitem__(self, item): + return self.data[item] + # endregion From efc3d569039078a6610547e7e688abc58cd9d1b6 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 22 Mar 2024 14:15:13 +0100 Subject: [PATCH 29/58] chore: add test_data_adapters_get_items function --- tests/test_data.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index c4f27cf..b56f022 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -156,6 +156,22 @@ def test_data_adapters_iter(self): ) self.assertEqual(list(iter(data.data))[1], ("Arthur", "Dent", 42)) + def test_data_adapters_get_items(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ], + headers=("name", "surname", "age"), + ) + ) + # Get row + self.assertEqual(data[1], ("Arthur", "Dent", 42)) + # Get column + self.assertEqual(data["name"], ["Matteo", "Arthur", "Matteo"]) + if __name__ == "__main__": unittest.main() From 118c37d324cdd021502973c291c813a654ce09df Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 23 Mar 2024 09:13:16 +0100 Subject: [PATCH 30/58] chore: change data attribute into property --- pyreports/datatools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 1741d4c..ca63582 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -38,10 +38,14 @@ class DataAdapters: def __init__(self, input_data: Dataset): # Discard all objects that are not Datasets if isinstance(input_data, Dataset): - self.data = input_data + self._data = input_data else: raise ReportDataError("only Dataset object is allowed for input") + @property + def data(self): + return self._data + def aggregate(self, *columns, fill_value=None): """Aggregate in the current Dataset other columns From e2ad9e66b6c58c62769f29fe8365cf017f44f35b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 24 Mar 2024 09:14:53 +0100 Subject: [PATCH 31/58] chore: add setter data property --- pyreports/datatools.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index ca63582..9f43db7 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -46,6 +46,10 @@ def __init__(self, input_data: Dataset): def data(self): return self._data + @data.setter + def data(self, dataset): + self._data = dataset + def aggregate(self, *columns, fill_value=None): """Aggregate in the current Dataset other columns From dee8e1a858fe138fb5354df91e23e2e6674433db Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 25 Mar 2024 09:25:37 +0100 Subject: [PATCH 32/58] chore: add DataObject class --- pyreports/datatools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 9f43db7..ea4b2a3 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -32,8 +32,8 @@ # region Classes -class DataAdapters: - """Data adapters class""" +class DataObject: + """Data object class""" def __init__(self, input_data: Dataset): # Discard all objects that are not Datasets @@ -50,6 +50,10 @@ def data(self): def data(self, dataset): self._data = dataset + +class DataAdapters(DataObject): + """Data adapters class""" + def aggregate(self, *columns, fill_value=None): """Aggregate in the current Dataset other columns From ee7b42288ae551e84be59b145015ad91f5f005b9 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 26 Mar 2024 13:16:45 +0100 Subject: [PATCH 33/58] chore: add DataPrinters class --- pyreports/datatools.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index ea4b2a3..6034dcd 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -111,6 +111,17 @@ def __getitem__(self, item): return self.data[item] +class DataPrinters(DataObject): + """Data printers class""" + + def print(self): + """Print data + + :return: None + """ + print(self.data) + + # endregion From 048a7101a0777fd9c49493ded47d6255d6669996 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 27 Mar 2024 10:19:02 +0100 Subject: [PATCH 34/58] chore: add DataObject class for import --- pyreports/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyreports/__init__.py b/pyreports/__init__.py index a326c40..b41df54 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -36,6 +36,7 @@ chunks, # noqa: F401 merge, # noqa: F401 deduplicate, # noqa: F401 + DataObject, # noqa: F401 DataAdapters, # noqa: F401 ) from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 From 17b33c1298aec5c129ad1354ec63f3e489fe982b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 28 Mar 2024 09:27:41 +0100 Subject: [PATCH 35/58] chore: add test_data_object function --- tests/test_data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index b56f022..1691ede 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -83,6 +83,11 @@ def test_deduplication(self): ) self.assertEqual(len(pyreports.deduplicate(data)), 2) + def test_data_object(self): + data = pyreports.DataObject(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataObject) + self.assertIsInstance(data.data, tablib.Dataset) + def test_data_adapters(self): data = pyreports.DataAdapters(Dataset(*[("Matteo", "Guadrini", 35)])) self.assertIsInstance(data, pyreports.DataAdapters) From 3f3900a4f73650fa590a55d4ded7a8a6a9fb4d79 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 29 Mar 2024 09:11:01 +0100 Subject: [PATCH 36/58] chore: add repr and str function into DataPrinters class --- pyreports/__init__.py | 1 + pyreports/datatools.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pyreports/__init__.py b/pyreports/__init__.py index b41df54..3a7cbd2 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -38,5 +38,6 @@ deduplicate, # noqa: F401 DataObject, # noqa: F401 DataAdapters, # noqa: F401 + DataPrinters, # noqa: F401 ) from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 6034dcd..5e40a51 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -121,6 +121,20 @@ def print(self): """ print(self.data) + def __repr__(self): + """Representation of DataObject + + :return: string + """ + return f"" + + def __str__(self): + """Pretty representation of DataObject + + :return: string + """ + return str(self.data) + # endregion From 4c52bc6a14f3ac30298cf9ce4c3d9d791a947932 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 30 Mar 2024 11:11:34 +0100 Subject: [PATCH 37/58] fix: change print data --- pyreports/datatools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 5e40a51..88da3c0 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -119,7 +119,7 @@ def print(self): :return: None """ - print(self.data) + print(self) def __repr__(self): """Representation of DataObject From 3cb2585da8f15df5c60b003b09c97ddb339d51a5 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 31 Mar 2024 12:21:30 +0200 Subject: [PATCH 38/58] change: add average method into DataPrinters class --- pyreports/datatools.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 88da3c0..29ede6c 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -121,6 +121,21 @@ def print(self): """ print(self) + def average(self, column): + """ + Average of list of integers or floats + + :param column: column name or index + :return: float + """ + # Select column + data = _select_column(self.data, column) + # Check if all item is integer or float + if not all(isinstance(item, (int, float)) for item in data): + raise ReportDataError("the column contains only int or float") + # Calculate average + return float(sum(data) / len(data)) + def __repr__(self): """Representation of DataObject From fbd90520eae4e12274a5abf96a2c85894fb4347b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 1 Apr 2024 12:23:18 +0200 Subject: [PATCH 39/58] change: add most_common method into DataPrinters class --- pyreports/datatools.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 29ede6c..5d074a7 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -122,19 +122,20 @@ def print(self): print(self) def average(self, column): - """ - Average of list of integers or floats + """Average of list of integers or floats :param column: column name or index :return: float """ - # Select column - data = _select_column(self.data, column) - # Check if all item is integer or float - if not all(isinstance(item, (int, float)) for item in data): - raise ReportDataError("the column contains only int or float") - # Calculate average - return float(sum(data) / len(data)) + average(self.data, column) + + def most_common(self, column): + """The most common element in a column + + :param column: column name or index + :return: Any + """ + most_common(self.data, column) def __repr__(self): """Representation of DataObject From 5995c4a8d45725042fab6f0264b96bd5f8cce5db Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 2 Apr 2024 12:27:12 +0200 Subject: [PATCH 40/58] chore: add percentage method into DataPrinters class --- pyreports/datatools.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 5d074a7..45d6ee9 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -127,7 +127,7 @@ def average(self, column): :param column: column name or index :return: float """ - average(self.data, column) + return average(self.data, column) def most_common(self, column): """The most common element in a column @@ -135,7 +135,15 @@ def most_common(self, column): :param column: column name or index :return: Any """ - most_common(self.data, column) + return most_common(self.data, column) + + def percentage(self, filter_): + """Calculating the percentage according to filter + + :param filter_: equality filter + :return: float + """ + return percentage(self.data, filter_) def __repr__(self): """Representation of DataObject @@ -208,7 +216,7 @@ def percentage(data, filter_): Calculating the percentage according to filter :param data: Dataset object - :param filter_: filter + :param filter_: equality filter :return: float """ # Filtering data... From 4093526993c2e1b2c9b9daa99b07d0847b57030d Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 3 Apr 2024 10:12:26 +0200 Subject: [PATCH 41/58] chore: add len method into DataPrinters class --- pyreports/datatools.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 45d6ee9..9848468 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -150,7 +150,7 @@ def __repr__(self): :return: string """ - return f"" + return f"" def __str__(self): """Pretty representation of DataObject @@ -159,6 +159,13 @@ def __str__(self): """ return str(self.data) + def __len__(self): + """Measure length of DataSet + + :return: int + """ + return len(self.data) + # endregion From b9866bf1b037171301b99a16bc0dfb5e7e939add Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 4 Apr 2024 10:35:03 +0200 Subject: [PATCH 42/58] chore: add test_data_printers function --- tests/test_data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 1691ede..e55561b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -177,6 +177,11 @@ def test_data_adapters_get_items(self): # Get column self.assertEqual(data["name"], ["Matteo", "Arthur", "Matteo"]) + def test_data_printers(self): + data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataPrinters) + self.assertIsInstance(data.data, tablib.Dataset) + if __name__ == "__main__": unittest.main() From f4219d656d55e6b19d3a8d62473bafc6b8e4ce74 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 5 Apr 2024 09:11:02 +0200 Subject: [PATCH 43/58] chore: add test_data_printers_len function --- tests/test_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index e55561b..80b1340 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -182,6 +182,10 @@ def test_data_printers(self): self.assertIsInstance(data, pyreports.DataPrinters) self.assertIsInstance(data.data, tablib.Dataset) + def test_data_printers_len(self): + data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertEqual(1, len(data)) + if __name__ == "__main__": unittest.main() From 2671de45217b5ef7a0b6ea0b1690f619ecaed5a1 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 6 Apr 2024 12:49:39 +0200 Subject: [PATCH 44/58] chore: add test_data_printers_average function --- tests/test_data.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 80b1340..4623f5d 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -186,6 +186,13 @@ def test_data_printers_len(self): data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) self.assertEqual(1, len(data)) + def test_data_printers_average(self): + data = pyreports.DataPrinters( + Dataset(*[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)]) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.average(2), 38.5) + if __name__ == "__main__": unittest.main() From f7daaa90388b56ec2db64f2e3f39e689b45c7f6e Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 7 Apr 2024 12:55:12 +0200 Subject: [PATCH 45/58] chore: add test_data_printers_most_common function --- tests/test_data.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index 4623f5d..d85b563 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -193,6 +193,19 @@ def test_data_printers_average(self): data.data.headers = ["Name", "Surname", "Age"] self.assertEqual(data.average(2), 38.5) + def test_data_printers_most_common(self): + data = pyreports.DataPrinters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.most_common("Age"), 42) + if __name__ == "__main__": unittest.main() From 2d21537c5ebf9f96970eae40d57256511c107bcd Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 8 Apr 2024 13:01:12 +0200 Subject: [PATCH 46/58] chore: add test_data_printers_percentage function --- tests/test_data.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_data.py b/tests/test_data.py index d85b563..c9d9e37 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -206,6 +206,19 @@ def test_data_printers_most_common(self): data.data.headers = ["Name", "Surname", "Age"] self.assertEqual(data.most_common("Age"), 42) + def test_data_printers_percentage(self): + data = pyreports.DataPrinters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.percentage(42), 66.66666666666666) + if __name__ == "__main__": unittest.main() From 7f54615713bb6fcce5a17113b302afe8f9c0d660 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 9 Apr 2024 14:25:16 +0200 Subject: [PATCH 47/58] chore: add DataAdapters and DataPrinters classes into Report class --- pyreports/core.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyreports/core.py b/pyreports/core.py index 5d83d5c..8377656 100644 --- a/pyreports/core.py +++ b/pyreports/core.py @@ -31,6 +31,7 @@ from email.mime.multipart import MIMEMultipart from email import encoders from email.mime.base import MIMEBase +from .datatools import DataAdapters, DataPrinters from .io import Manager, WRITABLE_MANAGER from .exception import ReportManagerError, ReportDataError, ExecutorError @@ -251,7 +252,7 @@ def clone(self): return Executor(self.origin, header=self.origin.headers) -class Report: +class Report(DataAdapters, DataPrinters): """Report represents the workflow for generating a report""" def __init__( @@ -277,10 +278,7 @@ def __init__( :param output: Manager object """ # Discard all objects that are not Datasets - if isinstance(input_data, tablib.Dataset): - self.data = input_data - else: - raise ReportDataError("Only Dataset object is allowed for input") + DataAdapters.__init__(self, input_data=input_data) # Set other arguments self.title = title self.filter = filters From e79bb9de9c71bd715fba134a3802b0041010eac3 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 10 Apr 2024 12:56:57 +0200 Subject: [PATCH 48/58] docs: add DataObject class --- docs/source/datatools.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 5294a88..0184a04 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -8,6 +8,20 @@ In this section we will see all these functions contained in the **datatools** m .. toctree:: +DataObject +---------- + +**DataObject** class represents a pure *Dataset*. + +.. autoclass:: pyreports.DataObject + :members: + +.. code-block:: python + + import pyreports + + data = pyreports.DataObject(tablib.Dataset(*[("Arthur", "Dent", 42)])) + assert isinstance(data.data, tablib.Dataset) == True Average From ced268cd385fc06093543105062b35caffe3e1ea Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 11 Apr 2024 09:52:10 +0200 Subject: [PATCH 49/58] docs: add DataAdapters class --- docs/source/datatools.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 0184a04..0aaf584 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -24,6 +24,24 @@ DataObject assert isinstance(data.data, tablib.Dataset) == True + +DataAdapters +------------ + +**DataAdapters** class is an object that contains methods that modifying *Dataset*. + +.. code-block:: python + + import pyreports + + data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)])) + assert isinstance(data.data, tablib.Dataset) == True + + +.. autoclass:: pyreports.DataAdapters + :members: + + Average ------- From c479e46891b10bf383f9d6be10cf094c105b05ab Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 12 Apr 2024 16:37:19 +0200 Subject: [PATCH 50/58] docs: add DataPrinters class --- docs/source/datatools.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 0aaf584..4215040 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -34,13 +34,29 @@ DataAdapters import pyreports - data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)])) + data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)], headers=["name", "surname", "age"])) assert isinstance(data.data, tablib.Dataset) == True .. autoclass:: pyreports.DataAdapters :members: +DataPrinters +------------ + +**DataPrinters** class is an object that contains methods that printing *Dataset*'s information. + +.. code-block:: python + + import pyreports + + data = pyreports.DataPrinters(tablib.Dataset(*[("Arthur", "Dent", 42)], headers=["name", "surname", "age"])) + assert isinstance(data.data, tablib.Dataset) == True + + +.. autoclass:: pyreports.DataPrinters + :members: + Average ------- From 31a9a6032fc570e5c56fcbb01a78f0ebde4d7287 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sat, 13 Apr 2024 12:21:38 +0200 Subject: [PATCH 51/58] docs: add all methods of DataAdapters class --- docs/source/datatools.rst | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 4215040..6926139 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -18,7 +18,7 @@ DataObject .. code-block:: python - import pyreports + import pyreports, tablib data = pyreports.DataObject(tablib.Dataset(*[("Arthur", "Dent", 42)])) assert isinstance(data.data, tablib.Dataset) == True @@ -32,12 +32,35 @@ DataAdapters .. code-block:: python - import pyreports + import pyreports, tablib - data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)], headers=["name", "surname", "age"])) + data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)])) assert isinstance(data.data, tablib.Dataset) == True + # Aggregate + planets = tablib.Dataset(*[("Heart",)]) + data.aggregate(planets) + + # Merge + others = tablib.Dataset(*[("Betelgeuse", "Ford", "Prefect", 42)]) + data.merge(others) + + # Counter + data = pyreports.DataAdapters(Dataset(*[("Heart", "Arthur", "Dent", 42)])) + data.merge(self.data) + counter = data.counter() + assert counter["Arthur"] == 2 + + # Chunks + data.data.headers = ["planet", "name", "surname", "age"] + assert list(data.chunks(4))[0][0] == ("Heart", "Arthur", "Dent", 42) + + # Deduplicate + data.deduplicate() + assert len(data.data) == 2 + + .. autoclass:: pyreports.DataAdapters :members: From 570bca97f54f9295a7895561377486db8f1ce8ab Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Sun, 14 Apr 2024 12:31:11 +0200 Subject: [PATCH 52/58] docs: add all dunder methods of DataAdapters class --- docs/source/datatools.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 6926139..c1c5132 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -60,6 +60,13 @@ DataAdapters data.deduplicate() assert len(data.data) == 2 + # Get items + assert data[1] == ("Betelgeuse", "Ford", "Prefect", 42) + + # Iter items + for item in data: + print(item) + .. autoclass:: pyreports.DataAdapters :members: From 147b492cbcaa081518f43bd2e0330eecedd65f62 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Mon, 15 Apr 2024 13:04:02 +0200 Subject: [PATCH 53/58] docs: add all methods of DataPrinters class --- docs/source/datatools.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index c1c5132..f8fbeba 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -78,11 +78,25 @@ DataPrinters .. code-block:: python - import pyreports + import pyreports, tablib - data = pyreports.DataPrinters(tablib.Dataset(*[("Arthur", "Dent", 42)], headers=["name", "surname", "age"])) + data = pyreports.DataPrinters(tablib.Dataset(*[("Arthur", "Dent", 42), ("Ford", "Prefect", 42)], headers=["name", "surname", "age"])) assert isinstance(data.data, tablib.Dataset) == True + # Print + data.print() + + # Average + assert data.average(2) == 42 + assert data.average("age") == 42 + + # Most common + data.data.append(("Ford", "Prefect", 42)) + assert data.most_common(0) == "Ford" + assert data.most_common("name") == "Ford" + + # Percentage + assert data.percentage("Ford") == 66.66666666666666 .. autoclass:: pyreports.DataPrinters :members: From 88066e8199a19e279a69592c00a98bc5fe0dc12b Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Tue, 16 Apr 2024 10:08:46 +0200 Subject: [PATCH 54/58] docs: add all dunder methods of DataPrinters class --- docs/source/datatools.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index f8fbeba..c63194b 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -98,6 +98,15 @@ DataPrinters # Percentage assert data.percentage("Ford") == 66.66666666666666 + # Representation + assert repr(data) == "" + + # String + assert str(data) == 'name |surname|age\n------|-------|---\nArthur|Dent |42 \nFord |Prefect|42 \nFord |Prefect|42 ' + + # Length + assert len(data) == 3 + .. autoclass:: pyreports.DataPrinters :members: From e5f956c4da26ce0ca48e1e70ec515d950aac17f0 Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Wed, 17 Apr 2024 09:25:19 +0200 Subject: [PATCH 55/58] docs: add deduplicate function --- docs/source/datatools.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index c63194b..ad78435 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -238,4 +238,20 @@ The **chunks** function divides a *Dataset* into pieces from *N* (``int``). This print(list(new_data)) # [[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], [('Tricia', 'McMillian', 55000), ('Zaphod', 'Beeblebrox', 65000)]] .. note:: - If the division does not result zero, the last tuple of elements will be a smaller number. \ No newline at end of file + If the division does not result zero, the last tuple of elements will be a smaller number. + +Deduplicate +----------- + +The **deduplicate** function remove duplicated rows into *Dataset* objects. + +.. code-block:: python + + import pyreports + + # Build a datasets + employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) + + # Remove duplicated rows (removed the last ('Ford', 'Prefect', 65000)) + pyreports.deduplicate(employee1) + print(len(employee1)) # 2 \ No newline at end of file From 3b2144c96d40da89e4aa5ba32fce0673401cc3fa Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Thu, 18 Apr 2024 10:15:05 +0200 Subject: [PATCH 56/58] chore: add DataObjectError exception class --- pyreports/__init__.py | 2 +- pyreports/datatools.py | 16 ++++++++-------- pyreports/exception.py | 6 +++++- tests/test_data.py | 6 +++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pyreports/__init__.py b/pyreports/__init__.py index 3a7cbd2..0221d24 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -26,7 +26,7 @@ from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 from .core import Executor, Report, ReportBook # noqa: F401 -from .exception import ReportDataError, ReportManagerError # noqa: F401 +from .exception import ReportDataError, ReportManagerError, DataObjectError # noqa: F401 from .datatools import ( average, # noqa: F401 most_common, # noqa: F401 diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 9848468..4c6c1f3 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -23,7 +23,7 @@ """Contains all functions for data processing.""" # region Imports -from .exception import ReportDataError +from .exception import DataObjectError from collections import Counter from tablib import Dataset, InvalidDimensions @@ -40,7 +40,7 @@ def __init__(self, input_data: Dataset): if isinstance(input_data, Dataset): self._data = input_data else: - raise ReportDataError("only Dataset object is allowed for input") + raise DataObjectError("only Dataset object is allowed for input") @property def data(self): @@ -62,7 +62,7 @@ def aggregate(self, *columns, fill_value=None): :return: None """ if not self.data: - raise ReportDataError("dataset is empty") + raise DataObjectError("dataset is empty") local_columns = [self.data.get_col(col) for col in range(self.data.width)] local_columns.extend(columns) self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) @@ -77,7 +77,7 @@ def merge(self, *datasets): datasets.append(self.data) # Check if all Datasets are not empties if not all([data for data in datasets]): - raise ReportDataError("one or more Datasets are empties") + raise DataObjectError("one or more Datasets are empties") self.data = merge(*datasets) def counter(self): @@ -180,7 +180,7 @@ def _select_column(data, column): """ # Check if dataset have a column if not data.headers: - raise ReportDataError("dataset object must have the headers") + raise DataObjectError("dataset object must have the headers") # Select column if isinstance(column, int): return data.get_col(column) @@ -200,7 +200,7 @@ def average(data, column): data = _select_column(data, column) # Check if all item is integer or float if not all(isinstance(item, (int, float)) for item in data): - raise ReportDataError("the column contains only int or float") + raise DataObjectError("the column contains only int or float") # Calculate average return float(sum(data) / len(data)) @@ -272,7 +272,7 @@ def aggregate(*columns, fill_empty: bool = False, fill_value=None): new_data.append_col(column) return new_data else: - raise ReportDataError("you can aggregate two or more columns") + raise DataObjectError("you can aggregate two or more columns") def merge(*datasets): @@ -292,7 +292,7 @@ def merge(*datasets): new_data.extend(data) return new_data else: - raise ReportDataError("you can merge two or more dataset object") + raise DataObjectError("you can merge two or more dataset object") def chunks(data, length): diff --git a/pyreports/exception.py b/pyreports/exception.py index ff6d96b..074e0c9 100644 --- a/pyreports/exception.py +++ b/pyreports/exception.py @@ -28,7 +28,11 @@ class ExecutorError(Exception): # ReportException hierarchy -class ReportException(Exception): +class DataObjectError(Exception): + pass + + +class ReportException(DataObjectError): pass diff --git a/tests/test_data.py b/tests/test_data.py index c9d9e37..4810e0b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -37,7 +37,7 @@ def test_aggregate(self): self.assertRaises( tablib.InvalidDimensions, pyreports.aggregate, names, surnames, ages ) - self.assertRaises(pyreports.ReportDataError, pyreports.aggregate, names) + self.assertRaises(pyreports.DataObjectError, pyreports.aggregate, names) def test_aggregate_fill_empty(self): names = self.data.get_col(0) @@ -98,7 +98,7 @@ def test_data_adapters_aggregate(self): ages = self.data.get_col(2) data = pyreports.DataAdapters(Dataset()) self.assertRaises( - pyreports.ReportDataError, data.aggregate, names, surnames, ages + pyreports.DataObjectError, data.aggregate, names, surnames, ages ) data = pyreports.DataAdapters(Dataset(*[("Heart",)])) data.aggregate(names, surnames, ages) @@ -106,7 +106,7 @@ def test_data_adapters_aggregate(self): def test_data_adapters_merge(self): data = pyreports.DataAdapters(Dataset()) - self.assertRaises(pyreports.ReportDataError, data.merge, self.data) + self.assertRaises(pyreports.DataObjectError, data.merge, self.data) data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) data.merge(self.data) From 86bcbca5f5730f4910bb32967f5b61d78261178f Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 19 Apr 2024 09:52:47 +0200 Subject: [PATCH 57/58] chore: change copyright year --- pyreports/cli.py | 2 +- pyreports/core.py | 2 +- pyreports/datatools.py | 2 +- pyreports/exception.py | 2 +- pyreports/io.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyreports/cli.py b/pyreports/cli.py index 427003e..808b5bb 100644 --- a/pyreports/cli.py +++ b/pyreports/cli.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # cli -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/pyreports/core.py b/pyreports/core.py index 8377656..15f9c15 100644 --- a/pyreports/core.py +++ b/pyreports/core.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # core -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 4c6c1f3..b15728d 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # datatools -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/pyreports/exception.py b/pyreports/exception.py index 074e0c9..2016796 100644 --- a/pyreports/exception.py +++ b/pyreports/exception.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # exception.py -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/pyreports/io.py b/pyreports/io.py index 377e068..645b553 100644 --- a/pyreports/io.py +++ b/pyreports/io.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # io -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 856c41b7871977d2c1880fb2279e801ebffe9e4c Mon Sep 17 00:00:00 2001 From: Matteo Guadrini Date: Fri, 19 Apr 2024 10:10:54 +0200 Subject: [PATCH 58/58] chore: new minor version --- CHANGES.md | 23 +++++++++++++++++++++++ docs/source/conf.py | 2 +- pyproject.toml | 2 +- pyreports/__init__.py | 4 ++-- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5877494..15d1a33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,28 @@ # Release notes +## 1.7.0 +Apr 19, 2024 + +- Remove **pymssql** support +- Add **DataObject** class +- Add **DataAdapters** class +- Add **DataPrinters** class +- Add **aggregate** into _DataAdapters_ class +- Add **merge** method into _DataAdapters_ class +- Add **counter** method into _DataAdapters_ class +- Add **chunks** method into _DataAdapters_ class +- Add **deduplicate** method into _DataAdapters_ class +- Add **iter** method into _DataAdapters_ class +- Add **getitem** method into _DataAdapters_ class +- Add **getitem** method into _DataAdapters_ class +- Add **repr** and **str** function into _DataPrinters_ class +- Add **average** method into _DataPrinters_ class +- Add **most_common** method into _DataPrinters_ class +- Add **percentage** method into _DataPrinters_ class +- Add **len** method into _DataPrinters_ class +- Add **DataAdapters** and **DataPrinters** classes into _Report_ class +- Add **DataObjectError** exception class + ## 1.6.0 Jul 14, 2023 diff --git a/docs/source/conf.py b/docs/source/conf.py index 0d19b94..2a350d1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,7 +25,7 @@ author = "Matteo Guadrini" # The full version, including alpha/beta/rc tags -release = "1.6.0" +release = "1.7.0" # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index c8ed923..dc0e17c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyreports" -version = "1.6.0" +version = "1.7.0" readme = "README.md" license = { text = "GNU General Public License v3.0" } keywords = ['pyreports', 'reports', 'report', 'csv', 'yaml', 'export', diff --git a/pyreports/__init__.py b/pyreports/__init__.py index 0221d24..866ea5d 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # __init__ -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,7 +22,7 @@ """Build complex reports from/to various formats.""" -__version__ = "1.6.0" +__version__ = "1.7.0" from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 from .core import Executor, Report, ReportBook # noqa: F401