diff --git a/.circleci/config.yml b/.circleci/config.yml index 84abb3c..f25c7de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,8 +8,7 @@ jobs: - checkout - run: sudo apt update && sudo apt install -y freetds-dev libssl-dev python-dev freetds-bin - run: sudo pip install --upgrade pip - - run: sudo pip install cython - - run: sudo pip install pymssql + - run: sudo pip install "Cython<3" - run: sudo pip install numpy - run: sudo pip install pandas - run: sudo python setup.py install diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..94ecf89 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 5877494..15d1a33 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,28 @@ # Release notes +## 1.7.0 +Apr 19, 2024 + +- Remove **pymssql** support +- Add **DataObject** class +- Add **DataAdapters** class +- Add **DataPrinters** class +- Add **aggregate** into _DataAdapters_ class +- Add **merge** method into _DataAdapters_ class +- Add **counter** method into _DataAdapters_ class +- Add **chunks** method into _DataAdapters_ class +- Add **deduplicate** method into _DataAdapters_ class +- Add **iter** method into _DataAdapters_ class +- Add **getitem** method into _DataAdapters_ class +- Add **getitem** method into _DataAdapters_ class +- Add **repr** and **str** function into _DataPrinters_ class +- Add **average** method into _DataPrinters_ class +- Add **most_common** method into _DataPrinters_ class +- Add **percentage** method into _DataPrinters_ class +- Add **len** method into _DataPrinters_ class +- Add **DataAdapters** and **DataPrinters** classes into _Report_ class +- Add **DataObjectError** exception class + ## 1.6.0 Jul 14, 2023 diff --git a/docs/requirements.txt b/docs/requirements.txt index 3b7dffb..0f96a07 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,4 @@ ldap3 -pymssql mysql-connector-python psycopg2-binary tablib diff --git a/docs/source/conf.py b/docs/source/conf.py index 2177fab..2a350d1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,28 +15,27 @@ # sys.path.insert(0, os.path.abspath('.')) import os import sys -sys.path.insert(0, os.path.abspath('../..')) -import __info__ +sys.path.insert(0, os.path.abspath("../..")) # -- Project information ----------------------------------------------------- -project = 'pyreports' -copyright = '2021, Matteo Guadrini' -author = __info__.__author__ +project = "pyreports" +copyright = "2024, Matteo Guadrini" +author = "Matteo Guadrini" # The full version, including alpha/beta/rc tags -release = __info__.__version__ +release = "1.7.0" # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -53,11 +52,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] -html_theme_options = { - 'logo_only': False -} +html_theme_options = {"logo_only": False} html_logo = "_static/pyreports.svg" -master_doc = 'index' +master_doc = "index" diff --git a/docs/source/datatools.rst b/docs/source/datatools.rst index 5294a88..ad78435 100644 --- a/docs/source/datatools.rst +++ b/docs/source/datatools.rst @@ -8,6 +8,107 @@ In this section we will see all these functions contained in the **datatools** m .. toctree:: +DataObject +---------- + +**DataObject** class represents a pure *Dataset*. + +.. autoclass:: pyreports.DataObject + :members: + +.. code-block:: python + + import pyreports, tablib + + data = pyreports.DataObject(tablib.Dataset(*[("Arthur", "Dent", 42)])) + assert isinstance(data.data, tablib.Dataset) == True + + + +DataAdapters +------------ + +**DataAdapters** class is an object that contains methods that modifying *Dataset*. + +.. code-block:: python + + import pyreports, tablib + + data = pyreports.DataAdapters(tablib.Dataset(*[("Arthur", "Dent", 42)])) + assert isinstance(data.data, tablib.Dataset) == True + + + # Aggregate + planets = tablib.Dataset(*[("Heart",)]) + data.aggregate(planets) + + # Merge + others = tablib.Dataset(*[("Betelgeuse", "Ford", "Prefect", 42)]) + data.merge(others) + + # Counter + data = pyreports.DataAdapters(Dataset(*[("Heart", "Arthur", "Dent", 42)])) + data.merge(self.data) + counter = data.counter() + assert counter["Arthur"] == 2 + + # Chunks + data.data.headers = ["planet", "name", "surname", "age"] + assert list(data.chunks(4))[0][0] == ("Heart", "Arthur", "Dent", 42) + + # Deduplicate + data.deduplicate() + assert len(data.data) == 2 + + # Get items + assert data[1] == ("Betelgeuse", "Ford", "Prefect", 42) + + # Iter items + for item in data: + print(item) + + +.. autoclass:: pyreports.DataAdapters + :members: + +DataPrinters +------------ + +**DataPrinters** class is an object that contains methods that printing *Dataset*'s information. + +.. code-block:: python + + import pyreports, tablib + + data = pyreports.DataPrinters(tablib.Dataset(*[("Arthur", "Dent", 42), ("Ford", "Prefect", 42)], headers=["name", "surname", "age"])) + assert isinstance(data.data, tablib.Dataset) == True + + # Print + data.print() + + # Average + assert data.average(2) == 42 + assert data.average("age") == 42 + + # Most common + data.data.append(("Ford", "Prefect", 42)) + assert data.most_common(0) == "Ford" + assert data.most_common("name") == "Ford" + + # Percentage + assert data.percentage("Ford") == 66.66666666666666 + + # Representation + assert repr(data) == "" + + # String + assert str(data) == 'name |surname|age\n------|-------|---\nArthur|Dent |42 \nFord |Prefect|42 \nFord |Prefect|42 ' + + # Length + assert len(data) == 3 + +.. autoclass:: pyreports.DataPrinters + :members: Average @@ -137,4 +238,20 @@ The **chunks** function divides a *Dataset* into pieces from *N* (``int``). This print(list(new_data)) # [[('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000)], [('Tricia', 'McMillian', 55000), ('Zaphod', 'Beeblebrox', 65000)]] .. note:: - If the division does not result zero, the last tuple of elements will be a smaller number. \ No newline at end of file + If the division does not result zero, the last tuple of elements will be a smaller number. + +Deduplicate +----------- + +The **deduplicate** function remove duplicated rows into *Dataset* objects. + +.. code-block:: python + + import pyreports + + # Build a datasets + employee1 = tablib.Dataset([('Arthur', 'Dent', 55000), ('Ford', 'Prefect', 65000), ('Ford', 'Prefect', 65000)], headers=['name', 'surname', 'salary']) + + # Remove duplicated rows (removed the last ('Ford', 'Prefect', 65000)) + pyreports.deduplicate(employee1) + print(len(employee1)) # 2 \ No newline at end of file diff --git a/docs/source/example.rst b/docs/source/example.rst index 345f3ef..1e9da63 100644 --- a/docs/source/example.rst +++ b/docs/source/example.rst @@ -99,7 +99,7 @@ In this example, we will take two different inputs, and combine them to export a # Config Unix application file: this is a FileManager object config_file = pyreports.manager('yaml', '/home/myapp.yml') # Console admin: this is a DatabaseManager object - mydb = pyreports.manager('mssql', server='mssql1.local', database='admins', user='sa', password='sa0000') + mydb = pyreports.manager('mysql', server='mysql1.local', database='admins', user='sa', password='sa0000') # Get data admin_app = config_file.read() # return Dataset object: three column (name, shell, login) mydb.execute('SELECT * FROM console_admins') diff --git a/docs/source/managers.rst b/docs/source/managers.rst index a2667f0..4e98fa4 100644 --- a/docs/source/managers.rst +++ b/docs/source/managers.rst @@ -16,7 +16,6 @@ Each type of manager is managed by micro types; Below is the complete list: #. Database #. sqlite (SQLite) - #. mssql (Microsoft SQL) #. mysql (MySQL or MariaDB) #. postgresql (PostgreSQL or EnterpriseDB) #. File @@ -41,7 +40,6 @@ Each type of manager is managed by micro types; Below is the complete list: # DatabaseManager object sqlite_db = pyreports.manager('sqlite', database='/tmp/mydb.db') - mssql_db = pyreports.manager('mssql', server='mssql1.local', database='test', user='dba', password='dba0000') mysql_db = pyreports.manager('mysql', host='mysql1.local', database='test', user='dba', password='dba0000') postgresql_db = pyreports.manager('postgresql', host='postgresql1.local', database='test', user='dba', password='dba0000') diff --git a/index.html b/index.html index 9641e58..33eed5e 100644 --- a/index.html +++ b/index.html @@ -245,24 +245,24 @@

Shell CLI


 $ cat car.yml
 reports:
-- report:
-  title: 'Red ford machine'
-  input:
-    manager: 'mysql'
-    source:
-    # Connection parameters of my mysql database
-      host: 'mysql1.local'
-      database: 'cars'
-      user: 'admin'
-      password: 'dba0000'
-    params:
-      query: 'SELECT * FROM cars WHERE brand = %s AND color = %s'
-      params: ['ford', 'red']
-  # Filter km
-  filters: [40000, 45000]
-  output:
-    manager: 'csv'
-    filename: '/tmp/car_csv.csv'
+  - report:
+    title: 'Red ford machine'
+    input:
+      manager: 'mysql'
+      source:
+      # Connection parameters of my mysql database
+        host: 'mysql1.local'
+        database: 'cars'
+        user: 'admin'
+        password: 'dba0000'
+      params:
+        query: 'SELECT * FROM cars WHERE brand = %s AND color = %s'
+        params: ['ford', 'red']
+    # Filter km
+    filters: [40000, 45000]
+    output:
+      manager: 'csv'
+      filename: '/tmp/car_csv.csv'
 
 $ report car.yaml
diff --git a/pyproject.toml b/pyproject.toml index 81338cd..dc0e17c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "pyreports" -version = "1.6.0" +version = "1.7.0" readme = "README.md" -license = {text = "GNU General Public License v3.0"} +license = { text = "GNU General Public License v3.0" } keywords = ['pyreports', 'reports', 'report', 'csv', 'yaml', 'export', - 'excel', 'database', 'ldap', 'dataset', 'file', 'executor', 'book'] + 'excel', 'database', 'ldap', 'dataset', 'file', 'executor', 'book'] authors = [{ name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }] maintainers = [ { name = "Matteo Guadrini", email = "matteo.guadrini@hotmail.it" }, @@ -20,8 +20,8 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ] -dependencies = ['ldap3', 'pymssql', 'mysql-connector-python', - 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] +dependencies = ['ldap3', 'mysql-connector-python', + 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'] [project.scripts] reports = "pyreports.cli:main" @@ -29,5 +29,5 @@ reports = "pyreports.cli:main" [project.urls] homepage = "https://github.com/MatteoGuadrini/pyreports'" documentation = "https://pyreports.readthedocs.io/en/latest/" -repository = "https://github.com/MatteoGuadrini/pyreports/pyreports.git" -changelog = "https://github.com/MatteoGuadrini/pyreports/blob/master/CHANGES.md" \ No newline at end of file +repository = "https://github.com/MatteoGuadrini/pyreports.git" +changelog = "https://github.com/MatteoGuadrini/pyreports/blob/master/CHANGES.md" diff --git a/pyreports/__init__.py b/pyreports/__init__.py index fb30e21..866ea5d 100644 --- a/pyreports/__init__.py +++ b/pyreports/__init__.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # __init__ -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,10 +20,24 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Build complex pyreports from/to various formats.""" +"""Build complex reports from/to various formats.""" -from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER -from .core import Executor, Report, ReportBook -from .exception import ReportDataError, ReportManagerError -from .datatools import average, most_common, percentage, counter, aggregate, chunks, merge, deduplicate -from .cli import make_manager, get_data, load_config, validate_config +__version__ = "1.7.0" + +from .io import manager, READABLE_MANAGER, WRITABLE_MANAGER # noqa: F401 +from .core import Executor, Report, ReportBook # noqa: F401 +from .exception import ReportDataError, ReportManagerError, DataObjectError # noqa: F401 +from .datatools import ( + average, # noqa: F401 + most_common, # noqa: F401 + percentage, # noqa: F401 + counter, # noqa: F401 + aggregate, # noqa: F401 + chunks, # noqa: F401 + merge, # noqa: F401 + deduplicate, # noqa: F401 + DataObject, # noqa: F401 + DataAdapters, # noqa: F401 + DataPrinters, # noqa: F401 +) +from .cli import make_manager, get_data, load_config, validate_config # noqa: F401 diff --git a/pyreports/cli.py b/pyreports/cli.py index 781a70f..808b5bb 100644 --- a/pyreports/cli.py +++ b/pyreports/cli.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # cli -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,37 +28,43 @@ import yaml import argparse import pyreports - -# endregion - -# region globals -__version__ = '1.5.1' +from pyreports import __version__ # endregion + # region functions def get_args(): """Get command-line arguments""" parser = argparse.ArgumentParser( - description='pyreports command line interface (CLI)', + description="pyreports command line interface (CLI)", formatter_class=argparse.ArgumentDefaultsHelpFormatter, - epilog='Full docs here: https://pyreports.readthedocs.io/en/latest/dev/cli.html' + epilog="Full docs here: https://pyreports.readthedocs.io/en/latest/dev/cli.html", ) - parser.add_argument('config', - metavar='CONFIG_FILE', - default=sys.stdin, - type=argparse.FileType('rt', encoding="utf-8"), - help='YAML configuration file') - parser.add_argument('-e', '--exclude', - help='Excluded title report list', - nargs=argparse.ZERO_OR_MORE, - default=[], - metavar='TITLE') - parser.add_argument('-v', '--verbose', help='Enable verbose mode', action='store_true') - parser.add_argument('-V', '--version', help='Print version', action='version', version=__version__) + parser.add_argument( + "config", + metavar="CONFIG_FILE", + default=sys.stdin, + type=argparse.FileType("rt", encoding="utf-8"), + help="YAML configuration file", + ) + parser.add_argument( + "-e", + "--exclude", + help="Excluded title report list", + nargs=argparse.ZERO_OR_MORE, + default=[], + metavar="TITLE", + ) + parser.add_argument( + "-v", "--verbose", help="Enable verbose mode", action="store_true" + ) + parser.add_argument( + "-V", "--version", help="Print version", action="version", version=__version__ + ) args = parser.parse_args() filename = args.config.name @@ -67,7 +73,7 @@ def get_args(): try: args.config = load_config(args.config) except yaml.YAMLError as err: - parser.error(f'file {filename} is not a valid YAML file: \n{err}') + parser.error(f"file {filename} is not a valid YAML file: \n{err}") # Validate config file try: @@ -75,7 +81,7 @@ def get_args(): except yaml.YAMLError as err: parser.error(str(err)) - print_verbose(f'parsed YAML file {filename}', verbose=args.verbose) + print_verbose(f"parsed YAML file {filename}", verbose=args.verbose) return args @@ -97,10 +103,10 @@ def validate_config(config): :return: None """ try: - reports = config['reports'] + reports = config["reports"] if reports is None or not isinstance(reports, list): raise yaml.YAMLError('"reports" section have not "report" list sections') - datas = all([bool(report.get('report').get('input')) for report in reports]) + datas = all([bool(report.get("report").get("input")) for report in reports]) if not datas: raise yaml.YAMLError( 'one of "report" section does not have "input" section' @@ -108,7 +114,9 @@ def validate_config(config): except KeyError as err: raise yaml.YAMLError(f'there is no "{err}" section') except AttributeError: - raise yaml.YAMLError('correctly indents the "report" section or "report" section not exists') + raise yaml.YAMLError( + 'correctly indents the "report" section or "report" section not exists' + ) def make_manager(input_config): @@ -118,12 +126,16 @@ def make_manager(input_config): :return: manager """ - type_ = input_config.get('manager') + type_ = input_config.get("manager") if type_ in pyreports.io.FILETYPE: - manager = pyreports.manager(input_config.get('manager'), input_config.get('filename', ())) + manager = pyreports.manager( + input_config.get("manager"), input_config.get("filename", ()) + ) else: - manager = pyreports.manager(input_config.get('manager'), **input_config.get('source', {})) + manager = pyreports.manager( + input_config.get("manager"), **input_config.get("source", {}) + ) return manager @@ -139,7 +151,7 @@ def get_data(manager, params=None): params = () data = None # FileManager - if manager.type == 'file': + if manager.type == "file": if params and isinstance(params, (list, tuple)): data = manager.read(*params) elif params and isinstance(params, dict): @@ -147,7 +159,7 @@ def get_data(manager, params=None): else: data = manager.read() # DatabaseManager - elif manager.type == 'sql': + elif manager.type == "sql": if params and isinstance(params, (list, tuple)): manager.execute(*params) data = manager.fetchall() @@ -155,13 +167,13 @@ def get_data(manager, params=None): manager.execute(**params) data = manager.fetchall() # LdapManager - elif manager.type == 'ldap': + elif manager.type == "ldap": if params and isinstance(params, (list, tuple)): data = manager.query(*params) elif params and isinstance(params, dict): data = manager.query(**params) # NosqlManager - elif manager.type == 'nosql': + elif manager.type == "nosql": if params and isinstance(params, (list, tuple)): data = manager.find(*params) elif params and isinstance(params, dict): @@ -178,7 +190,7 @@ def print_verbose(*messages, verbose=False): :return: None """ if verbose: - print('info:', *messages) + print("info:", *messages) def main(): @@ -188,71 +200,84 @@ def main(): args = get_args() # Take reports config = args.config - reports = config.get('reports', ()) + reports = config.get("reports", ()) - print_verbose(f'found {len(config.get("reports", ()))} report(s)', - verbose=args.verbose) + print_verbose( + f'found {len(config.get("reports", ()))} report(s)', verbose=args.verbose + ) # Build the data and report for report in reports: # Check if report isn't in excluded list - if args.exclude and report.get('report').get('title') in args.exclude: - print_verbose(f'exclude report "{report.get("report").get("title")}"', - verbose=args.verbose) + if args.exclude and report.get("report").get("title") in args.exclude: + print_verbose( + f'exclude report "{report.get("report").get("title")}"', + verbose=args.verbose, + ) continue # Make a manager object - input_ = report.get('report').get('input') - print_verbose(f'make an input manager of type {input_.get("manager")}', - verbose=args.verbose) + input_ = report.get("report").get("input") + print_verbose( + f'make an input manager of type {input_.get("manager")}', + verbose=args.verbose, + ) manager = make_manager(input_) # Get data - print_verbose(f'get data from manager {manager}', verbose=args.verbose) + print_verbose(f"get data from manager {manager}", verbose=args.verbose) try: # Make a report object - data = get_data(manager, input_.get('params')) - if 'map' in report.get('report'): - exec(report.get('report').get('map')) - map_func = globals().get('map_func') + data = get_data(manager, input_.get("params")) + if "map" in report.get("report"): + exec(report.get("report").get("map")) + map_func = globals().get("map_func") report_ = pyreports.Report( input_data=data, - title=report.get('report').get('title'), - filters=report.get('report').get('filters'), + title=report.get("report").get("title"), + filters=report.get("report").get("filters"), map_func=map_func, - negation=report.get('report').get('negation', False), - column=report.get('report').get('column'), - count=report.get('report').get('count', False), - output=make_manager(report.get('report').get('output')) if 'output' in report.get('report') else None + negation=report.get("report").get("negation", False), + column=report.get("report").get("column"), + count=report.get("report").get("count", False), + output=make_manager(report.get("report").get("output")) + if "output" in report.get("report") + else None, ) print_verbose(f'created report "{report_.title}"', verbose=args.verbose) except Exception as err: pyreports.Report(tablib.Dataset()) - exit(f'error: {err}') + exit(f"error: {err}") # Check output if report_.output: # Check if export or send report - if report.get('report').get('mail'): - print_verbose(f'send report to {report.get("report").get("mail").get("to")}', verbose=args.verbose) - mail_settings = report.get('report').get('mail') + if report.get("report").get("mail"): + print_verbose( + f'send report to {report.get("report").get("mail").get("to")}', + verbose=args.verbose, + ) + mail_settings = report.get("report").get("mail") report_.send( - server=mail_settings.get('server'), - _from=mail_settings.get('from'), - to=mail_settings.get('to'), - cc=mail_settings.get('cc'), - bcc=mail_settings.get('bcc'), - subject=mail_settings.get('subject'), - body=mail_settings.get('body'), - auth=tuple(mail_settings.get('auth')) if 'auth' in mail_settings else None, - _ssl=bool(mail_settings.get('ssl')), - headers=mail_settings.get('headers') + server=mail_settings.get("server"), + _from=mail_settings.get("from"), + to=mail_settings.get("to"), + cc=mail_settings.get("cc"), + bcc=mail_settings.get("bcc"), + subject=mail_settings.get("subject"), + body=mail_settings.get("body"), + auth=tuple(mail_settings.get("auth")) + if "auth" in mail_settings + else None, + _ssl=bool(mail_settings.get("ssl")), + headers=mail_settings.get("headers"), ) else: - print_verbose(f'export report to {report_.output}', - verbose=args.verbose) + print_verbose( + f"export report to {report_.output}", verbose=args.verbose + ) report_.export() else: # Print report in stdout - print_verbose('print report to stdout', verbose=args.verbose) - title = report.get('report').get('title') + print_verbose("print report to stdout", verbose=args.verbose) + title = report.get("report").get("title") report_.exec() print(f"{title}\n{'=' * len(title)}\n") print(report_) @@ -261,7 +286,7 @@ def main(): # endregion # region main -if __name__ == '__main__': +if __name__ == "__main__": main() # endregion diff --git a/pyreports/core.py b/pyreports/core.py index 70f9f03..15f9c15 100644 --- a/pyreports/core.py +++ b/pyreports/core.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # core -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,6 +31,7 @@ from email.mime.multipart import MIMEMultipart from email import encoders from email.mime.base import MIMEBase +from .datatools import DataAdapters, DataPrinters from .io import Manager, WRITABLE_MANAGER from .exception import ReportManagerError, ReportDataError, ExecutorError @@ -39,6 +40,7 @@ # region Classes + class Executor: """Executor receives, processes, transforms and writes data""" @@ -250,18 +252,20 @@ def clone(self): return Executor(self.origin, header=self.origin.headers) -class Report: +class Report(DataAdapters, DataPrinters): """Report represents the workflow for generating a report""" - def __init__(self, - input_data: tablib.Dataset, - title=None, - filters=None, - map_func=None, - negation=False, - column=None, - count=False, - output: Manager = None): + def __init__( + self, + input_data: tablib.Dataset, + title=None, + filters=None, + map_func=None, + negation=False, + column=None, + count=False, + output: Manager = None, + ): """Create Report object :param input_data: Dataset object @@ -274,10 +278,7 @@ def __init__(self, :param output: Manager object """ # Discard all objects that are not Datasets - if isinstance(input_data, tablib.Dataset): - self.data = input_data - else: - raise ReportDataError('Only Dataset object is allowed for input') + DataAdapters.__init__(self, input_data=input_data) # Set other arguments self.title = title self.filter = filters @@ -288,10 +289,12 @@ def __init__(self, if isinstance(output, Manager) or output is None: if output: if output.__class__.__name__ not in WRITABLE_MANAGER: - raise ReportManagerError(f'Only {WRITABLE_MANAGER} object is allowed for output') + raise ReportManagerError( + f"Only {WRITABLE_MANAGER} object is allowed for output" + ) self.output = output else: - raise ReportManagerError('Only Manager object is allowed for output') + raise ReportManagerError("Only Manager object is allowed for output") # Data for report self.report = None @@ -346,7 +349,7 @@ def _print_data(self): :return: data and count """ if self.count: - out = f'{self.report}\nrows: {int(self.count)}' + out = f"{self.report}\nrows: {int(self.count)}" return out else: return self.report @@ -380,22 +383,21 @@ def export(self): # Process data before export self.exec() if self.output is not None: - if self.output.type == 'file': + if self.output.type == "file": self.output.write(self.report) - elif self.output.type == 'sql': + elif self.output.type == "sql": if not self.report.headers: raise ReportDataError("Dataset object doesn't have a header") - table_name = self.title.replace(' ', '_').lower() - fields = ','.join([ - f'{field} VARCHAR(255)' - for field in self.report.headers - ]) + table_name = self.title.replace(" ", "_").lower() + fields = ",".join( + [f"{field} VARCHAR(255)" for field in self.report.headers] + ) # Create table with header self.output.execute( f"CREATE TABLE IF NOT EXISTS {table_name} ({fields});" ) # Insert data into table - table_header = ','.join(field for field in self.report.headers) + table_header = ",".join(field for field in self.report.headers) for row in self.report: row_table = [f"'{element}'" for element in row] self.output.execute( @@ -407,7 +409,19 @@ def export(self): else: print(self) - def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth=None, _ssl=True, headers=None): + def send( + self, + server, + _from, + to, + cc=None, + bcc=None, + subject=None, + body="", + auth=None, + _ssl=True, + headers=None, + ): """Send saved report to email :param server: server SMTP @@ -423,7 +437,9 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth :return: None """ if not self.output: - raise ReportDataError('if you want send a mail with a report in attachment, must be specified output') + raise ReportDataError( + "if you want send a mail with a report in attachment, must be specified output" + ) # Prepare mail header message = MIMEMultipart("alternative") @@ -441,7 +457,7 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth elif isinstance(headers, (tuple, list)): message.add_header(*headers) else: - raise ValueError('headers must be tuple or List[tuple]') + raise ValueError("headers must be tuple or List[tuple]") # Prepare body part = MIMEText(body, "html") @@ -450,18 +466,22 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth # Prepare attachment self.export() attach_file_name = self.output.data.file - attach_file = open(attach_file_name, 'rb') - payload = MIMEBase('application', 'octate-stream') + attach_file = open(attach_file_name, "rb") + payload = MIMEBase("application", "octate-stream") payload.set_payload(attach_file.read()) encoders.encode_base64(payload) - payload.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attach_file_name)) + payload.add_header( + "Content-Disposition", + "attachment", + filename=os.path.basename(attach_file_name), + ) message.attach(payload) # Prepare SMTP connection if _ssl: port = smtplib.SMTP_SSL_PORT protocol = smtplib.SMTP_SSL - kwargs = {'context': ssl.create_default_context()} + kwargs = {"context": ssl.create_default_context()} else: port = smtplib.SMTP_PORT protocol = smtplib.SMTP @@ -469,7 +489,11 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth with protocol(server, port, **kwargs) as srv: if auth: srv.login(*auth) - srv.sendmail(_from, [receiver for receiver in (to, cc, bcc) if receiver], message.as_string()) + srv.sendmail( + _from, + [receiver for receiver in (to, cc, bcc) if receiver], + message.as_string(), + ) class ReportBook: @@ -495,7 +519,7 @@ def __add__(self, other): :return: ReportBook """ if not isinstance(other, ReportBook): - raise ReportDataError('you can only add ReportBook object') + raise ReportDataError("you can only add ReportBook object") self.reports.extend(other.reports) return self @@ -518,9 +542,8 @@ def __str__(self): :return: string """ - output = f'ReportBook {self.title if self.title else None}\n' - output += '\n'.join('\t' + str(report.title) - for report in self.reports) + output = f"ReportBook {self.title if self.title else None}\n" + output += "\n".join("\t" + str(report.title) for report in self.reports) return output def __repr__(self): @@ -561,7 +584,7 @@ def add(self, report: Report): :return: None """ if not isinstance(report, Report): - raise ReportDataError('you can only add Report object') + raise ReportDataError("you can only add Report object") self.reports.append(report) def remove(self, index: int = None): @@ -588,13 +611,25 @@ def export(self, output=None): # Prepare book for export book = tablib.Databook(tuple([report.report for report in self])) # Save Excel WorkBook - with open(output, 'wb') as f: - f.write(book.export('xlsx')) + with open(output, "wb") as f: + f.write(book.export("xlsx")) else: for report in self: report.export() - def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth=None, _ssl=True, headers=None): + def send( + self, + server, + _from, + to, + cc=None, + bcc=None, + subject=None, + body="", + auth=None, + _ssl=True, + headers=None, + ): """Send saved report to email :param server: server SMTP @@ -610,7 +645,18 @@ def send(self, server, _from, to, cc=None, bcc=None, subject=None, body='', auth :return: None """ for report in self: - report.send(server, _from, to, cc=cc, bcc=bcc, subject=subject, body=body, auth=auth, - _ssl=_ssl, headers=headers) + report.send( + server, + _from, + to, + cc=cc, + bcc=bcc, + subject=subject, + body=body, + auth=auth, + _ssl=_ssl, + headers=headers, + ) + # endregion diff --git a/pyreports/datatools.py b/pyreports/datatools.py index 48b4a16..b15728d 100644 --- a/pyreports/datatools.py +++ b/pyreports/datatools.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # datatools -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,13 +23,153 @@ """Contains all functions for data processing.""" # region Imports -from .exception import ReportDataError +from .exception import DataObjectError from collections import Counter from tablib import Dataset, InvalidDimensions # endregion + +# region Classes +class DataObject: + """Data object class""" + + def __init__(self, input_data: Dataset): + # Discard all objects that are not Datasets + if isinstance(input_data, Dataset): + self._data = input_data + else: + raise DataObjectError("only Dataset object is allowed for input") + + @property + def data(self): + return self._data + + @data.setter + def data(self, dataset): + self._data = dataset + + +class DataAdapters(DataObject): + """Data adapters class""" + + def aggregate(self, *columns, fill_value=None): + """Aggregate in the current Dataset other columns + + :param columns: columns added + :param fill_value: fill value for empty field if "fill_empty" argument is specified + :return: None + """ + if not self.data: + raise DataObjectError("dataset is empty") + local_columns = [self.data.get_col(col) for col in range(self.data.width)] + local_columns.extend(columns) + self.data = aggregate(*local_columns, fill_empty=True, fill_value=fill_value) + + def merge(self, *datasets): + """Merge in the current Dataset other Dataset objects + + :param datasets: datasets that will merge + :return: None + """ + datasets = list(datasets) + datasets.append(self.data) + # Check if all Datasets are not empties + if not all([data for data in datasets]): + raise DataObjectError("one or more Datasets are empties") + self.data = merge(*datasets) + + def counter(self): + """Count value into the rows + + :return: Counter + """ + return Counter((item for row in self.data for item in row)) + + def chunks(self, length): + """ + Yield successive n-sized chunks from Dataset + + :param length: n-sized chunks + :return: generator + """ + for idx in range(0, len(self.data), length): + yield self.data[idx : idx + length] + + def deduplicate(self): + """Remove duplicated rows + + :return: None + """ + self.data = Dataset(*list(dict.fromkeys(iter(self.data)))) + + def __iter__(self): + return (row for row in self.data) + + def __getitem__(self, item): + return self.data[item] + + +class DataPrinters(DataObject): + """Data printers class""" + + def print(self): + """Print data + + :return: None + """ + print(self) + + def average(self, column): + """Average of list of integers or floats + + :param column: column name or index + :return: float + """ + return average(self.data, column) + + def most_common(self, column): + """The most common element in a column + + :param column: column name or index + :return: Any + """ + return most_common(self.data, column) + + def percentage(self, filter_): + """Calculating the percentage according to filter + + :param filter_: equality filter + :return: float + """ + return percentage(self.data, filter_) + + def __repr__(self): + """Representation of DataObject + + :return: string + """ + return f"" + + def __str__(self): + """Pretty representation of DataObject + + :return: string + """ + return str(self.data) + + def __len__(self): + """Measure length of DataSet + + :return: int + """ + return len(self.data) + + +# endregion + + # region Functions def _select_column(data, column): """Select Dataset column @@ -40,7 +180,7 @@ def _select_column(data, column): """ # Check if dataset have a column if not data.headers: - raise ReportDataError('dataset object must have the headers') + raise DataObjectError("dataset object must have the headers") # Select column if isinstance(column, int): return data.get_col(column) @@ -60,7 +200,7 @@ def average(data, column): data = _select_column(data, column) # Check if all item is integer or float if not all(isinstance(item, (int, float)) for item in data): - raise ReportDataError('the column contains only int or float') + raise DataObjectError("the column contains only int or float") # Calculate average return float(sum(data) / len(data)) @@ -83,14 +223,11 @@ def percentage(data, filter_): Calculating the percentage according to filter :param data: Dataset object - :param filter_: filter + :param filter_: equality filter :return: float """ # Filtering data... - data_filtered = [item - for row in data - for item in row - if filter_ == item] + data_filtered = [item for row in data for item in row if filter_ == item] quotient = len(data_filtered) / len(data) return quotient * 100 @@ -128,14 +265,14 @@ def aggregate(*columns, fill_empty: bool = False, fill_value=None): list_.append(fill_value() if callable(fill_value) else fill_value) else: if max_len != len(list_): - raise InvalidDimensions('the columns are not the same length') + raise InvalidDimensions("the columns are not the same length") max_len = len(list_) # Aggregate columns for column in columns: new_data.append_col(column) return new_data else: - raise ReportDataError('you can aggregate two or more columns') + raise DataObjectError("you can aggregate two or more columns") def merge(*datasets): @@ -151,11 +288,11 @@ def merge(*datasets): length_row = len(datasets[0][0]) for data in datasets: if length_row != len(data[0]): - raise InvalidDimensions('the row are not the same length') + raise InvalidDimensions("the row are not the same length") new_data.extend(data) return new_data else: - raise ReportDataError('you can merge two or more dataset object') + raise DataObjectError("you can merge two or more dataset object") def chunks(data, length): @@ -166,8 +303,8 @@ def chunks(data, length): :param length: n-sized chunks :return: generator """ - for i in range(0, len(data), length): - yield data[i:i + length] + for idx in range(0, len(data), length): + yield data[idx : idx + length] def deduplicate(data): @@ -176,6 +313,7 @@ def deduplicate(data): :param data: Dataset object :return: Dataset """ - return Dataset(*list(dict.fromkeys(data))) + return Dataset(*list(dict.fromkeys(iter(data)))) + # endregion diff --git a/pyreports/exception.py b/pyreports/exception.py index ff6d96b..2016796 100644 --- a/pyreports/exception.py +++ b/pyreports/exception.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # exception.py -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,7 +28,11 @@ class ExecutorError(Exception): # ReportException hierarchy -class ReportException(Exception): +class DataObjectError(Exception): + pass + + +class ReportException(DataObjectError): pass diff --git a/pyreports/io.py b/pyreports/io.py index ae3ca04..645b553 100644 --- a/pyreports/io.py +++ b/pyreports/io.py @@ -5,7 +5,7 @@ # created by: matteo.guadrini # io -- pyreports # -# Copyright (C) 2023 Matteo Guadrini +# Copyright (C) 2024 Matteo Guadrini # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,6 @@ import sqlite3 from typing import Union, List import nosqlapi -import pymssql import mysql.connector as mdb import psycopg2 import tablib @@ -114,6 +113,7 @@ def __iter__(self): class Manager(ABC): """Manager base class""" + pass @@ -128,8 +128,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write('\n'.join(str(line) for row in data for line in row)) + with open(self.file, mode="w") as file: + file.write("\n".join(str(line) for row in data for line in row)) def read(self, **kwargs): """Read with format @@ -139,7 +139,7 @@ def read(self, **kwargs): data = tablib.Dataset(**kwargs) with open(self.file) as file: for line in file: - data.append([line.strip('\n')]) + data.append([line.strip("\n")]) return data @@ -154,10 +154,10 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(*data) - with open(self.file, mode='w') as file: - file.write('\n'.join([' '.join(row).strip('\n') for row in data])) + with open(self.file, mode="w") as file: + file.write("\n".join([" ".join(row).strip("\n") for row in data])) - def read(self, pattern=r'(.*\n|.*$)', **kwargs): + def read(self, pattern=r"(.*\n|.*$)", **kwargs): """Read with format :param pattern: regular expression pattern @@ -186,8 +186,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('csv')) + with open(self.file, mode="w") as file: + file.write(data.export("csv")) def read(self, **kwargs): """Read csv format @@ -209,8 +209,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('json')) + with open(self.file, mode="w") as file: + file.write(data.export("json")) def read(self, **kwargs): """Read json format @@ -232,8 +232,8 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='w') as file: - file.write(data.export('yaml')) + with open(self.file, mode="w") as file: + file.write(data.export("yaml")) def read(self, **kwargs): """Read yaml format @@ -255,15 +255,15 @@ def write(self, data): """ if not isinstance(data, tablib.Dataset): data = tablib.Dataset(data) - with open(self.file, mode='wb') as file: - file.write(data.export('xlsx')) + with open(self.file, mode="wb") as file: + file.write(data.export("xlsx")) def read(self, **kwargs): """Read xlsx format :return: Dataset object """ - with open(self.file, 'rb') as file: + with open(self.file, "rb") as file: return tablib.import_set(file, **kwargs) @@ -279,18 +279,6 @@ def close(self): self.cursor.close() -class MSSQLConnection(Connection): - """Connection microsoft sql class""" - - def connect(self): - self.connection = pymssql.connect(*self.args, **self.kwargs) - self.cursor = self.connection.cursor() - - def close(self): - self.connection.close() - self.cursor.close() - - class MySQLConnection(Connection): """Connection mysql class""" @@ -323,7 +311,7 @@ def __init__(self, connection: Connection): :param connection: Connection based object """ - self.type = 'sql' + self.type = "sql" self.connector = connection # Connect database self.connector.connect() @@ -410,7 +398,9 @@ def fetchone(self) -> tablib.Dataset: :return: Dataset object """ header = [field[0] for field in self.description] - self.data = tablib.Dataset(list(self.connector.cursor.fetchone()), headers=header) + self.data = tablib.Dataset( + list(self.connector.cursor.fetchone()), headers=header + ) return self.data def fetchmany(self, size=1) -> tablib.Dataset: @@ -453,11 +443,11 @@ class NoSQLManager(Manager, APIManager): def __init__(self, connection: APIConnection, *args, **kwargs): APIManager.__init__(self, connection, *args, **kwargs) - self.type = 'nosql' + self.type = "nosql" @staticmethod def _response_to_dataset( - obj: Union[List[tuple], List[list], dict, nosqlapi.Response] + obj: Union[List[tuple], List[list], dict, nosqlapi.Response], ) -> tablib.Dataset: """Transform receive data into Dataset object""" data = tablib.Dataset() @@ -469,7 +459,9 @@ def _response_to_dataset( if isinstance(obj.data, (list, tuple)): data = tablib.Dataset([row for row in obj.data]) elif isinstance(obj.data, dict): - data = tablib.Dataset([obj.data[key] for key in obj], headers=list(obj.data.keys())) + data = tablib.Dataset( + [obj.data[key] for key in obj], headers=list(obj.data.keys()) + ) else: data.append(obj) @@ -492,7 +484,7 @@ def __init__(self, file: File): :param file: file object """ - self.type = 'file' + self.type = "file" self.data = file def __repr__(self): @@ -539,18 +531,24 @@ def __init__(self, server, username, password, ssl=False, tls=True): :param ssl: disable or enable SSL. Default is False. :param tls: disable or enable TLS. Default is True. """ - self.type = 'ldap' + self.type = "ldap" # Check ssl connection port = 636 if ssl else 389 - self.connector = ldap3.Server(server, - get_info=ldap3.ALL, - port=port, - use_ssl=ssl) + self.connector = ldap3.Server( + server, get_info=ldap3.ALL, port=port, use_ssl=ssl + ) # Check tls connection - self.auto_bind = ldap3.AUTO_BIND_TLS_BEFORE_BIND if tls else ldap3.AUTO_BIND_NONE + self.auto_bind = ( + ldap3.AUTO_BIND_TLS_BEFORE_BIND if tls else ldap3.AUTO_BIND_NONE + ) # Create a bind connection with user and password - self.bind = ldap3.Connection(self.connector, user=f'{username}', password=f'{password}', - auto_bind=self.auto_bind, raise_exceptions=True) + self.bind = ldap3.Connection( + self.connector, + user=f"{username}", + password=f"{password}", + auto_bind=self.auto_bind, + raise_exceptions=True, + ) self.bind.bind() def __repr__(self): @@ -572,8 +570,13 @@ def rebind(self, username, password): """ # Disconnect LDAP server self.bind.unbind() - self.bind = ldap3.Connection(self.connector, user=f'{username}', password=f'{password}', - auto_bind=self.auto_bind, raise_exceptions=True) + self.bind = ldap3.Connection( + self.connector, + user=f"{username}", + password=f"{password}", + auto_bind=self.auto_bind, + raise_exceptions=True, + ) self.bind.bind() def unbind(self): @@ -591,16 +594,20 @@ def query(self, base_search, search_filter, attributes) -> tablib.Dataset: :param attributes: list of returning LDAP attributes :return: Dataset object """ - if self.bind.search(search_base=base_search, search_filter=f'{search_filter}', attributes=attributes, - search_scope=ldap3.SUBTREE): + if self.bind.search( + search_base=base_search, + search_filter=f"{search_filter}", + attributes=attributes, + search_scope=ldap3.SUBTREE, + ): # Build Dataset data = tablib.Dataset() data.headers = attributes for result in self.bind.response: - if result.get('attributes'): + if result.get("attributes"): row = list() for index, _ in enumerate(attributes): - row.append(result.get('attributes').get(attributes[index])) + row.append(result.get("attributes").get(attributes[index])) data.append(row) # Return object return data @@ -611,24 +618,23 @@ def query(self, base_search, search_filter, attributes) -> tablib.Dataset: # region Variables DBTYPE = { - 'sqlite': SQLiteConnection, - 'mssql': MSSQLConnection, - 'mysql': MySQLConnection, - 'postgresql': PostgreSQLConnection + "sqlite": SQLiteConnection, + "mysql": MySQLConnection, + "postgresql": PostgreSQLConnection, } FILETYPE = { - 'file': TextFile, - 'log': LogFile, - 'csv': CsvFile, - 'json': JsonFile, - 'yaml': YamlFile, - 'xlsx': ExcelFile, + "file": TextFile, + "log": LogFile, + "csv": CsvFile, + "json": JsonFile, + "yaml": YamlFile, + "xlsx": ExcelFile, } -READABLE_MANAGER = ('FileManager', 'DatabaseManager', 'LdapManager', 'NoSQLManager') +READABLE_MANAGER = ("FileManager", "DatabaseManager", "LdapManager", "NoSQLManager") -WRITABLE_MANAGER = ('FileManager', 'DatabaseManager', 'NoSQLManager') +WRITABLE_MANAGER = ("FileManager", "DatabaseManager", "NoSQLManager") # endregion @@ -678,8 +684,10 @@ def create_nosql_manager(connection, *args, **kwargs): :return: NoSQLManager """ # Check if connection class is API compliant with nosqlapi - if not hasattr(connection, 'connect'): - raise nosqlapi.ConnectError('the Connection class is not API compliant. See https://nosqlapi.rtfd.io/') + if not hasattr(connection, "connect"): + raise nosqlapi.ConnectError( + "the Connection class is not API compliant. See https://nosqlapi.rtfd.io/" + ) # Create NoSQLManager object return NoSQLManager(connection=connection, *args, **kwargs) @@ -697,17 +705,18 @@ def manager(datatype, *args, **kwargs): return create_database_manager(datatype, *args, **kwargs) elif datatype in FILETYPE: return create_file_manager(datatype, *args, **kwargs) - elif datatype == 'ldap': + elif datatype == "ldap": return create_ldap_manager(*args, **kwargs) - elif datatype == 'nosql': - connection = kwargs.get('connection') or args[0] + elif datatype == "nosql": + connection = kwargs.get("connection") or args[0] nargs = args[1:] try: - kwargs.pop('connection') + kwargs.pop("connection") except KeyError: pass return create_nosql_manager(connection, *nargs, **kwargs) else: raise ValueError(f"data type {datatype} doesn't exists!") + # endregion diff --git a/setup.py b/setup.py index 264b2fd..9e25aca 100644 --- a/setup.py +++ b/setup.py @@ -1,39 +1,42 @@ from setuptools import setup -__version__ = '1.6.0' -__author__ = 'Matteo Guadrini' -__email__ = 'matteo.guadrini@hotmail.it' -__homepage__ = 'https://github.com/MatteoGuadrini/pyreports' +__version__ = "1.6.0" +__author__ = "Matteo Guadrini" +__email__ = "matteo.guadrini@hotmail.it" +__homepage__ = "https://github.com/MatteoGuadrini/pyreports" with open("README.md") as rme, open("CHANGES.md") as ch: long_description = rme.read() + "\n" + ch.read() setup( - name='pyreports', + name="pyreports", version=__version__, - packages=['pyreports'], + packages=["pyreports"], url=__homepage__, - license='GNU General Public License v3.0', + license="GNU General Public License v3.0", author=__author__, author_email=__email__, - keywords='pyreports reports report csv yaml export excel database ldap dataset file executor book', - maintainer='Matteo Guadrini', - maintainer_email='matteo.guadrini@hotmail.it', - install_requires=['ldap3', 'pymssql', 'mysql-connector-python', - 'psycopg2-binary', 'tablib', 'tablib[all]', 'nosqlapi', 'pyyaml'], - description='pyreports is a python library that allows you to create complex report from various sources', + keywords="pyreports reports report csv yaml export excel database ldap dataset file executor book", + maintainer="Matteo Guadrini", + maintainer_email="matteo.guadrini@hotmail.it", + install_requires=[ + "ldap3", + "mysql-connector-python", + "psycopg2-binary", + "tablib", + "tablib[all]", + "nosqlapi", + "pyyaml", + ], + description="pyreports is a python library that allows you to create complex report from various sources", long_description=long_description, long_description_content_type="text/markdown", classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Operating System :: OS Independent", - ], - entry_points={ - 'console_scripts': [ - 'reports = pyreports.cli:main' - ] - }, - python_requires='>=3.7' + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: OS Independent", + ], + entry_points={"console_scripts": ["reports = pyreports.cli:main"]}, + python_requires=">=3.7", ) diff --git a/tests/test_data.py b/tests/test_data.py index cab12fd..4810e0b 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -7,14 +7,16 @@ class TestDataTools(unittest.TestCase): - data = Dataset(*[('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.headers = ['name', 'surname', 'age'] + data = Dataset( + *[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42), ("Ford", "Prefect", 42)] + ) + data.headers = ["name", "surname", "age"] def test_average(self): - self.assertEqual(int(pyreports.average(self.data, 'age')), 39) + self.assertEqual(int(pyreports.average(self.data, "age")), 39) def test_most_common(self): - self.assertEqual(pyreports.most_common(self.data, 'age'), 42) + self.assertEqual(pyreports.most_common(self.data, "age"), 42) def test_percentage(self): self.assertEqual(int(pyreports.percentage(self.data, 42)), 66) @@ -28,36 +30,195 @@ def test_aggregate(self): names = self.data.get_col(0) surnames = self.data.get_col(1) ages = self.data.get_col(2) - self.assertEqual(pyreports.aggregate(names, surnames, ages)[0], ('Matteo', 'Guadrini', 35)) - ages = ['Name', 'Surname'] - self.assertRaises(tablib.InvalidDimensions, pyreports.aggregate, names, - surnames, ages) - self.assertRaises(pyreports.ReportDataError, pyreports.aggregate, names) + self.assertEqual( + pyreports.aggregate(names, surnames, ages)[0], ("Matteo", "Guadrini", 35) + ) + ages = ["Name", "Surname"] + self.assertRaises( + tablib.InvalidDimensions, pyreports.aggregate, names, surnames, ages + ) + self.assertRaises(pyreports.DataObjectError, pyreports.aggregate, names) def test_aggregate_fill_empty(self): names = self.data.get_col(0) surnames = self.data.get_col(1) - ages = ['Name', 'Surname'] - self.assertEqual(pyreports.aggregate(names, surnames, ages, - fill_empty=True)[2], - ('Ford', 'Prefect', None)) + ages = ["Name", "Surname"] + self.assertEqual( + pyreports.aggregate(names, surnames, ages, fill_empty=True)[2], + ("Ford", "Prefect", None), + ) def test_chunks(self): - data = Dataset(*[('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.extend([('Matteo', 'Guadrini', 35), ('Arthur', 'Dent', 42), ('Ford', 'Prefect', 42)]) - data.headers = ['name', 'surname', 'age'] - self.assertEqual(list(pyreports.chunks(data, 4))[0][0], ('Matteo', 'Guadrini', 35)) + data = Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.extend( + [ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.headers = ["name", "surname", "age"] + self.assertEqual( + list(pyreports.chunks(data, 4))[0][0], ("Matteo", "Guadrini", 35) + ) def test_merge(self): - self.assertEqual(pyreports.merge(self.data, self.data)[3], - ('Matteo', 'Guadrini', 35)) + self.assertEqual( + pyreports.merge(self.data, self.data)[3], ("Matteo", "Guadrini", 35) + ) def test_deduplication(self): - data = Dataset(*[('Matteo', 'Guadrini', 35), - ('Arthur', 'Dent', 42), - ('Matteo', 'Guadrini', 35)]) + data = Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) self.assertEqual(len(pyreports.deduplicate(data)), 2) + def test_data_object(self): + data = pyreports.DataObject(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataObject) + self.assertIsInstance(data.data, tablib.Dataset) -if __name__ == '__main__': + def test_data_adapters(self): + data = pyreports.DataAdapters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataAdapters) + + def test_data_adapters_aggregate(self): + names = self.data.get_col(0) + surnames = self.data.get_col(1) + ages = self.data.get_col(2) + data = pyreports.DataAdapters(Dataset()) + self.assertRaises( + pyreports.DataObjectError, data.aggregate, names, surnames, ages + ) + data = pyreports.DataAdapters(Dataset(*[("Heart",)])) + data.aggregate(names, surnames, ages) + self.assertEqual(data.data[0], ("Heart", "Matteo", "Guadrini", 35)) + + def test_data_adapters_merge(self): + data = pyreports.DataAdapters(Dataset()) + self.assertRaises(pyreports.DataObjectError, data.merge, self.data) + data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) + data.merge(self.data) + + def test_data_adapters_counter(self): + data = pyreports.DataAdapters(Dataset(*[("Arthur", "Dent", 42)])) + data.merge(self.data) + counter = data.counter() + self.assertEqual(counter["Arthur"], 2) + + def test_adapters_chunks(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.extend( + [ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + data.data.headers = ["name", "surname", "age"] + self.assertEqual(list(data.chunks(4))[0][0], ("Matteo", "Guadrini", 35)) + + def test_data_adapters_deduplicate(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) + ) + data.deduplicate() + self.assertEqual(len(data.data), 2) + + def test_data_adapters_iter(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ] + ) + ) + self.assertEqual(list(iter(data.data))[1], ("Arthur", "Dent", 42)) + + def test_data_adapters_get_items(self): + data = pyreports.DataAdapters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Matteo", "Guadrini", 35), + ], + headers=("name", "surname", "age"), + ) + ) + # Get row + self.assertEqual(data[1], ("Arthur", "Dent", 42)) + # Get column + self.assertEqual(data["name"], ["Matteo", "Arthur", "Matteo"]) + + def test_data_printers(self): + data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertIsInstance(data, pyreports.DataPrinters) + self.assertIsInstance(data.data, tablib.Dataset) + + def test_data_printers_len(self): + data = pyreports.DataPrinters(Dataset(*[("Matteo", "Guadrini", 35)])) + self.assertEqual(1, len(data)) + + def test_data_printers_average(self): + data = pyreports.DataPrinters( + Dataset(*[("Matteo", "Guadrini", 35), ("Arthur", "Dent", 42)]) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.average(2), 38.5) + + def test_data_printers_most_common(self): + data = pyreports.DataPrinters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.most_common("Age"), 42) + + def test_data_printers_percentage(self): + data = pyreports.DataPrinters( + Dataset( + *[ + ("Matteo", "Guadrini", 35), + ("Arthur", "Dent", 42), + ("Ford", "Prefect", 42), + ] + ) + ) + data.data.headers = ["Name", "Surname", "Age"] + self.assertEqual(data.percentage(42), 66.66666666666666) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_db.py b/tests/test_db.py index 5acb974..50b53df 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -5,7 +5,6 @@ class TestDBConnection(unittest.TestCase): - def test_connection(self): # pyreports.io.Connection object self.assertRaises(TypeError, pyreports.io.Connection) @@ -13,86 +12,65 @@ def test_connection(self): def test_sqllite_connection(self): # Simulate pyreports.io.SQLliteConnection object conn = MagicMock() - with patch(target='sqlite3.connect') as mock: + with patch(target="sqlite3.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.database = 'mydb.db' - self.assertEqual(conn.connection.database, 'mydb.db') + conn.connection.database = "mydb.db" + self.assertEqual(conn.connection.database, "mydb.db") # Test close conn.cursor.close() def test_mysql_connection(self): # Simulate pyreports.io.MySQLConnection object conn = MagicMock() - with patch(target='mysql.connector.connect') as mock: + with patch(target="mysql.connector.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'mysqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "mysqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 3306 - self.assertEqual(conn.connection.host, 'mysqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') + self.assertEqual(conn.connection.host, "mysqldb.local") + self.assertEqual(conn.connection.database, "mydb") + self.assertEqual(conn.connection.username, "username") + self.assertEqual(conn.connection.password, "password") self.assertEqual(conn.connection.port, 3306) # Test close conn.cursor.close() - def test_mssqldb_connection(self): - # Simulate pyreports.io.MSSQLConnection object - conn = MagicMock() - with patch(target='pymssql.connect') as mock: - # Test connect - conn.connection = mock.return_value - conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'mssqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' - conn.connection.port = 1433 - self.assertEqual(conn.connection.host, 'mssqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') - self.assertEqual(conn.connection.port, 1433) - # Test close - conn.cursor.close() - def test_postgresqldb_connection(self): # Simulate pyreports.io.PostgreSQLConnection object conn = MagicMock() - with patch(target='psycopg2.connect') as mock: + with patch(target="psycopg2.connect") as mock: # Test connect conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'postgresqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "postgresqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 5432 - self.assertEqual(conn.connection.host, 'postgresqldb.local') - self.assertEqual(conn.connection.database, 'mydb') - self.assertEqual(conn.connection.username, 'username') - self.assertEqual(conn.connection.password, 'password') + self.assertEqual(conn.connection.host, "postgresqldb.local") + self.assertEqual(conn.connection.database, "mydb") + self.assertEqual(conn.connection.username, "username") + self.assertEqual(conn.connection.password, "password") self.assertEqual(conn.connection.port, 5432) # Test close conn.cursor.close() class TestDBManager(unittest.TestCase): - conn = MagicMock() - with patch(target='psycopg2.connect') as mock: + with patch(target="psycopg2.connect") as mock: conn.connection = mock.return_value conn.cursor = conn.connection.cursor.return_value - conn.connection.host = 'postgresqldb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "postgresqldb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 5432 def test_db_manager(self): @@ -102,25 +80,24 @@ def test_db_manager(self): # Test reconnect db_manager.reconnect() # Test SELECT query - db_manager.execute('SELECT * from test') + db_manager.execute("SELECT * from test") data = db_manager.fetchall() self.assertIsInstance(data, Dataset) # Test store procedure - db_manager.callproc('myproc') + db_manager.callproc("myproc") data = db_manager.fetchone() self.assertIsInstance(data, Dataset) class TestNoSQLManager(unittest.TestCase): - conn = MagicMock() - with patch(target='nosqlapi.Connection') as mock: + with patch(target="nosqlapi.Connection") as mock: conn.connection = mock.return_value conn.session = conn.connection.connect() - conn.connection.host = 'mongodb.local' - conn.connection.database = 'mydb' - conn.connection.username = 'username' - conn.connection.password = 'password' + conn.connection.host = "mongodb.local" + conn.connection.database = "mydb" + conn.connection.username = "username" + conn.connection.password = "password" conn.connection.port = 27017 def test_nosql_manager(self): @@ -128,7 +105,7 @@ def test_nosql_manager(self): nosql_manager = pyreports.io.NoSQLManager(connection=self.conn) self.assertIsInstance(nosql_manager, pyreports.io.NoSQLManager) # Test get data - data = nosql_manager.get('doc1') + data = nosql_manager.get("doc1") self.assertIsInstance(data, Dataset) # Test find data data = nosql_manager.find({"name": "Arthur"}) @@ -136,11 +113,10 @@ def test_nosql_manager(self): class TestLDAPManager(unittest.TestCase): - conn = MagicMock() - with patch(target='ldap3.Server') as mock: + with patch(target="ldap3.Server") as mock: conn.connector = mock.return_value - with patch(target='ldap3.Connection') as mock: + with patch(target="ldap3.Connection") as mock: conn.bind = mock.return_value def test_bind(self): @@ -148,8 +124,10 @@ def test_bind(self): self.conn.bind.unbind() def test_query(self): - self.conn.bind.search('OU=test,DC=test,DC=local', 'objectCategory=person', ['name', 'sn', 'phone']) + self.conn.bind.search( + "OU=test,DC=test,DC=local", "objectCategory=person", ["name", "sn", "phone"] + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()