diff --git a/.gitignore b/.gitignore index b6e4761..bc97506 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ dmypy.json # Pyre type checker .pyre/ +/runtime/ diff --git a/.project b/.project new file mode 100644 index 0000000..dd01de9 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + eightanu + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..ede3d36 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,14 @@ + + + + + + /${PROJECT_DIR_NAME}/src + + + + python interpreter + + eightanu venv + + diff --git a/.settings/.gitignore b/.settings/.gitignore new file mode 100644 index 0000000..b012ade --- /dev/null +++ b/.settings/.gitignore @@ -0,0 +1 @@ +/org.eclipse.core.resources.prefs diff --git a/README.md b/README.md index b85161a..a9046d7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,8 @@ # eightanu Python script to export 8a.nu logbook + +# Introduction +Since the restart of 8a.nu you can no longer export your ascents. This project aims to provide a tool to export personal ascents. + +# Usage +more to come... diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3cf1073 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +selenium==3.141.0 +urllib3==1.25.9 diff --git a/src/eightanu/__init__.py b/src/eightanu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/eightanu/cli.py b/src/eightanu/cli.py new file mode 100644 index 0000000..a2d375e --- /dev/null +++ b/src/eightanu/cli.py @@ -0,0 +1,119 @@ +#!/usr/local/bin/python3.6 +# encoding: utf-8 +''' +eightanu.cli -- The command line interface to export the new 8a.new ascents data. + +eightanu.cli is an exporter that scrap your ascents from the new 8a.nu website. + +@author: lordyavin +@copyright: 2020 lordyavin. All rights reserved. +@license: MIT +@contact: github@klesatschke.net +@deffield updated: Updated +''' +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter +import os +import sys + +from eightanu import webdriver +from eightanu.export import export + +__all__ = [] +__version__ = 0.1 +__date__ = '2020-05-27' +__updated__ = '2020-05-28' + +DEBUG = 0 +TESTRUN = 0 +PROFILE = 0 + + +class CLIError(Exception): + '''Generic exception to raise and log different fatal errors.''' + + def __init__(self, msg): + super(CLIError).__init__(type(self)) + self.msg = "E: %s" % msg + + def __str__(self): + return self.msg + + def __unicode__(self): + return self.msg + + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by user_name on %s. + Copyright 2020 organization_name. All rights reserved. + + Licensed under the Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-b", "--browser", dest="browser", type=str, choices=webdriver.SUPPORTED_BROWSER, required=True) + parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]", default=0) + parser.add_argument('-V', '--version', action='version', version=program_version_message) + + parser.add_argument("username", type=str, help="Your 8a user name") + + # Process arguments + args = parser.parse_args() + + verbose = args.verbose + if verbose > 0: + print("Verbose mode on") + + export(args.browser, args.username, verbose) + + return 0 + except KeyboardInterrupt: + ### handle keyboard interrupt ### + return 0 + except Exception as e: + if DEBUG or TESTRUN: + raise e + indent = len(program_name) * " " + sys.stderr.write(program_name + ": " + repr(e) + "\n") + sys.stderr.write(indent + " for help use --help") + return 2 + + +if __name__ == "__main__": + if TESTRUN: + import doctest + doctest.testmod() + if PROFILE: + import cProfile + import pstats + profile_filename = 'eightanu.export_ascents_profile.txt' + cProfile.run('main()', profile_filename) + statsfile = open("profile_stats.txt", "wb") + p = pstats.Stats(profile_filename, stream=statsfile) + stats = p.strip_dirs().sort_stats('cumulative') + stats.print_stats() + statsfile.close() + sys.exit(0) + sys.exit(main()) diff --git a/src/eightanu/export.py b/src/eightanu/export.py new file mode 100644 index 0000000..ededef5 --- /dev/null +++ b/src/eightanu/export.py @@ -0,0 +1,95 @@ +''' +eightanu.export -- The 8a.nu logbook export + +@author: lordyavin +@copyright: 2020 lordyavin. All rights reserved. +@license: MIT +@contact: github@klesatschke.net +@deffield updated: Updated +''' +from selenium.webdriver.remote.webelement import WebElement + +from eightanu import webdriver +from selenium.webdriver.remote.webdriver import WebDriver +from eightanu.webdriver import SUPPORTED_BROWSER + +def _select_alltime_ascents(driver): + assert isinstance(driver, WebDriver) + + filters = driver.find_element_by_class_name("ascent-filters") + assert isinstance(filters, WebElement) + + all_options = filters.find_elements_by_tag_name("option") + for option in all_options: + if option.text == "All Time": + option.click() + +def _sort_by_date(driver): + assert isinstance(driver, WebDriver) + + order = driver.find_element_by_class_name("ascent-filters__order") + all_options = order.find_elements_by_tag_name("option") + for option in all_options: + if option.text == "Date": + option.click() + + +def _read_table_headers(table): + assert isinstance(table, WebElement) + + thead = table.find_element_by_tag_name("thead") + table_headers = thead.find_elements_by_tag_name("th")[1:] # skip first empty cell + headers = ["DATE"] + [cell.text.strip() for cell in table_headers] + return headers + + +def _read_ascents(table, headers): + assert isinstance(table, WebElement) + assert isinstance(headers, (list, tuple)) + + ascents = [] + name_idx = headers.index("NAME") + groups = table.find_elements_by_tag_name("tbody") + for group in groups: + date = group.find_element_by_tag_name("th").text + rows = group.find_elements_by_tag_name("tr") + for row in rows: + cells = row.find_elements_by_tag_name("td")[1:] # skip first empty cell + if cells: + ascent = [date] + [cell.text.strip() for cell in cells] + name = ascent[name_idx] + if "\n" in name: + ascent[name_idx] = name[:name.index("\n")] + ascents.append(ascent) + + return ascents + + +def _print_logbook(headers, ascents): + assert isinstance(headers, (list, tuple)) + assert isinstance(ascents, (list, tuple)) + + print(";".join(headers)) + for ascent in ascents: + print(";".join(ascent)) + +def export(browser, username, verbose=0): + assert browser in SUPPORTED_BROWSER + assert isinstance(username, str) + assert isinstance(verbose, int) + + username = username.replace(" ", "-").lower() + url = "https://www.8a.nu/user/{username}/sportclimbing".format(username=username) + + driver = webdriver.get(browser, verbose) + try: + driver.get(url) + _select_alltime_ascents(driver) + _sort_by_date(driver) + + table = driver.find_element_by_class_name("user-ascents") + headers = _read_table_headers(table) + ascents = _read_ascents(table, headers) + _print_logbook(headers, ascents) + finally: + driver.close() diff --git a/src/eightanu/webdriver.py b/src/eightanu/webdriver.py new file mode 100644 index 0000000..ef1968e --- /dev/null +++ b/src/eightanu/webdriver.py @@ -0,0 +1,73 @@ +# encoding: utf-8 +''' +eightanu.webdriver -- Handles the selenium webdrivers. + +@author: lordyavin +@copyright: 2020 lordyavin. All rights reserved. +@license: MIT +@contact: github@klesatschke.net +@deffield updated: Updated +''' +import os +from zipfile import ZipFile + +from selenium import webdriver +from selenium.common.exceptions import WebDriverException +import urllib3 +from urllib3.response import HTTPResponse + +FIREFOX_DRIVER_URL = "https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-win64.zip" +CHROME_DRIVER_URL = "https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_win32.zip" + +FIREFOX = "Firefox" +CHROME = "Chrome" + +SUPPORTED_BROWSER = ( + FIREFOX, + CHROME, +) + +DOWNLOAD = { + FIREFOX: FIREFOX_DRIVER_URL, + CHROME: CHROME_DRIVER_URL +} + +DRIVER = { + FIREFOX: webdriver.Firefox, + CHROME: webdriver.Chrome, +} + + +def download(url, verbose=0): + http = urllib3.PoolManager() + + if verbose: + print("Downloading %s" % url) + + response = http.request('GET', url) + assert isinstance(response, HTTPResponse) + zipfilename = os.path.basename(url) + + with open(zipfilename, "wb") as zipfile: + zipfile.write(response.data) + + if verbose: + print("Extracting %s" % zipfilename) + + with ZipFile(zipfilename) as zipfile: + zipfile.extractall(".") + + os.remove(zipfilename) + + +def get(browser, verbose=0): + assert browser in SUPPORTED_BROWSER + try: + Driver = DRIVER[browser] + driver = Driver() + except WebDriverException as e: + if verbose: + print(e.msg) + download(DOWNLOAD[browser], verbose) + driver = Driver() + return driver