From e6a13f3c0b45573c6cf85f4dbbb60d74e5989c7d Mon Sep 17 00:00:00 2001 From: Jim Vrckovski <100883+jvrck@users.noreply.github.com> Date: Wed, 14 Jun 2023 05:48:20 +1000 Subject: [PATCH] 1.0.0 (#114) * Integrate BBHelper (#109) * Build for multiple platforms (#110) * Update security policy. * Add linting support (#111) * Update documentation (#113) * Update CHANGELOG.md --- .github/workflows/build-and-test.yml | 1 + .github/workflows/push-image-workflow.yml | 5 ++ .gitignore | 3 +- .pylintrc | 39 ++++++++++ CHANGELOG.md | 9 +++ README.md | 40 ++++++---- SECURITY.md | 2 +- bb-ripper/__main__.py | 31 +++++--- bb-ripper/bbhelper.py | 94 +++++++++++++++++++++++ bb-ripper/ripper_utils.py | 71 ++++++++++------- docenv.example | 4 + requirements.txt | 52 ------------- 12 files changed, 241 insertions(+), 110 deletions(-) create mode 100644 .pylintrc create mode 100644 bb-ripper/bbhelper.py create mode 100644 docenv.example diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index dc5ae50..e9b3e7d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -19,6 +19,7 @@ jobs: with: python-version: '3.11' - run: pip install -r requirements.txt + - run: pip install pylint && pylint ./bb-ripper/ - run: python3 ./bb-ripper/. env: BB_USER : ${{ secrets.BB_USER }} diff --git a/.github/workflows/push-image-workflow.yml b/.github/workflows/push-image-workflow.yml index 04682f7..6220967 100644 --- a/.github/workflows/push-image-workflow.yml +++ b/.github/workflows/push-image-workflow.yml @@ -12,6 +12,9 @@ jobs: steps: - name: Check out repo uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub uses: docker/login-action@v2 @@ -30,6 +33,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + platforms: linux/amd64,linux/arm64 file: ./Dockerfile push: true tags: | @@ -42,6 +46,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + platforms: linux/amd64,linux/arm64 file: ./Dockerfile build-args: AWSCLI=TRUE push: true diff --git a/.gitignore b/.gitignore index d401dbc..b4ebabb 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,5 @@ dmypy.json localdev env_setup.sh dockenv -.DS_Store \ No newline at end of file +.DS_Store +.vscode \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..21b208c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,39 @@ +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + W0603, + W1510, + W0621, + C0103, + R0903 + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef8cad..7d94607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.0] - 2023-06-14 + +### Added + +- Integrated BBHelper. Removed requirement to use external package. +- Support for multiple architectures amd64 and arm64. +- Documention update. +- Linting support. + ## [0.0.13] - 2023-06-10 ### Added diff --git a/README.md b/README.md index 7153f3a..54f5d5b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -# bb-ripper -# Bitbucket Repo Ripper +# bb-ripper - Bitbucket Repo Ripper This project downloads all Bitbucket repositories and every branch for each repository. The resulting downloads are compressed with `tar`. @@ -27,51 +26,58 @@ You can use the optional environment variable `BB_TEST_COUNTER` to only pull dow This has been developed and tested on `Python 3.11.2` and MacOS -To install python dependencies -``` +To install python dependencies + +```bash pip install -r requirements.txt ``` -### Set the environment variables +### Set the environment variables + Open `env_setup.sh.example` and add the values for your environment. Rename the file to `env_setup.sh` Source the environment variables to add the to you session. -``` + +```bash source env_setup.sh ``` - ### Run - To run the ripper -``` + +```bash cd bb-ripper python3 . ``` ### Running the docker image + To run the image, create a docker environment file with the variables required named `docenv`. Create a directory named `data` to store the repositories. This directory will be mounted to the `/data` volume in the container. -``` +The repository contains a file named `docenv.example` that is a template for the `docenv` file. + +```bash docker pull jvrck/bbripper docker run --env-file dockenv -v $(pwd)/data:/data --rm -it jvrck/bbripper:latest ``` - ### Building and running the docker image + To build the docker image -``` + +```bash docker build --no-cache -t ripper . ``` To run the image, create a docker environment file with the variables required named `docenv`. Create a directory named `data` to store the repositories. This directory will be mounted to the `/data` volume in the container. -``` +```bash docker run --env-file dockenv -v $(pwd)/data:/data --rm -it ripper:latest ``` -## Docker image with AWS CLI. -There is a variant of the image that has the AWS CLI preinstalled. This is done during the build by passing a build argument to the Docker command. +## Docker image with AWS CLI -``` +There is a variant of the image that has the AWS CLI preinstalled. This is done during the build by passing a build argument to the Docker `build` command. + +```bash docker build --no-cache -t bbripper-aws --build-arg AWSCLI=TRUE . -``` \ No newline at end of file +``` diff --git a/SECURITY.md b/SECURITY.md index 7e6ae22..6e57de3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ | Version | Supported | | ------- | ------------------ | -| 0.0.x | :white_check_mark: | +| 1.x.x | :white_check_mark: | ## Reporting a Vulnerability diff --git a/bb-ripper/__main__.py b/bb-ripper/__main__.py index 295c19d..639c19a 100644 --- a/bb-ripper/__main__.py +++ b/bb-ripper/__main__.py @@ -1,9 +1,18 @@ -from bbhelper import bbhelper -import sys,os +""" +The main entrypoint for the application. +""" +import time +import sys +import os +from ripper_utils import create_output_directory +from ripper_utils import check_git +from ripper_utils import clone_repo +from ripper_utils import zip_output_dir +from ripper_utils import delete_output_directory +from bbhelper import BBRepo + BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/bb-ripper' sys.path.insert(0, BASE) -from ripper_utils import * -import time start_time = time.time() @@ -12,9 +21,9 @@ print(output_dir) # # Get all repos in workspace - workspace_repos = bbhelper.BBRepo.GetWorkspaceRepos(os.environ['BB_WORKSPACE']) + workspace_repos = BBRepo.GetWorkspaceRepos(os.environ['BB_WORKSPACE']) - counter = 0 + COUNTER = 0 test_counter = os.environ.get('BB_TEST_COUNTER') for repo in workspace_repos: @@ -24,14 +33,14 @@ clone_repo(repo, output_dir) - counter += 1 - + COUNTER += 1 + # test that test_counter is present if test_counter is not None: # test test_counter is a valid integer if test_counter.isdigit(): - # test for valid test_counter > 0 and counter is equal to test_counter - if (int(test_counter) == counter) and (int(test_counter) > 0): + # test for valid test_counter > 0 and counter is equal to test_counter + if (int(test_counter) == COUNTER) and (int(test_counter) > 0): break zip_output_dir() @@ -41,5 +50,5 @@ print('git is not installed...Exiting') print("--- COMPLETE ---") -print("--- %s seconds ---" % (time.time() - start_time)) +print(f"--- {(time.time() - start_time)} seconds ---") print("You have been ripped by the fist") diff --git a/bb-ripper/bbhelper.py b/bb-ripper/bbhelper.py new file mode 100644 index 0000000..fa1d175 --- /dev/null +++ b/bb-ripper/bbhelper.py @@ -0,0 +1,94 @@ +"""Module that provides bitbucket REST API helper functions""" +import os +import json +import logging +import requests + +BB_API_ENDPOINT = 'https://api.bitbucket.org/2.0' + +LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() +logging.basicConfig(level=LOGLEVEL) + + +class BBProject: + """ + Bitbucket Projects + Contains the structure of a bitbucket project + Takes json response object in constructor + """ + + def __init__(self, json): + self.name = json['name'] + self.key = json['key'] + self.repo_rest_url = json['links']['repositories']['href'] + + @staticmethod + def GetProjects(workspace_name): + """ + Static method to get all projects for a workspace + """ + request_endpoint = f"{BB_API_ENDPOINT}/workspaces/{workspace_name}/projects" + logging.info("Get projects request endpoint %s", request_endpoint) + + session = requests.session() + session.auth = (os.environ['BB_USER'], os.environ['BB_PASSWORD']) + + projects_request = session.get(request_endpoint) + projects_json = json.loads(projects_request.text) + projects = [] + + for p in projects_json['values']: + projects.append(BBProject(p)) + + session.close() + return projects + + +class BBRepo: + """ + Bitbucket Repo + Contains the structure of a bitbucket repo + Takes json response object in constructor + """ + + def __init__(self, data): + self.name = data['name'] + self.project = data['project'] + self.links = data['links']['clone'] + + for l in self.links: + if l['name'] == 'ssh': + self.ssh = l['href'] + if l['name'] == 'https': + self.https = l['href'] + + @staticmethod + def GetWorkspaceRepos(workspace_name): + """ + A function that returns repos for a workspace + """ + request_endpoint = f"{BB_API_ENDPOINT}/repositories/{workspace_name}" + logging.info("Get workspace repos request endpoint %s", + request_endpoint) + + session = requests.session() + session.auth = (os.environ['BB_USER'], os.environ['BB_PASSWORD']) + + repos = [] + + process = True + while process: + repos_request = session.get(request_endpoint) + repos_json = json.loads(repos_request.text) + for r in repos_json['values']: + repos.append(BBRepo(r)) + + logging.info("Repos returned count ... %s", request_endpoint) + + if 'next' in repos_json: + request_endpoint = repos_json['next'] + else: + process = False + + session.close() + return repos diff --git a/bb-ripper/ripper_utils.py b/bb-ripper/ripper_utils.py index 63f6bc8..d074f46 100644 --- a/bb-ripper/ripper_utils.py +++ b/bb-ripper/ripper_utils.py @@ -1,5 +1,7 @@ -from asyncio import constants, subprocess -import os +""" +Module that provides utilities for the ripper program. +""" +import os import subprocess from datetime import datetime from shutil import which @@ -8,8 +10,10 @@ output_dir = "" output_base = "" -# Creates the output directory def create_output_directory(): + """ + Function that creates the output directory. + """ global output_base if "BB_RIPPER_EXPORT_DIRECTORY" in os.environ: output_base = os.environ['BB_RIPPER_EXPORT_DIRECTORY'] @@ -19,40 +23,47 @@ def create_output_directory(): # Append backslash to output base it it is not present. if not output_base.endswith('/'): output_base += '/' - + global output_dir - output_dir = '{0}bbr-{1}-{2}'.format(output_base, os.environ['BB_WORKSPACE'], datetime.today().strftime('%Y-%m-%d-T%H.%M.%S')) - print("the output dir === {0}".format(output_dir)) + datetime_str = datetime.today().strftime('%Y-%m-%d-T%H.%M.%S') + output_dir = f"{output_base}bbr-{os.environ['BB_WORKSPACE']}-{datetime_str}" + print(f"the output dir === {output_dir}") isExists = os.path.exists(output_dir) - + if not isExists: os.makedirs(output_dir) return output_dir -# Delete output directory def delete_output_directory(): + """ + Delete output directory. + """ if os.path.exists(output_dir): shutil.rmtree(output_dir) -# Checks if git is installed def check_git(): - return(which('git')) + """ + Function that checks if git is installed. + """ + return which('git') -# clones a repo and all it branches def clone_repo(repo, output_dir): - clone_dir = "{0}/{1}".format(output_dir, repo.name) - + """ + Function that clones the repo to the output directory. + """ + clone_dir = f"{output_dir}/{repo.name}" + if not os.path.exists(clone_dir): os.makedirs(clone_dir) os.chdir(clone_dir) # make sure to suppress output when using https - os.system("git clone {0} . >/dev/null 2>&1".format(get_https_url(repo.https))) - - branches = subprocess.run(["git", "branch", "-r"], - stdout=subprocess.PIPE, - universal_newlines=True) - + os.system(f"git clone {get_https_url(repo.https)} . >/dev/null 2>&1") + + branches = subprocess.run(["git", "branch", "-r"], + stdout=subprocess.PIPE, + universal_newlines=True) + branch_list = branches.stdout.split("\n") for b in branch_list: @@ -61,18 +72,22 @@ def clone_repo(repo, output_dir): print("SKIPPING HEAD") else: clone_branch = b.replace(" ", "").replace("origin/", "") - clone_branch_cmd = "git checkout {0}".format(clone_branch) + clone_branch_cmd = f"git checkout {clone_branch}" os.system(clone_branch_cmd) -# zips the output directory def zip_output_dir(): - dir = output_dir.replace(output_base, "") - print("Dir to zip: {0}".format(dir)) + """ + Function that zips the output directory into a tarball archive. + """ + directory = output_dir.replace(output_base, "") + print(f"Dir to zip: {directory}") os.chdir(output_base) - os.system("tar -cvzf {0}.tar.gz {0}/".format(dir)) + os.system(f"tar -cvzf {directory}.tar.gz {directory}/") -# get a https url that contains the username and password def get_https_url(url): - search_text = "https://{0}".format(os.environ['BB_USER']) - replace_text = "https://{0}:{1}".format(os.environ['BB_USER'], os.environ['BB_PASSWORD']) - return url.replace(search_text, replace_text) \ No newline at end of file + """ + Function that returns a https uld that contains the username and password. + """ + search_text = f"https://{os.environ['BB_USER']}" + replace_text = f"https://{ os.environ['BB_USER']}:{os.environ['BB_PASSWORD']}" + return url.replace(search_text, replace_text) diff --git a/docenv.example b/docenv.example new file mode 100644 index 0000000..28d48cd --- /dev/null +++ b/docenv.example @@ -0,0 +1,4 @@ +BB_USER= +BB_PASSWORD= +BB_WORKSPACE= +BB_RIPPER_EXPORT_DIRECTORY= \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dd9559c..cf89e54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,57 +1,5 @@ -bbhelper==0.0.1 -bleach==6.0.0 -build==0.10.0 -certifi==2023.5.7 -charset-normalizer -commonmark==0.9.1 -docutils==0.20.1 -idna==3.4 -importlib-metadata==6.6.0 -keyring==23.13.1 -packaging==23.1 -pep517==0.13.0 -pkginfo==1.9.6 -Pygments==2.15.1 -pyparsing==3.0.9 -readme-renderer==37.3 -requests -requests-toolbelt -rfc3986==2.0.0 -rich==13.4.1 -six==1.16.0 -tomli==2.0.1 -twine==4.0.2 -typing_extensions==4.6.3 -urllib3==2.0.3 -webencodings==0.5.1 -zipp==3.15.0 -bbhelper==0.0.1 -bleach==6.0.0 -build==0.10.0 certifi==2023.5.7 charset-normalizer==3.1.0 -commonmark==0.9.1 -docutils==0.20.1 idna==3.4 -importlib-metadata==6.6.0 -importlib-resources==5.12.0 -jaraco.classes==3.2.3 -keyring==23.13.1 -more-itertools==9.1.0 -packaging==23.1 -pep517==0.13.0 -pkginfo==1.9.6 -Pygments==2.15.1 -pyparsing==3.0.9 -readme-renderer==37.3 requests==2.31.0 -requests-toolbelt==1.0.0 -rfc3986==2.0.0 -rich==13.4.1 -six==1.16.0 -tomli==2.0.1 -twine==4.0.2 -typing_extensions==4.6.3 urllib3==2.0.3 -webencodings==0.5.1 -zipp==3.15.0