1.0.0 (#114)

* Integrate BBHelper (#109)

* Build for multiple platforms (#110)

* Update security policy.

* Add linting support (#111)

* Update documentation (#113)

* Update CHANGELOG.md

.github/workflows/build-and-test.yml

@@ -19,6 +19,7 @@ jobs:
         with:
           python-version: '3.11'
       - run: pip install -r requirements.txt
+      - run: pip install pylint && pylint ./bb-ripper/
       - run: python3 ./bb-ripper/.
         env:
           BB_USER : ${{ secrets.BB_USER }}

.github/workflows/push-image-workflow.yml

@@ -12,6 +12,9 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -30,6 +33,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           file: ./Dockerfile
           push: true
           tags: |
@@ -42,6 +46,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           file: ./Dockerfile
           build-args: AWSCLI=TRUE
           push: true

.gitignore

@@ -131,4 +131,5 @@ dmypy.json
 localdev
 env_setup.sh
 dockenv
-.DS_Store
+.DS_Store
+.vscode

.pylintrc

@@ -0,0 +1,39 @@
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
+# UNDEFINED.
+confidence=HIGH,
+           CONTROL_FLOW,
+           INFERENCE,
+           INFERENCE_FAILURE,
+           UNDEFINED
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then re-enable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        W0603,
+        W1510,
+        W0621,
+        C0103,
+        R0903
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0] - 2023-06-14
+
+### Added
+
+- Integrated BBHelper. Removed requirement to use external package.
+- Support for multiple architectures amd64 and arm64.
+- Documention update.
+- Linting support.
+
 ## [0.0.13] - 2023-06-10
 
 ### Added

README.md

@@ -1,5 +1,4 @@
-# bb-ripper
-# Bitbucket Repo Ripper
+# bb-ripper - Bitbucket Repo Ripper
 
 This project downloads all Bitbucket repositories and every branch for each repository. The resulting downloads are compressed with `tar`.
 
@@ -27,51 +26,58 @@ You can use the optional environment variable `BB_TEST_COUNTER` to only pull dow
 
 This has been developed and tested on `Python 3.11.2` and MacOS
 
-To install python dependencies 
-```
+To install python dependencies
+
+```bash
 pip install -r requirements.txt
 ```
 
-### Set the environment variables 
+### Set the environment variables
+
 Open `env_setup.sh.example` and add the values for your environment. Rename the file to `env_setup.sh` Source the environment variables to add the to you session.
-```
+
+```bash
 source env_setup.sh
 ```
 
-
 ### Run
 
-
 To run the ripper
-```
+
+```bash
 cd bb-ripper
 python3 .
 ```
 
 ### Running the docker image
+
 To run the image, create a docker environment file with the variables required named `docenv`. Create a directory named `data` to store the repositories. This directory will be mounted to the `/data` volume in the container.
 
-```
+The repository contains a file named `docenv.example` that is a template for the `docenv` file.
+
+```bash
 docker pull jvrck/bbripper
 docker run --env-file dockenv -v $(pwd)/data:/data  --rm -it  jvrck/bbripper:latest
 ```
 
-
 ### Building and running the docker image
+
 To build the docker image
-```
+
+```bash
 docker build --no-cache -t ripper .
 ```
 
 To run the image, create a docker environment file with the variables required named `docenv`. Create a directory named `data` to store the repositories. This directory will be mounted to the `/data` volume in the container.
 
-```
+```bash
  docker run --env-file dockenv -v $(pwd)/data:/data  --rm -it  ripper:latest
 ```
 
-## Docker image with AWS CLI.
-There is a variant of the image that has the AWS CLI preinstalled. This is done during the build by passing a build argument to the Docker command.
+## Docker image with AWS CLI
 
-```
+There is a variant of the image that has the AWS CLI preinstalled. This is done during the build by passing a build argument to the Docker `build` command.
+
+```bash
 docker build --no-cache -t bbripper-aws --build-arg AWSCLI=TRUE .
-```
+```

SECURITY.md

@@ -4,7 +4,7 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 0.0.x   | :white_check_mark: |
+| 1.x.x   | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

bb-ripper/__main__.py

@@ -1,9 +1,18 @@
-from bbhelper import bbhelper
-import sys,os
+"""
+The main entrypoint for the application.
+"""
+import time
+import sys
+import os
+from ripper_utils import create_output_directory
+from ripper_utils import check_git
+from ripper_utils import clone_repo
+from ripper_utils import zip_output_dir
+from ripper_utils import delete_output_directory
+from bbhelper import BBRepo
+
 BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/bb-ripper'
 sys.path.insert(0, BASE)
-from ripper_utils import *
-import time
 
 start_time = time.time()
 
@@ -12,9 +21,9 @@
     print(output_dir)
 
     # # Get all repos in workspace
-    workspace_repos = bbhelper.BBRepo.GetWorkspaceRepos(os.environ['BB_WORKSPACE'])
+    workspace_repos = BBRepo.GetWorkspaceRepos(os.environ['BB_WORKSPACE'])
 
-    counter = 0
+    COUNTER = 0
     test_counter = os.environ.get('BB_TEST_COUNTER')
 
     for repo in workspace_repos:
@@ -24,14 +33,14 @@
 
         clone_repo(repo, output_dir)
 
-        counter += 1
-        
+        COUNTER += 1
+
         # test that test_counter is present
         if test_counter is not None:
             # test test_counter is a valid integer
             if test_counter.isdigit():
-                # test for valid test_counter > 0 and counter is equal to test_counter 
-                if (int(test_counter) == counter) and (int(test_counter) > 0):
+                # test for valid test_counter > 0 and counter is equal to test_counter
+                if (int(test_counter) == COUNTER) and (int(test_counter) > 0):
                     break
 
     zip_output_dir()
@@ -41,5 +50,5 @@
     print('git is not installed...Exiting')
 
 print("--- COMPLETE ---")
-print("--- %s seconds ---" % (time.time() - start_time))
+print(f"--- {(time.time() - start_time)} seconds ---")
 print("You have been ripped by the fist")

bb-ripper/bbhelper.py

@@ -0,0 +1,94 @@
+"""Module that provides bitbucket REST API helper functions"""
+import os
+import json
+import logging
+import requests
+
+BB_API_ENDPOINT = 'https://api.bitbucket.org/2.0'
+
+LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
+logging.basicConfig(level=LOGLEVEL)
+
+
+class BBProject:
+    """
+    Bitbucket Projects 
+    Contains the structure of a bitbucket project
+    Takes json response object in constructor
+    """
+
+    def __init__(self, json):
+        self.name = json['name']
+        self.key = json['key']
+        self.repo_rest_url = json['links']['repositories']['href']
+
+    @staticmethod
+    def GetProjects(workspace_name):
+        """
+        Static method to get all projects for a workspace
+        """
+        request_endpoint = f"{BB_API_ENDPOINT}/workspaces/{workspace_name}/projects"
+        logging.info("Get projects request endpoint %s", request_endpoint)
+
+        session = requests.session()
+        session.auth = (os.environ['BB_USER'], os.environ['BB_PASSWORD'])
+
+        projects_request = session.get(request_endpoint)
+        projects_json = json.loads(projects_request.text)
+        projects = []
+
+        for p in projects_json['values']:
+            projects.append(BBProject(p))
+
+        session.close()
+        return projects
+
+
+class BBRepo:
+    """
+    Bitbucket Repo
+    Contains the structure of a bitbucket repo
+    Takes json response object in constructor
+    """
+
+    def __init__(self, data):
+        self.name = data['name']
+        self.project = data['project']
+        self.links = data['links']['clone']
+
+        for l in self.links:
+            if l['name'] == 'ssh':
+                self.ssh = l['href']
+            if l['name'] == 'https':
+                self.https = l['href']
+
+    @staticmethod
+    def GetWorkspaceRepos(workspace_name):
+        """
+        A function that returns repos for a workspace
+        """
+        request_endpoint = f"{BB_API_ENDPOINT}/repositories/{workspace_name}"
+        logging.info("Get workspace repos request endpoint %s",
+                     request_endpoint)
+
+        session = requests.session()
+        session.auth = (os.environ['BB_USER'], os.environ['BB_PASSWORD'])
+
+        repos = []
+
+        process = True
+        while process:
+            repos_request = session.get(request_endpoint)
+            repos_json = json.loads(repos_request.text)
+            for r in repos_json['values']:
+                repos.append(BBRepo(r))
+
+            logging.info("Repos returned count ... %s", request_endpoint)
+
+            if 'next' in repos_json:
+                request_endpoint = repos_json['next']
+            else:
+                process = False
+
+        session.close()
+        return repos