From f0f728190b9b67ed3ea34b6c2135ff9a2a4c72c5 Mon Sep 17 00:00:00 2001 From: Irakli Mchedlishvili Date: Tue, 17 Aug 2021 16:10:47 +0400 Subject: [PATCH] Feature/deployment logs (#150) * Intorduce Drone logging * Introduce deployments status * Intoroduce deployment version * Introduce image get and set Co-authored-by: Irakli Mchedlishvili Co-authored-by: Patricio Del Boca --- ckan_cloud_operator/drivers/helm/driver.py | 8 ++ .../providers/ckan/deployment/cli.py | 50 +++++++++- .../ckan/deployment/drone/__init__.py | 0 .../providers/ckan/deployment/drone/cli.py | 20 ++++ .../ckan/deployment/drone/constants.py | 1 + .../ckan/deployment/drone/manager.py | 91 +++++++++++++++++++ .../providers/ckan/deployment/manager.py | 36 ++++++++ .../providers/ckan/env/manager.py | 6 ++ 8 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 ckan_cloud_operator/providers/ckan/deployment/drone/__init__.py create mode 100644 ckan_cloud_operator/providers/ckan/deployment/drone/cli.py create mode 100644 ckan_cloud_operator/providers/ckan/deployment/drone/constants.py create mode 100644 ckan_cloud_operator/providers/ckan/deployment/drone/manager.py diff --git a/ckan_cloud_operator/drivers/helm/driver.py b/ckan_cloud_operator/drivers/helm/driver.py index a256676a..dfe068ba 100644 --- a/ckan_cloud_operator/drivers/helm/driver.py +++ b/ckan_cloud_operator/drivers/helm/driver.py @@ -81,5 +81,13 @@ def delete(tiller_namespace, release_name): tiller_cmd = '' if _check_helm_version() == 3 else f' --tiller-namespace {tiller_namespace}' subprocess.check_call(f'helm delete --purge --timeout 5 {release_name}' + tiller_cmd, shell=True) + +def check_status(instance_id): + subprocess.check_call(f'helm status ckan-cloud-{instance_id} -n {instance_id}', shell=True) + +def get_values(instance_id): + return subprocess.check_output(f'helm get values ckan-cloud-{instance_id} -n {instance_id} -o json', shell=True) + + def _check_helm_version(): return 3 if 'v3.' in str(subprocess.check_output('helm version -c', shell=True)) else 2 diff --git a/ckan_cloud_operator/providers/ckan/deployment/cli.py b/ckan_cloud_operator/providers/ckan/deployment/cli.py index 35871886..13cd6c0b 100644 --- a/ckan_cloud_operator/providers/ckan/deployment/cli.py +++ b/ckan_cloud_operator/providers/ckan/deployment/cli.py @@ -1,13 +1,61 @@ import click - from .helm import cli as helm_cli +from .drone import cli as drone_cli +from .drone import manager +from ckan_cloud_operator.providers.ckan.deployment import manager as deployment_manager +drone_manager = manager.Drone() @click.group() def deployment(): """Manage CKAN instance deployments""" pass +@click.group() +def image(): + """Manage CKAN instance deployments""" + pass + +@deployment.command() +@click.option('--branch', default='develop', help='Source Branch for build [default: develop]') +def logs(branch): + """See CKAN instances deployment Logs""" + drone_manager.initialize() + drone_manager.builds_logs(branch) + + +@deployment.command() +@click.argument('instance-id') +def status(instance_id): + """See CKAN instances deployment status""" + helm_driver.check_status(instance_id) + + +@deployment.command() +@click.argument('instance-id') +def version(instance_id): + """See CKAN instances deployment version""" + deployment_manager.get_deployment_version(instance_id) + + +@image.command() +@click.argument('instance-id') +@click.option('--service', default='ckan', help='Source Branch for build [default: develop]') +def get(instance_id, service): + """Get instances container image""" + deployment_manager.get_image(instance_id, service=service) + + +@image.command() +@click.argument('instance-id') +@click.argument('image-name') +@click.option('--service', default='ckan', help='Source Branch for build [default: develop]') +def set(instance_id, image_name, service): + """Set instances container image""" + deployment_manager.set_image(instance_id, image_name, service=service) + deployment.add_command(helm_cli.helm) +deployment.add_command(drone_cli.drone) +deployment.add_command(image) diff --git a/ckan_cloud_operator/providers/ckan/deployment/drone/__init__.py b/ckan_cloud_operator/providers/ckan/deployment/drone/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ckan_cloud_operator/providers/ckan/deployment/drone/cli.py b/ckan_cloud_operator/providers/ckan/deployment/drone/cli.py new file mode 100644 index 00000000..f1b657a5 --- /dev/null +++ b/ckan_cloud_operator/providers/ckan/deployment/drone/cli.py @@ -0,0 +1,20 @@ +import click + +from ckan_cloud_operator import logs + +from . import manager + + +drone_manager = manager.Drone() + +@click.group() +def drone(): + """Manage Drone CI/CD""" + pass + + +@drone.command() +@click.option('--force-update', default=False, help='Force update drone configurations [default: develop]', is_flag=True) +def initialize(force_update): + """Initialize drone""" + drone_manager.initialize(force_update) diff --git a/ckan_cloud_operator/providers/ckan/deployment/drone/constants.py b/ckan_cloud_operator/providers/ckan/deployment/drone/constants.py new file mode 100644 index 00000000..633589fb --- /dev/null +++ b/ckan_cloud_operator/providers/ckan/deployment/drone/constants.py @@ -0,0 +1 @@ +PROVIDER_ID='drone' diff --git a/ckan_cloud_operator/providers/ckan/deployment/drone/manager.py b/ckan_cloud_operator/providers/ckan/deployment/drone/manager.py new file mode 100644 index 00000000..0d3708d2 --- /dev/null +++ b/ckan_cloud_operator/providers/ckan/deployment/drone/manager.py @@ -0,0 +1,91 @@ +import requests +from urllib.parse import urljoin + +from ...env import manager as env_manager + + +class Drone(object): + + def initialize(self, force_prompt=False): + self.conf = env_manager._read_yaml().get('cicd', {}) + self.server_url = self.conf.get('serverUrl') + self.api_token = self.conf.get('token') + self.org = self.conf.get('organization') + self.repo = self.conf.get('repo') + if force_prompt or not self.conf: + self.server_url = input('Please enter Drone Server URL: ') + self.api_token = input(f'Please enter Drone API Token. See {self. server_url} acount: ') + self.org = input('Please enter Github organization name: ') + self.repo = input('Please enter Github repository name: ') + self.conf['cicd'] = { + 'serverUrl': self.server_url, + 'token': self.api_token, + 'organization': self.org, + 'repo': self.repo + } + env_manager._write_yaml(self.conf) + self.api_url = urljoin(self.server_url, '/'.join(['api/repos', f'{self.org}/{self.repo}', 'builds/'])) + self.header = {'Authorization': f'Bearer {self.api_token}'} + print('CCO is configured for Drone ') + + + def builds_list(self): + resp = requests.get(self.api_url, headers=self.header) + if not _check_for_200(resp): + return [] + return resp.json() + + + def builds_info(self, branch='develop'): + build_number = self.get_build_number(branch=branch) + if build_number is None: + print(f'Build {build_number} not found') + return None + url = urljoin(self.api_url, build_number) + builds_info_resp = requests.get(url, headers=self.header) + if not _check_for_200(builds_info_resp): + return {} + return builds_info_resp.json() + + + def builds_logs(self, branch='develop'): + build_info = self.builds_info(branch=branch) + if build_info is None: + return None + stages = build_info.get('stages', []) + for stage in stages: + steps = stage.get('steps', []) + stage_name = stage.get('name') + for step in steps: + step_name = step.get('name') + print(f'--- Build Stage: {stage_name} | Builds Step: {step_name}') + url = urljoin(self.api_url, '/'.join([ + self.build_number, + 'logs', + str(stage.get('number')), + str(step.get('number'))]) + ) + log_resp = requests.get(url, headers=self.header) + log_list = log_resp.json() + for _log in log_list: + print(_log.get('out').rstrip('\n')) + print(f'Showing logs for "{branch}" Branch') + + + def get_build_number(self, branch='develop'): + for build in self.builds_list(): + if build.get('source') == branch: + self.build_number = str(build.get('number')) + return str(build.get('number')) + print(f'No build found for branch {branch}') + return None + + +def _check_for_200(resp): + if resp.status_code == 200: + return True + if resp.status_code == 401: + print("Seems like you are not authorized") + print("Please run `cco ckan deployment drone initialize --force-update` and rerun") + return False + return False diff --git a/ckan_cloud_operator/providers/ckan/deployment/manager.py b/ckan_cloud_operator/providers/ckan/deployment/manager.py index 334de436..f66d0be8 100644 --- a/ckan_cloud_operator/providers/ckan/deployment/manager.py +++ b/ckan_cloud_operator/providers/ckan/deployment/manager.py @@ -1,3 +1,11 @@ +import json +import requests +from urllib.parse import urljoin + +from ckan_cloud_operator import kubectl +from ckan_cloud_operator.drivers.helm import driver as helm_driver + + def initialize(interactive=False): from .helm.manager import initialize as ckan_helm_initialize ckan_helm_initialize(interactive=interactive) @@ -32,6 +40,34 @@ def pre_update_hook(instance_id, instance_type, instance, override_spec, skip_ro dry_run=dry_run) +def get_deployment_version(instance_id): + values = helm_driver.get_values(instance_id) + values = json.loads(values) + site_url = values.get('siteUrl') + print('Current deployment version: ', requests.get(urljoin(site_url, 'version')).text) + + +def get_image(instance_id, service='ckan'): + deployment_info = kubectl.get('deployment', namespace=instance_id) + image_name = None + for item in deployment_info.get('items', []): + if item['metadata'].get('name') == service: + containers = item.get('spec', {}).get('template', {}).get('spec', {}).get('containers', []) + for container in containers: + image_name = container.get('image') + print(image_name) + if image_name is None: + print(f'Not able to find image for service "{service}", please make sure service name spelled correctly') + + +def set_image(instance_id, image_name, service='ckan', container_name=None): + cont_name = container_name or service + deployment_info = kubectl.call( + f'set image deployment/{service} {cont_name}={image_name}', + namespace=instance_id + ) + + def create_ckan_admin_user(instance_id, instance_type, instance, user): _get_deployment_provider(instance_type).create_ckan_admin_user(instance_id, instance, user) diff --git a/ckan_cloud_operator/providers/ckan/env/manager.py b/ckan_cloud_operator/providers/ckan/env/manager.py index 1a1d18b5..3ae8e8d2 100644 --- a/ckan_cloud_operator/providers/ckan/env/manager.py +++ b/ckan_cloud_operator/providers/ckan/env/manager.py @@ -163,6 +163,12 @@ def _write_yaml(data): _file.close() +def _read_yaml(): + with open(_get_config_file_name()) as _file: + yaml_conf = yaml.load(_file, Loader=yaml.FullLoader) + return yaml_conf + + def _mkconfdir(): dir = _get_config_dir() Path(f'{dir}').mkdir(parents=True, exist_ok=True)