From f900b26e649c5a335d63c956b1a3f80f97f62b3e Mon Sep 17 00:00:00 2001 From: Irakli Mchedlishvili Date: Tue, 6 Jul 2021 10:04:00 +0400 Subject: [PATCH] Feature/ckan exec command (#148) * Add and remove syadmin * search index and other commands --- .../providers/ckan/instance/cli.py | 47 ++++++--- .../providers/ckan/instance/manager.py | 96 ++++++++++++++++++- 2 files changed, 127 insertions(+), 16 deletions(-) diff --git a/ckan_cloud_operator/providers/ckan/instance/cli.py b/ckan_cloud_operator/providers/ckan/instance/cli.py index c79ea847..9a7365a7 100644 --- a/ckan_cloud_operator/providers/ckan/instance/cli.py +++ b/ckan_cloud_operator/providers/ckan/instance/cli.py @@ -163,8 +163,10 @@ def ckan_logs(command): @instance.command('ckan-exec') +@click.argument('INSTANCE_ID') @click.option('--command', help='command to pass down to ckan CLI, without path to config file') -def ckan_exec(command): +@click.option('--use-paster', help='Use paster over ckan CLI (supported in ckan v2.9)', default=False) +def ckan_exec(instance_id, command, use_paster): ''' Executes ckan CLI commands @@ -175,7 +177,7 @@ def ckan_exec(command): cco ckan instance ckan-exec --command='jobs list' cco ckan instance ckan-exec --command='dataset show dataset-id' ''' - pass + manager.run_ckan_commands(instance_id, command) @instance.command('ssh') @@ -209,24 +211,31 @@ def sysadmin(): @sysadmin.command('add') -@click.argument('USERNAME') -@click.option('--password', help='Passowrd for user if user does not exist') +@click.argument('INSTANCE_ID') +@click.option('--username', required=True, help='User name for user if user does not exist') +@click.option('--password', help='Password for user if user does not exist') @click.option('--email', help='Valid Email address for user if user does not exist') -def sysadmin_add(username, password, email): +@click.option('--use-paster', help='Use paster over ckan CLI (supported in ckan v2.9)', default=False) +def sysadmin_add(instance_id, username, password, email, use_paster): ''' Creates or makes given user system administrator cco ckan instance sysadmin add USERNAME --pasword pasword --email email@email.com ''' + manager.create_ckan_admin_user(instance_id, username, password, email, use_paster) + @sysadmin.command('rm') -@click.argument('USERNAME') -def sysadmin_rm(username): +@click.argument('INSTANCE_ID') +@click.option('--username', required=True, help='Passowrd for user if user does not exist') +@click.option('--use-paster', help='Use paster over ckan CLI (supported in ckan v2.9)', default=False) +def sysadmin_rm(instance_id, username, use_paster): ''' Removes System administrator privileges from given user cco ckan instance sysadmin rm USERNAME ''' + manager.delete_ckan_admin_user(instance_id, username, use_paster) @click.group() @@ -238,35 +247,45 @@ def solr(): @solr.command('check') -def solr_check(): +@click.argument('INSTANCE_ID') +def solr_check(instance_id): ''' Check the search index ''' + manager.run_solr_commands(instance_id, 'check') @solr.command('clear') -def solr_clear(): +@click.argument('INSTANCE_ID') +def solr_clear(instance_id): ''' Clear the search index ''' + manager.run_solr_commands(instance_id, 'clear') @solr.command('rebuild') -def solr_rebuild(): +@click.argument('INSTANCE_ID') +def solr_rebuild(instance_id): ''' Rebuild search index ''' + manager.run_solr_commands(instance_id, 'rebuild') @solr.command('rebuild-fast') -def solr_rebuild_fast(): +@click.argument('INSTANCE_ID') +def solr_rebuild_fast(instance_id): ''' Reindex with multiprocessing ''' + manager.run_solr_commands(instance_id, 'check') @solr.command('show') -@click.option('--dataset', help='Dataset name to show index for') -def solr_show(dataset): +@click.argument('INSTANCE_ID') +@click.option('--dataset-id', help='Dataset name to show index for') +def solr_show(instance_id, dataset_id): ''' - show --dataset=dataset-id-or-name + show --dataset-id=dataset-id-or-name ''' + manager.run_solr_commands(instance_id, 'show', dataset_id=dataset_id) @click.group() def deployment(): diff --git a/ckan_cloud_operator/providers/ckan/instance/manager.py b/ckan_cloud_operator/providers/ckan/instance/manager.py index bd69d785..5d771eea 100644 --- a/ckan_cloud_operator/providers/ckan/instance/manager.py +++ b/ckan_cloud_operator/providers/ckan/instance/manager.py @@ -336,7 +336,7 @@ def set_storage(instance_id, instance_name, dry_run=False): kubectl.apply(resource) -def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, dry_run=False): +def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, dry_run=False, use_paster=False): if not email: default_root_domain = routers_manager.get_default_root_domain() email = f'{name}@{instance_id_or_name}.{default_root_domain}' @@ -349,7 +349,21 @@ def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, 'password': password } if not dry_run: - deployment_manager.create_ckan_admin_user(instance_id, instance_type, instance, user) + pod_name = _get_running_pod_name(instance_id) + name, password, email = [user[k] for k in ['name', 'password', 'email']] + logs.info(f'Creating CKAN admin user with {name} ({email}) on pod {pod_name}') + + if use_paster: + logs.subprocess_check_call( + f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini add {name} password={password} email={email}', + shell=True + ) + else: + logs.subprocess_check_call( + f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini sysadmin add {name} password={password} email={email}', + shell=True + ) + return { 'instance-id': instance_id, 'instance-type': instance_type, @@ -358,6 +372,84 @@ def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, } +def delete_ckan_admin_user(instance_id_or_name, name, dry_run=False, use_paster=False): + instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) + + if not dry_run: + pod_name = _get_running_pod_name(instance_id) + logs.info(f'Removing CKAN admin user {name} from sys-admins') + + if use_paster: + logs.subprocess_check_call( + f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini remove {name}', + shell=True + ) + else: + logs.subprocess_check_call( + f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini sysadmin remove {name}', + shell=True + ) + +def run_solr_commands(instance_id_or_name, command, dataset_id='', dry_run=False, use_paster=False): + instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) + + if not dry_run: + pod_name = _get_running_pod_name(instance_id) + logs.info(f'Running Search Index {command}') + if use_paster: + answer = logs.subprocess_check_output( + f'kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan -c /etc/ckan/production.ini search-index {command}', + shell=True + ) + for line in str(answer).replace('\\r', '\\n').split('\\n'): + if line: + logs.info(str(line)) + else: + answer = logs.subprocess_check_output( + f'kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini search-index {command} {dataset_id}', + shell=True + ) + for line in str(answer).replace('\\r', '\\n').split('\\n'): + if line: + logs.info(str(line)) + + +def run_ckan_commands(instance_id_or_name, command, dry_run=False, use_paster=False): + instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) + if not dry_run: + pod_name = _get_running_pod_name(instance_id) + logs.info(f'Running Search Index {command}') + if use_paster: + answer = logs.subprocess_check_output( + f'kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan -c /etc/ckan/production.ini {command}', + shell=True + ) + for line in str(answer).replace('\\r', '\\n').split('\\n'): + if line: + logs.info(str(line)) + else: + answer = logs.subprocess_check_output( + f'kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini {command}', + shell=True + ) + for line in str(answer).replace('\\r', '\\n').split('\\n'): + if line: + logs.info(str(line)) + + +def _get_running_pod_name(instance_id): + pod_name = None + while not pod_name: + try: + pod_name = kubectl.get_deployment_pod_name('ckan', instance_id, use_first_pod=True, required_phase='Running') + break + except Exception as e: + logs.warning('Failed to find running ckan pod', str(e)) + time.sleep(20) + return pod_name + + + def _get_instance_id_and_type(instance_id_or_name=None, instance_id=None, required=True): if instance_id: logs.debug(f'Getting instance type using instance_id', instance_id=instance_id)