From e37da297c2f5b1c0a92083de6c2631775758ead0 Mon Sep 17 00:00:00 2001 From: prnadmin Date: Fri, 5 Sep 2014 16:37:25 -0700 Subject: [PATCH 01/23] Choose the most recent inventory --- glacier/GlacierWrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier/GlacierWrapper.py b/glacier/GlacierWrapper.py index d7613e3..0418093 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/GlacierWrapper.py @@ -1615,7 +1615,7 @@ def inventory(self, vault_name, refresh): # in progress job. job_list = self.list_jobs(vault_name) inventory_done = False - for job in job_list: + for job in sorted(job_list, key=lambda x: x['CompletionDate'], reverse=True): if job['Action'] == "InventoryRetrieval": # As soon as a finished inventory job is found, we're done. From 2cd80fa641bfe855a2c65c1dbe03a650fc19d72a Mon Sep 17 00:00:00 2001 From: Adi Date: Sat, 3 Jan 2015 11:35:08 +0100 Subject: [PATCH 02/23] Update GlacierWrapper.py Adding eu-central-1 Frankfurt region --- glacier/GlacierWrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/glacier/GlacierWrapper.py b/glacier/GlacierWrapper.py index d7613e3..d436725 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/GlacierWrapper.py @@ -113,7 +113,7 @@ class GlacierWrapper(object): MAX_VAULT_NAME_LENGTH = 255 MAX_VAULT_DESCRIPTION_LENGTH = 1024 MAX_PARTS = 10000 - AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', + AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1','eu-central-1', 'eu-west-1', 'ap-northeast-1', 'ap-southeast-2') AVAILABLE_REGIONS_MESSAGE = """\ Invalid region. Available regions for Amazon Glacier are: @@ -121,6 +121,7 @@ class GlacierWrapper(object): us-west-1 (US - N. California) us-west-2 (US - Oregon) eu-west-1 (EU - Ireland) +eu-central-1 (EU - Frankfurt) ap-northeast-1 (Asia-Pacific - Tokyo) ap-southeast-2 (Asia-Pacific - Sydney)\ """ From 1b93f1e189a42e8ad279808810de2c839d2d0cb5 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 11 Feb 2015 05:15:58 -0800 Subject: [PATCH 03/23] Emit archive ID when uploading a single file Previously glacier-cmd truncated output with headers over 100 characters which elided the 138 character archive ID. Fixes #163. --- glacier/glacier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index 5bd5abe..358ce73 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -34,7 +34,7 @@ def output_headers(headers, output): if output == 'print': table = PrettyTable(["Header", "Value"]) for row in rows: - if len(str(row[1])) < 100: + if len(str(row[1])) <= 138: table.add_row(row) print table From 4115be900ce10008538e9bbc7bf062c30daaddfd Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 11 Feb 2015 15:10:17 -0800 Subject: [PATCH 04/23] Do not emit None message Previously "glacier-cmd download" emitted a trailing None message: $ glacier-cmd download VAULTNAME zwvjqvd25vxcYLuaGOUhTMi8GW_PQ9M7-a9-Xpr6OX38WhABMBrG4U3w21vO08sgYFNrjRBgNuYiomc7mEkiaojIbkoETF6X1qnSDl-AR-ORzlgjIVud6zWM2R8yjuhg9g5q_M0LVg --outfile out Wrote 4.0 bytes. Rate 4.00 bytes/s. Rate 1.22 MB/s, average 4.00 bytes/s, ETA 15:08:53. None --- glacier/glacier.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index 5bd5abe..64d0f8b 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -96,15 +96,16 @@ def output_msg(msg, output, success=True): :param success: whether the operation was a success or not. :type success: boolean """ - if output == 'print': - print msg - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - csvwriter.writerow(msg) - - if output == 'json': - print json.dumps(msg) + if msg is not None: + if output == 'print': + print msg + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + csvwriter.writerow(msg) + + if output == 'json': + print json.dumps(msg) if not success: sys.exit(125) From 33d6e9e96db2abac287682ecdc375ca3a3a1eab6 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Thu, 9 Apr 2015 23:54:02 -0500 Subject: [PATCH 05/23] Handle uploading data from FIFO pipes. - FIFO pipes don't handle random access and behave more like stdin. --- README.md | 2 +- glacier/GlacierWrapper.py | 26 ++++++++++++++++++++++---- glacier/glacier.py | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3722da8..c90f797 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ To upload from stdin: $ cat file | glacier-cmd upload Test --description "Some description" --stdin --name /path/BetterName -IMPORTANT NOTE: If you're uploading from stdin, and you don't specify a +IMPORTANT NOTE: If you're uploading from stdin or a pipe, and you don't specify a --partsize option, your upload will be limited to 1.3Tb, and the progress report will come out every 128Mb. For more details, run: diff --git a/glacier/GlacierWrapper.py b/glacier/GlacierWrapper.py index d7613e3..ceae903 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/GlacierWrapper.py @@ -12,6 +12,7 @@ import re import logging import os.path +import stat import time import sys import traceback @@ -985,7 +986,13 @@ def upload(self, vault_name, file_name, description, region, total_size = 0 reader = None mmapped_file = None - if not stdin: + + if stdin: + is_pipe = False + else: + is_pipe = stat.S_ISFIFO(os.stat(file_name).st_mode) + + if not stdin and not is_pipe: if not file_name: raise InputException( "No file name given for upload.", @@ -1001,6 +1008,15 @@ def upload(self, vault_name, file_name, description, region, cause=e, code='FileError') + elif is_pipe: + try: + reader = open(file_name, 'rb') + total_size = 0 + except IOError: + raise InputException( + "Could not access pipe: %s."% file_name, + cause = e, code = 'FileError') + elif select.select([sys.stdin,],[],[],0.0)[0]: reader = sys.stdin total_size = 0 @@ -1069,9 +1085,9 @@ def upload(self, vault_name, file_name, description, region, start, stop = (int(p) for p in part['RangeInBytes'].split('-')) stop += 1 if not start == current_position: - if stdin: + if stdin or is_pipe: raise InputException( - 'Cannot verify non-sequential upload data from stdin.', + 'Cannot verify non-sequential upload data from stdin or pipe.', code='ResumeError') if reader: reader.seek(start) @@ -1195,7 +1211,9 @@ def upload(self, vault_name, file_name, description, region, self.logger.debug(msg) writer.close() - if not stdin: + if is_pipe: + reader.close() + elif not stdin: f.close() current_time = time.time() overall_rate = int(writer.uploaded_size/(current_time - start_time)) diff --git a/glacier/glacier.py b/glacier/glacier.py index 5bd5abe..af8a3c8 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -716,7 +716,7 @@ def main(): If not given, the smallest possible part size will be used when uploading a file, and 128 MB -when uploading from stdin.''') +when uploading from stdin or from a FIFO pipe.''') parser_upload.add_argument('--description', default=None, help='''\ Description of the file to be uploaded. Use quotes From ccf179e8eebdb5c387aeece4b2d6c135475d7c90 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Fri, 10 Apr 2015 22:14:48 -0500 Subject: [PATCH 06/23] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c90f797..8a09f40 100644 --- a/README.md +++ b/README.md @@ -306,7 +306,7 @@ Short Notification Service (SNS) is Amazon's technology that allows you to be no If you run `glacier-cmd sns sync` without specifing anything in your configuration file, it will automatically subscribe all your vaults to `aws-glacier-notifications` topic. - $ glacier.py sns sync + $ glacier-cmd sns sync +------------+-------------------------------------------------+ | Vault Name | Request Id | +------------+-------------------------------------------------+ From 04f756eb1258766b2e6994655c3bfc023d17f064 Mon Sep 17 00:00:00 2001 From: Tom Hummel Date: Wed, 26 Mar 2014 13:53:37 -0700 Subject: [PATCH 07/23] added note to --output flag usage re: print mode removing long output lines --- glacier/glacier.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index af8a3c8..95c6034 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -626,7 +626,11 @@ def main(): required=False, default=default('output') if default('output') else 'print', choices=['print', 'csv', 'json'], - help='Set how to return results: print to the screen, or as csv resp. json string.') + help='Set how to return results: print to the screen,\ + or as csv resp. json string. NOTE: to receive full\ + output use csv or json. `print` removes lines\ + longer than 99 chars, such as `archiveId`\ + on successful upload') # SimpleDB settings group = parser.add_argument_group('sdb') From 23dc0c08751874dbc09d071a9ff962c473c5d4c6 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sat, 11 Apr 2015 21:39:13 -0500 Subject: [PATCH 08/23] Retry on 408 HTTP responses --- glacier/glaciercorecalls.py | 102 ++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index 9b73802..b7eb686 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -121,51 +121,63 @@ def write(self, data): "x-amz-content-sha256": hashlib.sha256(data).hexdigest() } - response = self.connection.upload_part(self.vault_name, - self.uploadid, - hashlib.sha256(data).hexdigest(), - bytes_to_hex(part_tree_hash), - (self.uploaded_size, self.uploaded_size+len(data)-1), - data) - response.read() - -## retries = 0 -## while True: -## response = self.connection.make_request( -## "PUT", -## self.upload_url, -## headers, -## data) -## -## # Success. -## if response.status == 204: -## break -## -## # Time-out recieved: sleep for 5 minutes and try again. -## # Do not try more than five times; after that it's over. -## elif response.status == 408: -## if retries >= 5: -## resp = json.loads(response.read()) -## raise ResonseException( -## resp['message'], -## cause='Timeout', -## code=resp['code']) -## -## if self.logger: -## logger.warning(resp['message']) -## logger.warning('sleeping 300 seconds (5 minutes) before retrying.') -## -## retries += 1 -## time.sleep(300) -## -## else: -## raise ResponseException( -## "Multipart upload part expected response status 204 (got %s):\n%s"\ -## % (response.status, response.read()), -## cause=resp['message'], -## code=resp['code']) - -## response.read() + retries = 0 + while True: + try: + response = self.connection.upload_part(self.vault_name, + self.uploadid, + hashlib.sha256(data).hexdigest(), + bytes_to_hex(part_tree_hash), + (self.uploaded_size, self.uploaded_size+len(data)-1), + data) + break + + except Exception as e: + if '408' in e.message: + if retries >= 5: + raise e + + if self.logger: + self.logger.warning(e.message) + self.logger.warning('sleeping 300 seconds (5 minutes) before retrying.') + retries += 1 + time.sleep(300) + else: + raise e + + response.read() + + + # response.read() + + # # Success. + # if response.status == 204: + # break + + # # Time-out recieved: sleep for 5 minutes and try again. + # # Do not try more than five times; after that it's over. + # elif response.status == 408: + # if retries >= 5: + # resp = json.loads(response.read()) + # raise ResponseException( + # resp['message'], + # cause = 'Timeout', + # code = resp['code']) + + # if self.logger: + # logger.warning(resp['message']) + # logger.warning('sleeping 300 seconds (5 minutes) before retrying.') + + # retries += 1 + # time.sleep(300) + + # else: + # raise ResponseException( + # "Multipart upload part expected response status 204 (got %s):\n%s"\ + # % (response.status, response.read()), + # cause = resp['message'], + # code = resp['code']) + self.uploaded_size += len(data) def close(self): From 85ef4aa6dd58a36a77e4406366ddc27c7813cf5e Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sun, 10 May 2015 00:36:24 -0500 Subject: [PATCH 09/23] Reset retry count after a successful upload. --- README.md | 3 +-- glacier/glacier.py | 3 +-- glacier/glaciercorecalls.py | 9 ++++++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8a09f40..e478892 100644 --- a/README.md +++ b/README.md @@ -287,8 +287,7 @@ Usage description(help): --no-bookkeeping If present, overrides either CLI or configuration file options provided for bookkeeping either beforehand or afterwards - --logfile LOGFILE File to write log messages to. (default: /home/wouter - /.glacier-cmd.log) + --logfile LOGFILE File to write log messages to. (default: ~/.glacier-cmd.log) --loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL} Set the lowest level of messages you want to log. (default: DEBUG) diff --git a/glacier/glacier.py b/glacier/glacier.py index 8fde09d..3bfd982 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -630,8 +630,7 @@ def main(): help='Set how to return results: print to the screen,\ or as csv resp. json string. NOTE: to receive full\ output use csv or json. `print` removes lines\ - longer than 99 chars, such as `archiveId`\ - on successful upload') + longer than 138 chars') # SimpleDB settings group = parser.add_argument_group('sdb') diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index b7eb686..a5aa71e 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -122,6 +122,7 @@ def write(self, data): } retries = 0 + total_retries = 0 while True: try: response = self.connection.upload_part(self.vault_name, @@ -130,6 +131,7 @@ def write(self, data): bytes_to_hex(part_tree_hash), (self.uploaded_size, self.uploaded_size+len(data)-1), data) + retries = 0 break except Exception as e: @@ -137,10 +139,15 @@ def write(self, data): if retries >= 5: raise e + if total_retries >= 200: + raise e + if self.logger: self.logger.warning(e.message) - self.logger.warning('sleeping 300 seconds (5 minutes) before retrying.') + self.logger.warning('Sleeping 300 seconds (5 minutes) before retrying.') + self.logger.warning('Current retry = %d, total retries = %d' % (retries, total_retries)) retries += 1 + total_retries += 1 time.sleep(300) else: raise e From ee48ed3593d8701f273179727c876f463ce06462 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sun, 31 May 2015 15:10:11 -0500 Subject: [PATCH 10/23] Fixed http retry error --- glacier/glaciercorecalls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index a5aa71e..2bb09bc 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -131,6 +131,7 @@ def write(self, data): bytes_to_hex(part_tree_hash), (self.uploaded_size, self.uploaded_size+len(data)-1), data) + response.read() retries = 0 break @@ -152,7 +153,6 @@ def write(self, data): else: raise e - response.read() # response.read() From 00bdc3940ee8422cad27dff489fd947dac1420db Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sat, 13 Jun 2015 22:19:56 -0500 Subject: [PATCH 11/23] Improved failure logging for HTTP 408. --- glacier/glaciercorecalls.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index 2bb09bc..52f875e 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -73,6 +73,7 @@ class GlacierWriter(object): Archive. The data is written using the multi-part upload API. """ DEFAULT_PART_SIZE = 128 # in MB, power of 2. + MAX_TOTAL_RETRIES = 200 def __init__(self, connection, vault_name, description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, @@ -83,6 +84,7 @@ def __init__(self, connection, vault_name, self.connection = connection ## self.location = None self.logger = logger + self.total_retries = 0 if uploadid: self.uploadid = uploadid @@ -122,7 +124,6 @@ def write(self, data): } retries = 0 - total_retries = 0 while True: try: response = self.connection.upload_part(self.vault_name, @@ -132,23 +133,25 @@ def write(self, data): (self.uploaded_size, self.uploaded_size+len(data)-1), data) response.read() - retries = 0 break except Exception as e: if '408' in e.message: if retries >= 5: + self.logger.warning('Retries exhausted.') raise e - if total_retries >= 200: + if self.total_retries >= 200: + self.logger.warning('Total retries exhausted.') raise e if self.logger: self.logger.warning(e.message) self.logger.warning('Sleeping 300 seconds (5 minutes) before retrying.') - self.logger.warning('Current retry = %d, total retries = %d' % (retries, total_retries)) + self.logger.warning('Uploaded size = %d, hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) + self.logger.warning('Current retry = %d, total retries = %d' % (retries, self.total_retries)) retries += 1 - total_retries += 1 + self.total_retries += 1 time.sleep(300) else: raise e From 27d9f9b00dc0473fef80fff2e26223b8666eba66 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sat, 29 Aug 2015 22:55:51 -0500 Subject: [PATCH 12/23] Fixed possible error if logger is None - Better logging --- glacier/glaciercorecalls.py | 97 +++++++++++++++---------------------- 1 file changed, 39 insertions(+), 58 deletions(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index 52f875e..e929503 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -1,18 +1,18 @@ #!/usr/bin/env python # encoding: utf-8 """ -.. module:: botocorecalls - :platform: Unix, Windows - :synopsis: boto calls to access Amazon Glacier. - +.. module:: botocorecalls + :platform: Unix, Windows + :synopsis: boto calls to access Amazon Glacier. + This depends on the boto library, use version 2.6.0 or newer. - - writer = GlacierWriter(glacierconn, GLACIER_VAULT) - writer.write(block of data) - writer.close() - # Get the id of the newly created archive - archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection + + writer = GlacierWriter(glacierconn, GLACIER_VAULT) + writer.write(block of data) + writer.close() + # Get the id of the newly created archive + archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection """ import urllib @@ -30,7 +30,7 @@ class GlacierConnection(boto.glacier.layer1.Layer1): pass - + def chunk_hashes(data): """ @@ -74,7 +74,7 @@ class GlacierWriter(object): """ DEFAULT_PART_SIZE = 128 # in MB, power of 2. MAX_TOTAL_RETRIES = 200 - + def __init__(self, connection, vault_name, description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, uploadid=None, logger=None): @@ -100,7 +100,7 @@ def __init__(self, connection, vault_name, ## self.upload_url = response.getheader("location") def write(self, data): - + if self.closed: raise CommunicationError( "Tried to write to a GlacierWriter that is already closed.", @@ -110,7 +110,7 @@ def write(self, data): raise InputException ( 'Block of data provided must be equal to or smaller than the set block size.', code='InternalError') - + part_tree_hash = tree_hash(chunk_hashes(data)) self.tree_hashes.append(part_tree_hash) headers = { @@ -123,7 +123,11 @@ def write(self, data): "x-amz-content-sha256": hashlib.sha256(data).hexdigest() } + # How many times we tried uploading this block retries = 0 + # How much to sleep between re-tries. + sleep_time = 150 + while True: try: response = self.connection.upload_part(self.vault_name, @@ -137,64 +141,41 @@ def write(self, data): except Exception as e: if '408' in e.message: - if retries >= 5: - self.logger.warning('Retries exhausted.') + if retries >= 10: + if self.logger: + self.logger.warning('Retries exhausted for this block.') raise e - if self.total_retries >= 200: - self.logger.warning('Total retries exhausted.') + if self.total_retries >= MAX_TOTAL_RETRIES: + if self.logger: + self.logger.warning('Total retries exhausted.') raise e + retries += 1 + self.total_retries += 1 + if self.logger: self.logger.warning(e.message) - self.logger.warning('Sleeping 300 seconds (5 minutes) before retrying.') - self.logger.warning('Uploaded size = %d, hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) - self.logger.warning('Current retry = %d, total retries = %d' % (retries, self.total_retries)) - retries += 1 - self.total_retries += 1 - time.sleep(300) - else: - raise e + if sys.version_info < (2, 7, 0): + self.logger.warning('Total uploaded size = %d, block hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) + else: + # Commify large numbers + self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) + self.logger.warning('Retries (this block, total) = %d, %d' % (retries, self.total_retries)) + self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (sleep_time, sleep_time / 60.0)) + time.sleep(sleep_time) - - # response.read() - - # # Success. - # if response.status == 204: - # break - - # # Time-out recieved: sleep for 5 minutes and try again. - # # Do not try more than five times; after that it's over. - # elif response.status == 408: - # if retries >= 5: - # resp = json.loads(response.read()) - # raise ResponseException( - # resp['message'], - # cause = 'Timeout', - # code = resp['code']) - - # if self.logger: - # logger.warning(resp['message']) - # logger.warning('sleeping 300 seconds (5 minutes) before retrying.') - - # retries += 1 - # time.sleep(300) - - # else: - # raise ResponseException( - # "Multipart upload part expected response status 204 (got %s):\n%s"\ - # % (response.status, response.read()), - # cause = resp['message'], - # code = resp['code']) + else: + raise e self.uploaded_size += len(data) def close(self): - + if self.closed: return - + # Complete the multiplart glacier upload response = self.connection.complete_multipart_upload(self.vault_name, self.uploadid, From 1deb408d6512ae61b9cffbce3fa845301045351c Mon Sep 17 00:00:00 2001 From: Gideon Goldberg Date: Fri, 18 Sep 2015 15:33:11 +0100 Subject: [PATCH 13/23] example download using vault-name and archive-id --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3722da8..c7dfd9e 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,8 @@ its file name, its description, or limit the search by region and vault. If that is not enough you should use `getarchive` and specify the archive ID of the archive you want to retrieve: - $ TODO: example here + $ glacier-cmd download Test eBLl4DbMbZ4YMA7fD9cNacf2z1kGxpYxBqTV4qFVsuzgjuNlKSkWm2rFpw6Gq-bFT6Vt9cUZ1lGqSbtZjtbeh0jYn9tJC-MczQyA3tP6bezYUeN8dGGvqNqT3la79wjRRair1am1JA --outfile filename + Read 1 GB of 10 GB (10%). Rate 3.05 MB/s, average 2.71 MB/s, ETA 1:00:00. To remove uploaded archive use `rmarchive`. You can currently delete only by archive id (notice the use of `--` when the archive ID starts with a dash): From 2c0a07f460e420f9245aef0e3858fcb80ba02f4c Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Fri, 18 Sep 2015 19:57:01 -0500 Subject: [PATCH 14/23] Fixed variable usage - Added more details to some of the results. --- glacier/glacier.py | 6 ++++-- glacier/glaciercorecalls.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index 3bfd982..39b0bf8 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -310,7 +310,8 @@ def upload(args): args.name, args.partsize, args.uploadid, args.resume) results.append({"Uploaded file": g, "Created archive with ID": response[0], - "Archive SHA256 tree hash": response[1]}) + "Archive SHA256 tree hash": response[1], + "Description": args.description}) else: raise InputException( "File name given for upload can not be found: %s."% f, @@ -322,7 +323,8 @@ def upload(args): response = glacier.upload(args.vault, None, args.description, args.region, args.stdin, args.name, args.partsize, args.uploadid, args.resume) results = [{"Created archive with ID": response[0], - "Archive SHA256 tree hash": response[1]}] + "Archive SHA256 tree hash": response[1], + "Description": args.description}] else: raise InputException( diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index e929503..07eee29 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -146,7 +146,7 @@ def write(self, data): self.logger.warning('Retries exhausted for this block.') raise e - if self.total_retries >= MAX_TOTAL_RETRIES: + if self.total_retries >= self.MAX_TOTAL_RETRIES: if self.logger: self.logger.warning('Total retries exhausted.') raise e From d04ab69ca78f29e4b57a7567f2fcac9f17439073 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Mon, 28 Sep 2015 21:46:46 -0500 Subject: [PATCH 15/23] Retry on server errors to handle AWS outages. --- glacier/glaciercorecalls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index 07eee29..e30575f 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -126,7 +126,7 @@ def write(self, data): # How many times we tried uploading this block retries = 0 # How much to sleep between re-tries. - sleep_time = 150 + sleep_time = 300 while True: try: @@ -140,7 +140,7 @@ def write(self, data): break except Exception as e: - if '408' in e.message: + if '408' in e.message or e.code == "ServiceUnavailableException" or e.type == "Server": if retries >= 10: if self.logger: self.logger.warning('Retries exhausted for this block.') @@ -161,12 +161,16 @@ def write(self, data): else: # Commify large numbers self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) + self.logger.warning('Retries (this block, total) = %d, %d' % (retries, self.total_retries)) + self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (sleep_time, sleep_time / 60.0)) time.sleep(sleep_time) else: + self.logger.warning(e.message) + self.logger.warning('Not re-trying on this error') raise e self.uploaded_size += len(data) From 19aef87bfe30094e75312043c44f41d66732ebd8 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Sat, 3 Jun 2017 10:32:32 -0500 Subject: [PATCH 16/23] Exponential back-off when retrying - The further along we are in the backup, the harder we try to complete the upload. A large backup will thus allow for more failures and retries before being aborted. --- glacier/glacier.py | 2 +- glacier/glaciercorecalls.py | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index 39b0bf8..a615c9f 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -207,7 +207,7 @@ def describevault(args): 'SizeInBytes': "Size", 'VaultARN': "ARN", 'CreationDate': "Created"} - output_headers(response, args.output) + output_table([response], args.output, keys=headers) @handle_errors def listmultiparts(args): diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index e30575f..c9f2b4c 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -73,7 +73,19 @@ class GlacierWriter(object): Archive. The data is written using the multi-part upload API. """ DEFAULT_PART_SIZE = 128 # in MB, power of 2. - MAX_TOTAL_RETRIES = 200 + # After every failed block upload we sleep (SLEEP_TIME * retries) seconds. + # The more retries we've made for one particular block, the longer we sleep + # before re-attempting to re-upload that block. + SLEEP_TIME = 300 + # How many retries we should make to upload a particular block. We will not + # give up unless we've made at LEAST this many attempts to upload a block. + BLOCK_RETRIES = 10 + # How many retries we should allow for the whole upload. We will not give up + # unless we've made at LEAST this many attempts to upload the archive. + TOTAL_RETRIES = 100 + # For large files, the limits above could be surpassed. We also set a per-Gb + # criteria that allows more errors for larger uploads. + MAX_TOTAL_RETRY_PER_GB = 2 def __init__(self, connection, vault_name, description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, @@ -125,8 +137,6 @@ def write(self, data): # How many times we tried uploading this block retries = 0 - # How much to sleep between re-tries. - sleep_time = 300 while True: try: @@ -141,12 +151,17 @@ def write(self, data): except Exception as e: if '408' in e.message or e.code == "ServiceUnavailableException" or e.type == "Server": - if retries >= 10: + uploaded_gb = self.uploaded_size / (1024 * 1024 * 1024) + if retries >= self.BLOCK_RETRIES and retries > math.log10(uploaded_gb) * 10: if self.logger: self.logger.warning('Retries exhausted for this block.') raise e - if self.total_retries >= self.MAX_TOTAL_RETRIES: + if uploaded_gb > 0: + retry_per_gb = self.total_retries / uploaded_gb + else: + retry_per_gb = 0 + if self.total_retries >= self.TOTAL_RETRIES and retry_per_gb > self.MAX_TOTAL_RETRY_PER_GB: if self.logger: self.logger.warning('Total retries exhausted.') raise e @@ -162,11 +177,11 @@ def write(self, data): # Commify large numbers self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) - self.logger.warning('Retries (this block, total) = %d, %d' % (retries, self.total_retries)) + self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, self.BLOCK_RETRIES, self.total_retries, self.TOTAL_RETRIES)) self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') - self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (sleep_time, sleep_time / 60.0)) + self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (self.SLEEP_TIME, self.SLEEP_TIME / 60.0)) - time.sleep(sleep_time) + time.sleep(self.SLEEP_TIME * retries) else: self.logger.warning(e.message) From 91da994d63ef48fe7b1c55b535d859bc85f6b280 Mon Sep 17 00:00:00 2001 From: marzona Date: Sat, 10 Jun 2017 10:14:07 +0100 Subject: [PATCH 17/23] - added constants.py file and started to move there all the constants - removed %s and replaced with .format() as usage of %s si deprecated --- glacier/constants.py | 68 +++++++++ glacier/glacier.py | 320 ++++++++++++++++++++++++++----------------- 2 files changed, 264 insertions(+), 124 deletions(-) create mode 100644 glacier/constants.py diff --git a/glacier/constants.py b/glacier/constants.py new file mode 100644 index 0000000..5fee23f --- /dev/null +++ b/glacier/constants.py @@ -0,0 +1,68 @@ + +# constants definition +CHUNK_SIZE = 1024 +DEFAULT_PART_SIZE = 128 # in MB, power of 2. +# After every failed block upload we sleep (SLEEP_TIME * retries) seconds. +# The more retries we've made for one particular block, the longer we sleep +# before re-attempting to re-upload that block. +SLEEP_TIME = 300 +# How many retries we should make to upload a particular block. We will not +# give up unless we've made at LEAST this many attempts to upload a block. +BLOCK_RETRIES = 10 +# How many retries we should allow for the whole upload. We will not give up +# unless we've made at LEAST this many attempts to upload the archive. +TOTAL_RETRIES = 100 +# For large files, the limits above could be surpassed. We also set a per-Gb +# criteria that allows more errors for larger uploads. +MAX_TOTAL_RETRY_PER_GB = 2 +HEADERS_OUTPUT_FORMAT = ["csv","json"] +TABLE_OUTPUT_FORMAT = ["csv","json", "print"] +SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" +USER_CONFIG_FILENAME = "~/.glacier-cmd" +HELP_MESSAGE_CONFIG = u"(Required if you have not created a " + "~/.glacier-cmd or /etc/glacier-cmd.conf config file)" +ERRORCODE = {'InternalError': 127, # Library internal error. + 'UndefinedErrorCode': 126, # Undefined code. + 'NoResults': 125, # Operation yielded no results. + 'GlacierConnectionError': 1, # Can not connect to Glacier. + 'SdbConnectionError': 2, # Can not connect to SimpleDB. + 'CommandError': 3, # Command line is invalid. + 'VaultNameError': 4, # Invalid vault name. + 'DescriptionError': 5, # Invalid archive description. + 'IdError': 6, # Invalid upload/archive/job ID given. + 'RegionError': 7, # Invalid region given. + 'FileError': 8, # Error related to reading/writing a file. + 'ResumeError': 9, # Problem resuming a multipart upload. + 'NotReady': 10, # Requested download is not ready yet. + 'BookkeepingError': 11, # Bookkeeping not available. + 'SdbCommunicationError': 12, # Problem reading/writing SimpleDB data. + 'ResourceNotFoundException': 13, # Glacier can not find the requested resource. + 'InvalidParameterValueException': 14, # Parameter not accepted. + 'DownloadError': 15, # Downloading an archive failed. + 'SNSConnectionError': 126, # Can not connect to SNS + 'SNSConfigurationError': 127, # Problem with configuration file + 'SNSParameterError':128, # Problem with arguments passed to SNS + } +VAULT_NAME_ALLOWED_CHARACTERS = "[a-zA-Z\.\-\_0-9]+" +ID_ALLOWED_CHARACTERS = "[a-zA-Z\-\_0-9]+" +MAX_VAULT_NAME_LENGTH = 255 +MAX_VAULT_DESCRIPTION_LENGTH = 1024 +MAX_PARTS = 10000 +AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', + 'eu-west-1', 'eu-central-1', 'sa-east-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') +AVAILABLE_REGIONS_MESSAGE = """\ + Invalid region. Available regions for Amazon Glacier are: + us-east-1 (US - Virginia) + us-west-1 (US - N. California) + us-west-2 (US - Oregon) + eu-west-1 (EU - Ireland) + eu-central-1 (EU - Frankfurt) + sa-east-1 (South America - Sao Paulo) + ap-northeast-1 (Asia-Pacific - Tokyo) + ap-southeast-1 (Asia Pacific (Singapore) + ap-southeast-2 (Asia-Pacific - Sydney)\ + """ + + +UPLOAD_DATA_ERROR_MSG = "Received data does not match uploaded data; please check your uploadid and try again." diff --git a/glacier/glacier.py b/glacier/glacier.py index a615c9f..d55c97a 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -16,12 +16,12 @@ import csv import json -from prettytable import PrettyTable +import glacierexception +import constants +from prettytable import PrettyTable from GlacierWrapper import GlacierWrapper - from functools import wraps -from glacierexception import * def output_headers(headers, output): """ @@ -36,21 +36,27 @@ def output_headers(headers, output): for row in rows: if len(str(row[1])) <= 138: table.add_row(row) - + print table - + + if output not in constants.HEADERS_OUTPUT_FORMAT: + raise ValueError("Output format must be {}, got" + ":{}".format(constants.HEADERS_OUTPUT_FORMAT, + output)) + if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) for row in rows: csvwriter.writerow(row) - + if output == 'json': print json.dumps(headers) def output_table(results, output, keys=None, sort_key=None): """ Prettyprints results. Expects a list of identical dicts. - Use the dict keys as headers unless keys is given; one line for each item. + Use the dict keys as headers unless keys is given; + one line for each item. Expected format of data is a list of dicts: [{'key1':'data1.1', 'key2':'data1.2', ... }, @@ -62,6 +68,10 @@ def output_table(results, output, keys=None, sort_key=None): sort_key: the key to use for sorting the table. """ + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) if output == 'print': if len(results) == 0: print 'No output!' @@ -74,16 +84,16 @@ def output_table(results, output, keys=None, sort_key=None): if sort_key: table.sortby = keys[sort_key] if keys else sort_key - + print table - + if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) keys = results[0].keys() csvwriter.writerow(keys) for row in results: csvwriter.writerow([row[k] for k in keys]) - + if output == 'json': print json.dumps(results) @@ -96,6 +106,12 @@ def output_msg(msg, output, success=True): :param success: whether the operation was a success or not. :type success: boolean """ + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be{}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + if msg is not None: if output == 'print': print msg @@ -106,22 +122,23 @@ def output_msg(msg, output, success=True): if output == 'json': print json.dumps(msg) - + if not success: sys.exit(125) def size_fmt(num, decimals = 1): """ - Formats file sizes in human readable format. Anything bigger than TB - is returned is TB. Number of decimals is optional, defaults to 1. + Formats file sizes in human readable format. Anything bigger than + TB is returned is TB. Number of decimals is optional, + defaults to 1. """ fmt = "%%3.%sf %%s"% decimals for x in ['bytes','KB','MB','GB']: if num < 1024.0: return fmt % (num, x) - + num /= 1024.0 - + return fmt % (num, 'TB') def default_glacier_wrapper(args, **kwargs): @@ -155,10 +172,10 @@ def handle_errors(fn): def wrapper(*args, **kwargs): try: return fn(*args, **kwargs) - except GlacierException as e: + except glacierexception.GlacierException as e: - # We are only interested in the error message in case it is a - # self-caused exception. + # We are only interested in the error message in case + # it is a self-caused exception. e.write(indentation='|| ', stack=False, message=True) sys.exit(e.exitcode) @@ -217,7 +234,8 @@ def listmultiparts(args): glacier = default_glacier_wrapper(args) response = glacier.listmultiparts(args.vault) if not response: - output_msg('No active multipart uploads.', args.output, success=False) + output_msg('No active multipart uploads.', args.output, + success=False) else: output_table(response, args.output) @@ -265,7 +283,8 @@ def download(args): """ glacier = default_glacier_wrapper(args) response = glacier.download(args.vault, args.archive, args.partsize, - out_file_name=args.outfile, overwrite=args.overwrite) + out_file_name=args.outfile, + overwrite=args.overwrite) if args.outfile: output_msg(response, args.output, success=True) @@ -279,9 +298,9 @@ def upload(args): # This is /path/to/vol001|vol002|vol003 if args.bacula: if len(args.filename) > 1: - raise InputException( - 'Bacula-style file name input can accept only one file name argument.') - + raise glacierexception.InputException("Bacula-style file name input can"\ + "accept only one file name argument.") + fileset = args.filename[0].split('|') if len(fileset) > 1: dirname = os.path.dirname(fileset[0]) @@ -298,7 +317,7 @@ def upload(args): # be read over stdin. if args.filename: for f in args.filename: - + # In case the shell does not expand wildcards, if any, do this here. if f[0] == '~': f = os.path.expanduser(f) @@ -306,34 +325,41 @@ def upload(args): globbed = glob.glob(f) if globbed: for g in globbed: - response = glacier.upload(args.vault, g, args.description, args.region, args.stdin, - args.name, args.partsize, args.uploadid, args.resume) + response = glacier.upload(args.vault, g, + args.description, + args.region, args.stdin, + args.name, args.partsize, + args.uploadid, + args.resume) results.append({"Uploaded file": g, "Created archive with ID": response[0], "Archive SHA256 tree hash": response[1], "Description": args.description}) else: - raise InputException( - "File name given for upload can not be found: %s."% f, + raise glacierexception.InputException( + "File name given for upload can not "\ + "be found: {}.".format(f), code='CommandError') - + elif args.stdin: # No file name; using stdin. - response = glacier.upload(args.vault, None, args.description, args.region, args.stdin, - args.name, args.partsize, args.uploadid, args.resume) + response = glacier.upload(args.vault, None, args.description, + args.region, args.stdin, + args.name, args.partsize, + args.uploadid, args.resume) results = [{"Created archive with ID": response[0], "Archive SHA256 tree hash": response[1], "Description": args.description}] else: - raise InputException( + raise glacierexception.InputException( '''No input given. Either give a file name or file names -on the command line, or use the --stdin switch and pipe -in the data over stdin.''', + on the command line, or use the --stdin switch and pipe + in the data over stdin.''', cause='No file name and no stdin pipe.', code='CommandError') - + output_table(results, args.output) if len(results) > 1 \ else output_headers(results[0], args.output) @@ -377,26 +403,30 @@ def inventory(args): if sys.stdout.isatty() and output == 'print': print 'Checking inventory, please wait.\r', sys.stdout.flush() - + job, inventory = glacier.inventory(args.vault, args.refresh) if inventory: if sys.stdout.isatty() and output == 'print': - print "Inventory of vault: %s" % (inventory["VaultARN"],) - print "Inventory Date: %s\n" % (inventory['InventoryDate'],) + print "Inventory of vault: {}".format(inventory["VaultARN"]) + print "Inventory Date: {}\n".format(['InventoryDate']) print "Content:" - + headers = {'ArchiveDescription': 'Archive Description', 'CreationDate': 'Uploaded', 'Size': 'Size', 'ArchiveId': 'Archive ID', 'SHA256TreeHash': 'SHA256 tree hash'} - output_table(inventory['ArchiveList'], args.output, keys=headers) + output_table(inventory['ArchiveList'], + args.output, + keys=headers) if sys.stdout.isatty() and output == 'print': size = 0 for item in inventory['ArchiveList']: size += int(item['Size']) - print 'This vault contains %s items, total size %s.'% (len(inventory['ArchiveList']), size_fmt(size)) + print "This vault contains {} items, total size "\ + "{}.".format(len(inventory['ArchiveList']), + size_fmt(size)) else: result = {'Status':'Inventory retrieval in progress.', @@ -414,10 +444,11 @@ def treehash(args): for f in args.filename: if f: - # In case the shell does not expand wildcards, if any, do this here. + # In case the shell does not expand wildcards, + # if any, do this here. if f[0] == '~': f = os.path.expanduser(f) - + globbed = glob.glob(f) if globbed: for g in globbed: @@ -425,24 +456,26 @@ def treehash(args): {'File name': g, 'SHA256 tree hash': glacier.get_tree_hash(g)}) else: - raise InputException( - 'No file name given.', - code='CommandError') + raise glacierexception.InputException('No file name given.', + code='CommandError') output_table(hash_results, args.output) def snssync(args): """ - If monitored_vaults is specified in configuration file, subscribe vaults - specificed in it to notifications, otherwiser subscribe all vault. + If monitored_vaults is specified in configuration file, subscribe + vaults specificed in it to notifications, otherwiser + subscribe all vault. """ glacier = default_glacier_wrapper(args) - response = glacier.sns_sync(sns_options=args.sns_options, output=args.output) + response = glacier.sns_sync(sns_options=args.sns_options, + output=args.output) output_table(response, args.output) def snssubscribe(args): """ - Subscribe individual vaults to notifications by method specified by user. + Subscribe individual vaults to notifications by method + specified by user. """ protocol = args.protocol endpoint = args.endpoint @@ -450,7 +483,9 @@ def snssubscribe(args): topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_subscribe(protocol, endpoint, topic, vault_names=vault_names, sns_options=args.sns_options) + response = glacier.sns_subscribe(protocol, endpoint, topic, + vault_names=vault_names, + sns_options=args.sns_options) output_table(response, args.output) def snslistsubscriptions(args): @@ -462,7 +497,9 @@ def snslistsubscriptions(args): topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_list_subscriptions(protocol, endpoint, topic, sns_options=args.sns_options) + response = glacier.sns_list_subscriptions(protocol, endpoint, + topic, + sns_options=args.sns_options) output_table(response, args.output) def snslisttopics(args): @@ -472,16 +509,18 @@ def snslisttopics(args): def snsunsubscribe(args): """ - Unsubscribe individual vaults from notifications for specified protocol, - endpoint and vault. + Unsubscribe individual vaults from notifications for + specified protocol, endpoint and vault. """ protocol = args.protocol endpoint = args.endpoint topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_unsubscribe(protocol, endpoint, topic, sns_options=args.sns_options) - output_table(response, args.output) + response = glacier.sns_unsubscribe(protocol, endpoint, + topic, + sns_options=args.sns_options) + output_table(response, args.output) def main(): program_description = u""" @@ -490,25 +529,32 @@ def main(): # Config parser conf_parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - add_help=False) - - conf_parser.add_argument("-c", "--conf", default="~/.glacier-cmd", - help="Name of the file to log messages to.", metavar="FILE") - conf_parser.add_argument('--logtostdout', action='store_true', - help='Send log messages to stdout instead of the config file.') + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + add_help=False) + + conf_parser.add_argument("-c", + "--conf", + default="~/.glacier-cmd", + help="Name of the file to log messages to.", + metavar="FILE") + conf_parser.add_argument('--logtostdout', + action='store_true', + help="Send log messages "\ + "to stdout instead of "\ + "the config file.") args, remaining_argv = conf_parser.parse_known_args() - # Here we parse config from files in home folder or in current folder - # We use separate topics for aws and glacier specific configs + # Here we parse config from files in home folder or in current + # folder We use separate topics for aws and glacier + # specific configs aws = glacier = sdb = {} config = ConfigParser.SafeConfigParser() sns = {'topics_present':False, 'topic':'aws-glacier-notifications'} - configs_read = config.read(['/etc/glacier-cmd.conf', - os.path.expanduser('~/.glacier-cmd'), + configs_read = config.read([constants.SYSTEM_WIDE_CONFIG_FILENAME, + os.path.expanduser(constants.USER_CONFIG_FILENAME), args.conf]) if configs_read: try: @@ -522,7 +568,7 @@ def main(): try: sdb = dict(config.items("sdb")) for key,value in sdb.items(): - sdb["sdb_%s"%key]=value + sdb["sdb_{}".format(key)]=value del sdb[key] except ConfigParser.NoSectionError: pass @@ -539,7 +585,7 @@ def main(): 'options':dict(config.items(topic)) } sns_topics += [s] - + if sns_topics: sns['topics'] = sns_topics elif any(topic for topic in config.sections() if topic == "SNS"): @@ -572,23 +618,24 @@ def main(): # Main configuration parser parser = argparse.ArgumentParser(parents=[conf_parser], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=program_description) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=program_description) subparsers = parser.add_subparsers(title='Subcommands', help=u"For subcommand help, use: glacier-cmd -h") # Amazon Web Services settings group = parser.add_argument_group('aws') - help_msg_config = u"(Required if you have not created a \ - ~/.glacier-cmd or /etc/glacier-cmd.conf config file)" + group.add_argument('--aws-access-key', required=a_required("aws-access-key"), default=a_default("aws-access-key"), - help="Your aws access key " + help_msg_config) + help="Your aws access key "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--aws-secret-key', required=a_required("aws-secret-key"), default=a_default("aws-secret-key"), - help="Your aws secret key " + help_msg_config) + help="Your aws secret key "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) # Glacier settings group = parser.add_argument_group('glacier') @@ -596,25 +643,29 @@ def main(): required=required("region"), default=default("region"), help="Region where you want to store \ - your archives " + help_msg_config) + your archives "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) bookkeeping = True if default('bookkeeping') == 'True' else False group.add_argument('--bookkeeping', required=False, default=bookkeeping, action="store_true", - help="Should we keep book of all created archives.\ - This requires a Amazon SimpleDB account and its \ - bookkeeping domain name set") + help="Should we keep book of all created "\ + "archives. This requires a Amazon "\ + "SimpleDB account and its "\ + "bookkeeping domain name set") group.add_argument('--no-bookkeeping', required=False, default=False, action="store_true", - help="Explicitly disables bookkeeping, regardless of other\ - configuration or command line options.") + help="Explicitly disables bookkeeping, "\ + "regardless of other configuration "\ + "or command line options.") group.add_argument('--bookkeeping-domain-name', required=False, default=default("bookkeeping-domain-name"), - help="Amazon SimpleDB domain name for bookkeeping.") + help="Amazon SimpleDB domain name " + "for bookkeeping.") group.add_argument('--logfile', required=False, default=os.path.expanduser('~/.glacier-cmd.log'), @@ -629,26 +680,31 @@ def main(): required=False, default=default('output') if default('output') else 'print', choices=['print', 'csv', 'json'], - help='Set how to return results: print to the screen,\ - or as csv resp. json string. NOTE: to receive full\ - output use csv or json. `print` removes lines\ - longer than 138 chars') + help="Set how to return results: print to "\ + "the screen, or as csv resp. json string. "\ + "NOTE: to receive full output use csv or "\ + "json. `print` removes lines "\ + "longer than 138 chars") # SimpleDB settings group = parser.add_argument_group('sdb') group.add_argument('--sdb-access-key', required=False, - default=s_default("sdb-access-key") or a_default("aws-access-key"), - help="aws access key to be used with bookkeeping" + help_msg_config) + default=(s_default("sdb-access-key") or + a_default("aws-access-key")), + help="aws access key to be used with \ + bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--sdb-secret-key', required=False, - default=s_default("sdb-secret-key") or a_default("aws-secret-key"), - help="aws secret key to be used with bookkeeping" + help_msg_config) + default=(s_default("sdb-secret-key") or + a_default("aws-secret-key")), + help="aws secret key to be used with "\ + "bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--sdb-region', required=False, default=s_default("sdb-region") or default("region"), - help="Region where you want to store \ - your bookkeeping " + help_msg_config) + help="Region where you want to store "\ + "bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) # glacier-cmd mkvault parser_mkvault = subparsers.add_parser("mkvault", @@ -657,7 +713,7 @@ def main(): help='The vault to be created.') parser_mkvault.set_defaults(func=mkvault) - # glacier-cmd lsvault + # glacier-cmd lsvault parser_lsvault = subparsers.add_parser("lsvault", help="List available vaults.") parser_lsvault.set_defaults(func=lsvault) @@ -683,19 +739,18 @@ def main(): help='Upload an archive to Amazon Glacier.') parser_upload.add_argument('vault', help='The vault the archive is to be stored in.') -## group = parser_upload.add_mutually_exclusive_group(required=True) parser_upload.add_argument('filename', nargs='*', default=None, help='''\ The name(s) of the local file(s) to be uploaded. Wildcards are accepted. Can not be used if --stdin is used.''') parser_upload.add_argument('--stdin', action='store_true', help='''\ -Read data from stdin, instead of local file. +Read data from stdin, instead of local file. Can not be used if is given.''') parser_upload.add_argument('--name', default=None, help='''\ -Use the given name as the filename for bookkeeping -purposes. To be used in conjunction with --stdin or +Use the given name as the filename for bookkeeping +purposes. To be used in conjunction with --stdin or when the file being uploaded is a temporary file.''') parser_upload.add_argument('--partsize', type=int, default=-1, help='''\ @@ -764,13 +819,14 @@ def main(): # glacier-cmd inventory [--refresh] parser_inventory = subparsers.add_parser('inventory', - help='List inventory of a vault, if available. If not available, \ - creates inventory retrieval job if none running already.') + help="List inventory of a vault, if available. If not "\ + "available, creates inventory retrieval job if none "\ + "running already.") parser_inventory.add_argument('vault', help='The vault to list the inventory of.') parser_inventory.add_argument('--refresh', action='store_true', - help='Create an inventory retrieval job, even if inventory is \ - available or with another retrieval job running.') + help="Create an inventory retrieval job, even if inventory is "\ + "available or with another retrieval job running.") parser_inventory.set_defaults(func=inventory) # glacier-cmd getarchive @@ -866,47 +922,61 @@ def main(): # glacier-cmd hash parser_describejob = subparsers.add_parser('treehash', - help='Calculate the tree-hash (Amazon style sha256-hash) of a file.') + help="Calculate the tree-hash (Amazon style sha256-hash) "\ + "of a file.") parser_describejob.add_argument('filename', nargs='*', help='The filename to calculate the treehash of.') parser_describejob.set_defaults(func=treehash) - # SNS related commands are located in their own subparser - parser_sns = subparsers.add_parser('sns', + # SNS related commands are located in their own subparser + parser_sns = subparsers.add_parser('sns', help="Subcommands related to SNS") - sns_subparsers = parser_sns.add_subparsers(title="Subcommands related to SNS") + sns_subparsers = parser_sns.add_subparsers(title="Subcommands "\ + "related to SNS") # glacier-cmd sns syncs sns_parser_sync = sns_subparsers.add_parser('sync', - help="Go through configuration file and either subscribe all vaults to default topic or, if sections are present, create separate topics and subscribe specified vaults to that topic.") + help="Go through configuration file and either "\ + "subscribe all vaults to default topic or, "\ + "if sections are present, create separate "\ + "topics and subscribe specified vaults to that topic.") sns_parser_sync.set_defaults(func=snssync, sns_options=sns) # glacier-cmd sns subscribe protocol endpoint topic [--vault] sns_parser_subscribe = sns_subparsers.add_parser('subscribe', help="Subscribe to topic.") sns_parser_subscribe.add_argument("protocol", - help="Protocol used for notifications. Can be email, http, https or sms.") + help="Protocol used for notifications. Can be email, "\ + "http, https or sms.") sns_parser_subscribe.add_argument("endpoint", - help="Valid applicable endpoint - email address, URL or phone number.") - sns_parser_subscribe.add_argument("topic", - help="Topic for which notifications will be sent to specified protocol and endpoint.") + help="Valid applicable endpoint - email address, "\ + "URL or phone number.") + sns_parser_subscribe.add_argument("topic", + help="Topic for which notifications will be sent "\ + "to specified protocol and endpoint.") sns_parser_subscribe.add_argument("--vault", - help="Optional vault names, seperated by comma, for this a new topic will be created and subscribed to.") - sns_parser_subscribe.set_defaults(func=snssubscribe, sns_options={ "options":sns, }) + help="Optional vault names, seperated by comma, "\ + "for this a new topic will be created and subscribed to.") + sns_parser_subscribe.set_defaults(func=snssubscribe, + sns_options={ "options":sns, }) # glacier-cmd sns unsubscribe [--protocol ] [--endpoint ] [--topic ] sns_parser_unsubscribe = sns_subparsers.add_parser('unsubscribe', help="Unsubscribe from a specified topic.") sns_parser_unsubscribe.add_argument("--protocol", - help="Protocol used for notifications. Can be email, http, https or sms.") + help="Protocol used for notifications. Can be email, "\ + "http, https or sms.") sns_parser_unsubscribe.add_argument("--endpoint", - help="Valid applicable endpoint - email address, URL or phone number.") + help="Valid applicable endpoint - email address, "\ + "URL or phone number.") sns_parser_unsubscribe.add_argument("--topic", - help="Topic for which notifications will be sent to specified protocol and endpoint.") - sns_parser_unsubscribe.set_defaults(func=snsunsubscribe, sns_options=sns) + help="Topic for which notifications will be sent to "\ + "specified protocol and endpoint.") + sns_parser_unsubscribe.set_defaults(func=snsunsubscribe, + sns_options=sns) # glacier-cmd sns lssub [--protocol ] [--endpoint ] [--topic ] - sns_parser_listsubs = sns_subparsers.add_parser('lssub', + sns_parser_listsubs = sns_subparsers.add_parser('lssub', help="List subscriptions. Other arguments are ANDed together.") sns_parser_listsubs.add_argument("--protocol", help="Show only subscriptions on a specified protocol.") @@ -914,23 +984,25 @@ def main(): help="Show only subscriptions to a specified endpoint.") sns_parser_listsubs.add_argument("--topic", help="Show only subscriptions for a specified topic.") - sns_parser_listsubs.set_defaults(func=snslistsubscriptions, sns_options=sns) + sns_parser_listsubs.set_defaults(func=snslistsubscriptions, + sns_options=sns) # glacier-cmd sns lstopic sns_parser_listtopics = sns_subparsers.add_parser('lstopic', help="List all topics.") - sns_parser_listtopics.set_defaults(func=snslisttopics, sns_options=sns) - + sns_parser_listtopics.set_defaults(func=snslisttopics, + sns_options=sns) - # TODO args.logtostdout becomes false when parsing the remaining_argv - # so here we bridge this. An ugly hack but it works. + + # TODO args.logtostdout becomes false when parsing the + # remaining_argv so here we bridge this. An ugly hack but it works. logtostdout = args.logtostdout # Process the remaining arguments. args = parser.parse_args(remaining_argv) - + args.logtostdout = logtostdout - + # Run the subcommand. args.func(args) From 6788de86c0fcd9df3cd0454d786e8667960d46c2 Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Mon, 19 Jun 2017 23:53:23 -0500 Subject: [PATCH 18/23] Try to catch HTTP exception --- glacier/glaciercorecalls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index c9f2b4c..1cf1f9a 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -25,6 +25,7 @@ import boto.glacier.layer1 from glacierexception import * +from boto.glacier.exceptions import UnexpectedHTTPResponseError # Placeholder, effectively renaming the class. class GlacierConnection(boto.glacier.layer1.Layer1): @@ -150,7 +151,7 @@ def write(self, data): break except Exception as e: - if '408' in e.message or e.code == "ServiceUnavailableException" or e.type == "Server": + if '408' in e.message or e.code == "ServiceUnavailableException" or isinstance(e, UnexpectedHTTPResponseError): uploaded_gb = self.uploaded_size / (1024 * 1024 * 1024) if retries >= self.BLOCK_RETRIES and retries > math.log10(uploaded_gb) * 10: if self.logger: From d9b0a313f79bae197cdbf2cd7a9e5a81090e389c Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Tue, 20 Jun 2017 07:57:55 -0500 Subject: [PATCH 19/23] Fix unicode string concatenation --- glacier/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/glacier/constants.py b/glacier/constants.py index 5fee23f..e843108 100644 --- a/glacier/constants.py +++ b/glacier/constants.py @@ -19,8 +19,7 @@ TABLE_OUTPUT_FORMAT = ["csv","json", "print"] SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" USER_CONFIG_FILENAME = "~/.glacier-cmd" -HELP_MESSAGE_CONFIG = u"(Required if you have not created a " - "~/.glacier-cmd or /etc/glacier-cmd.conf config file)" +HELP_MESSAGE_CONFIG = u"(Required if you have not created a ~/.glacier-cmd or /etc/glacier-cmd.conf config file)" ERRORCODE = {'InternalError': 127, # Library internal error. 'UndefinedErrorCode': 126, # Undefined code. 'NoResults': 125, # Operation yielded no results. From f09bd44f6b7224b86c67ed578b126e7c2c83951f Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Wed, 21 Jun 2017 22:16:36 -0500 Subject: [PATCH 20/23] Fix reporting output format error --- glacier/constants.py | 3 +-- glacier/glacier.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/glacier/constants.py b/glacier/constants.py index e843108..c7a446a 100644 --- a/glacier/constants.py +++ b/glacier/constants.py @@ -15,8 +15,7 @@ # For large files, the limits above could be surpassed. We also set a per-Gb # criteria that allows more errors for larger uploads. MAX_TOTAL_RETRY_PER_GB = 2 -HEADERS_OUTPUT_FORMAT = ["csv","json"] -TABLE_OUTPUT_FORMAT = ["csv","json", "print"] +TABLE_OUTPUT_FORMAT = ["csv", "json", "print"] SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" USER_CONFIG_FILENAME = "~/.glacier-cmd" HELP_MESSAGE_CONFIG = u"(Required if you have not created a ~/.glacier-cmd or /etc/glacier-cmd.conf config file)" diff --git a/glacier/glacier.py b/glacier/glacier.py index d55c97a..1ccf396 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -31,6 +31,12 @@ def output_headers(headers, output): :type headers: dict """ rows = [(k, headers[k]) for k in headers.keys()] + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be {}, got" + ": {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + if output == 'print': table = PrettyTable(["Header", "Value"]) for row in rows: @@ -39,11 +45,6 @@ def output_headers(headers, output): print table - if output not in constants.HEADERS_OUTPUT_FORMAT: - raise ValueError("Output format must be {}, got" - ":{}".format(constants.HEADERS_OUTPUT_FORMAT, - output)) - if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) for row in rows: @@ -69,7 +70,7 @@ def output_table(results, output, keys=None, sort_key=None): """ if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " + raise ValueError("Output format must be {}, " "got {}".format(constants.TABLE_OUTPUT_FORMAT, output)) if output == 'print': @@ -108,7 +109,7 @@ def output_msg(msg, output, success=True): """ if output not in constants.TABLE_OUTPUT_FORMAT: - raise ValueError("Output format must be{}, " + raise ValueError("Output format must be {}, " "got {}".format(constants.TABLE_OUTPUT_FORMAT, output)) @@ -298,7 +299,7 @@ def upload(args): # This is /path/to/vol001|vol002|vol003 if args.bacula: if len(args.filename) > 1: - raise glacierexception.InputException("Bacula-style file name input can"\ + raise glacierexception.InputException("Bacula-style file name input can "\ "accept only one file name argument.") fileset = args.filename[0].split('|') @@ -679,7 +680,7 @@ def main(): group.add_argument('--output', required=False, default=default('output') if default('output') else 'print', - choices=['print', 'csv', 'json'], + choices=constants.TABLE_OUTPUT_FORMAT, help="Set how to return results: print to "\ "the screen, or as csv resp. json string. "\ "NOTE: to receive full output use csv or "\ From d5ddb78f8a8488eeaff9b91607c213ede486c32b Mon Sep 17 00:00:00 2001 From: Daniil Kupchenko Date: Thu, 21 Sep 2017 13:51:49 +0300 Subject: [PATCH 21/23] account-id support has been added --- glacier/GlacierWrapper.py | 17 ++++++++++++----- glacier/glacier.py | 5 +++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/glacier/GlacierWrapper.py b/glacier/GlacierWrapper.py index 7053533..0f6995a 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/GlacierWrapper.py @@ -220,14 +220,17 @@ def glacier_connect_wrap(*args, **kwargs): Connecting to Amazon Glacier with aws_access_key %s aws_secret_key %s - region %s\ + region %s + account_id %s\ """, self.aws_access_key, self.aws_secret_key, - self.region) + self.region, + self.account_id) self.glacierconn = GlacierConnection(self.aws_access_key, self.aws_secret_key, - region_name=self.region) + region_name=self.region, + account_id=self.account_id) except boto.exception.AWSConnectionError as e: raise ConnectionException( "Cannot connect to Amazon Glacier.", @@ -1956,7 +1959,7 @@ def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): return unsubscribed - def __init__(self, aws_access_key, aws_secret_key, region, + def __init__(self, aws_access_key, aws_secret_key, region, account_id='-', bookkeeping=False, no_bookkeeping=None, bookkeeping_domain_name=None, sdb_access_key=None, sdb_secret_key=None, sdb_region=None, logfile=None, loglevel='WARNING', logtostdout=True): @@ -1969,6 +1972,8 @@ def __init__(self, aws_access_key, aws_secret_key, region, :type aws_secret_key: str :param region: name of your default region, see :ref:`regions`. :type region: str + :param account_id: AWS account ID + :type account_id: str :param bookkeeping: whether to enable bookkeeping, see :reg:`bookkeeping`. :type bookkeeping: boolean :param bookkeeping_domain_name: your Amazon SimpleDB domain name where the bookkeeping information will be stored. @@ -1997,6 +2002,7 @@ def __init__(self, aws_access_key, aws_secret_key, region, self.bookkeeping_domain_name = bookkeeping_domain_name self.region = region + self.account_id = account_id self.sdb_access_key = sdb_access_key if sdb_access_key else aws_access_key self.sdb_secret_key = sdb_secret_key if sdb_secret_key else aws_secret_key @@ -2015,6 +2021,7 @@ def __init__(self, aws_access_key, aws_secret_key, region, nobookkeeping=%s, bookkeeping_domain_name=%s, region=%s, + account_id=%s, sdb_access_key=%s, sdb_secret_key=%s, sdb_region=%s, @@ -2023,6 +2030,6 @@ def __init__(self, aws_access_key, aws_secret_key, region, logging to stdout %s.""", aws_access_key, aws_secret_key, bookkeeping, no_bookkeeping, - bookkeeping_domain_name, region, + bookkeeping_domain_name, region, account_id, sdb_access_key, sdb_secret_key, sdb_region, logfile, loglevel, logtostdout) diff --git a/glacier/glacier.py b/glacier/glacier.py index 1ccf396..c0e184d 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -150,6 +150,7 @@ def default_glacier_wrapper(args, **kwargs): return GlacierWrapper(args.aws_access_key, args.aws_secret_key, args.region, + args.account_id, bookkeeping=args.bookkeeping, no_bookkeeping=args.no_bookkeeping, bookkeeping_domain_name=args.bookkeeping_domain_name, @@ -646,6 +647,10 @@ def main(): help="Region where you want to store \ your archives "\ "{}".format(constants.HELP_MESSAGE_CONFIG)) + group.add_argument('--account-id', + required=False, + default=default("account-id") if default("account-id") else '-', + help="AWS account ID of the account that owns the vault") bookkeeping = True if default('bookkeeping') == 'True' else False group.add_argument('--bookkeeping', required=False, From 94d46230636d8fa2613173751c5502ceba67615b Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Tue, 26 Sep 2017 20:02:04 -0500 Subject: [PATCH 22/23] Update README --- README.md | 141 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index cfb8b7b..594d25d 100644 --- a/README.md +++ b/README.md @@ -222,79 +222,102 @@ Usage description(help): usage: glacier-cmd [-h] [-c FILE] [--logtostdout] [--aws-access-key AWS_ACCESS_KEY] [--aws-secret-key AWS_SECRET_KEY] [--region REGION] - [--bookkeeping] + [--account-id ACCOUNT_ID] [--bookkeeping] [--no-bookkeeping] [--bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME] [--logfile LOGFILE] [--loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL}] - [--output {print,csv,json}] - - - {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash} - ... + [--output {csv,json,print}] + [--sdb-access-key SDB_ACCESS_KEY] + [--sdb-secret-key SDB_SECRET_KEY] [--sdb-region SDB_REGION] + {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash,sns} + ... Command line interface for Amazon Glacier optional arguments: - -h, --help show this help message and exit - -c FILE, --conf FILE Name of the file to log messages to. (default: - ~/.glacier-cmd) - --logtostdout Send log messages to stdout instead of the config - file. (default: False) + -h, --help show this help message and exit + -c FILE, --conf FILE Name of the file to log messages to. (default: + ~/.glacier-cmd) + --logtostdout Send log messages to stdout instead of the config + file. (default: False) Subcommands: - {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash} - For subcommand help, use: glacier-cmd -h - mkvault Create a new vault. - lsvault List available vaults. - describevault Describe a vault. - rmvault Remove a vault. - upload Upload an archive to Amazon Glacier. - listmultiparts List all active multipart uploads. - abortmultipart Abort a multipart upload. - inventory List inventory of a vault, if available. If not - available, creates inventory retrieval job if none - running already. - getarchive Requests to make an archive available for download. - download Download a file by archive id. - rmarchive Remove archive from Amazon Glacier. - search Search Amazon SimpleDB database for available archives - (requires bookkeeping to be enabled). - listjobs List active jobs in a vault. - describejob Describe a job. - treehash Calculate the tree-hash (Amazon style sha256-hash) of - a file. + {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash,sns} + For subcommand help, use: glacier-cmd -h + mkvault Create a new vault. + lsvault List available vaults. + describevault Describe a vault. + rmvault Remove a vault. + upload Upload an archive to Amazon Glacier. + listmultiparts List all active multipart uploads. + abortmultipart Abort a multipart upload. + inventory List inventory of a vault, if available. If not + available, creates inventory retrieval job if none + running already. + getarchive Requests to make an archive available for download. + download Download a file by archive id. + rmarchive Remove archive from Amazon Glacier. + search Search Amazon SimpleDB database for available archives + (requires bookkeeping to be enabled). + listjobs List active jobs in a vault. + describejob Describe a job. + treehash Calculate the tree-hash (Amazon style sha256-hash) of + a file. + sns Subcommands related to SNS aws: - --aws-access-key AWS_ACCESS_KEY - Your aws access key (Required if you have not created - a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) - (default: AKIAIP5VPUSCSJQ6BSSQ) - --aws-secret-key AWS_SECRET_KEY - Your aws secret key (Required if you have not created - a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) - (default: WDgq6ZZn7Y4Lkt5LxPuionw2pTLbonwdFZz1BGtS) + --aws-access-key AWS_ACCESS_KEY + Your aws access key (Required if you have not created + a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) + (default: AKIAINJIQK32YOKKYIPA) + --aws-secret-key AWS_SECRET_KEY + Your aws secret key (Required if you have not created + a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) + (default: Tl1NT/8b5sRxr0Dzz9ySUv50hoJM64hGa8QpiL5k) glacier: - --region REGION Region where you want to store your archives (Required - if you have not created a ~/.glacier-cmd or /etc - /glacier-cmd.conf config file) (default: us-east-1) - --bookkeeping Should we keep book of all created archives. This - requires a Amazon SimpleDB account and its bookkeeping - domain name set (default: True) - --bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME - Amazon SimpleDB domain name for bookkeeping. (default: - squirrel) - --no-bookkeeping If present, overrides either CLI or configuration file - options provided for bookkeeping either beforehand or - afterwards - --logfile LOGFILE File to write log messages to. (default: ~/.glacier-cmd.log) - --loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL} - Set the lowest level of messages you want to log. - (default: DEBUG) - --output {print,csv,json} - Set how to return results: print to the screen, or as - csv resp. json string. (default: print) + --region REGION Region where you want to store your archives (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: us-east-1) + --account-id ACCOUNT_ID + AWS account ID of the account that owns the vault + (default: -) + --bookkeeping Should we keep book of all created archives. This + requires a Amazon SimpleDB account and its bookkeeping + domain name set (default: False) + --no-bookkeeping Explicitly disables bookkeeping, regardless of other + configuration or command line options. (default: + False) + --bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME + Amazon SimpleDB domain name for bookkeeping. (default: + amazon-glacier) + --logfile LOGFILE File to write log messages to. (default: /home/gburca + /.glacier-cmd.log) + --loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL} + Set the lowest level of messages you want to log. + (default: WARNING) + --output {csv,json,print} + Set how to return results: print to the screen, or as + csv resp. json string. NOTE: to receive full output + use csv or json. `print` removes lines longer than 138 + chars (default: print) + + sdb: + --sdb-access-key SDB_ACCESS_KEY + aws access key to be used with bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: + AKIAINJIQK32YOKKYIPA) + --sdb-secret-key SDB_SECRET_KEY + aws secret key to be used with bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: + Tl1NT/8b5sRxr0Dzz9ySUv50hoJM64hGa8QpiL5k) + --sdb-region SDB_REGION + Region where you want to store bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: us-east-1) SimpleDB bookkeeping (custom) domain name ----------------------------------------- From c4df56e6e51d6c3e05041eefac534d495074067a Mon Sep 17 00:00:00 2001 From: Gabriel Burca Date: Wed, 27 Sep 2017 23:02:51 -0500 Subject: [PATCH 23/23] Hide passwords when printing help --- glacier/glacier.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/glacier/glacier.py b/glacier/glacier.py index c0e184d..8611483 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -524,6 +524,25 @@ def snsunsubscribe(args): sns_options=args.sns_options) output_table(response, args.output) +class CustomArgParseFormatter(argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter): + def _get_help_string(self, action): + """ + This method is identical to the base one, except that if the argument + ends in '-key', the default value is suppressed so that we don't print + out sensitive passwords (from the config file). + """ + help = action.help + if '%(default)' not in action.help: + if action.default is not argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + if action.option_strings[0].endswith('-key'): + pass + else: + help += ' (default: %(default)s)' + return help + def main(): program_description = u""" Command line interface for Amazon Glacier @@ -620,7 +639,7 @@ def main(): # Main configuration parser parser = argparse.ArgumentParser(parents=[conf_parser], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, + formatter_class=CustomArgParseFormatter, description=program_description) subparsers = parser.add_subparsers(title='Subcommands', help=u"For subcommand help, use: glacier-cmd -h")