From 50774829c94a08c50c2ada3e3e788dffdf033751 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Tue, 14 Feb 2012 18:11:38 -0800 Subject: [PATCH 01/16] Quick reformat to get to modern Python --- s3cmd | 4276 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 2208 insertions(+), 2068 deletions(-) diff --git a/s3cmd b/s3cmd index d4f2e00..0f8f1c0 100755 --- a/s3cmd +++ b/s3cmd @@ -7,9 +7,9 @@ import sys -if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.4: - sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n") - sys.exit(1) +if float("%d.%d" % (sys.version_info[0], sys.version_info[1])) < 2.4: + sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n") + sys.exit(1) import logging import time @@ -32,2126 +32,2266 @@ from logging import debug, info, warning, error from distutils.spawn import find_executable - - def output(message): - sys.stdout.write(message + "\n") + sys.stdout.write(message + "\n") + def check_args_type(args, type, verbose_type): - for arg in args: - if S3Uri(arg).type != type: - raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg)) + for arg in args: + if S3Uri(arg).type != type: + raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg)) + def _fswalk_follow_symlinks(path): - ''' - Walk filesystem, following symbolic links (but without recursion), on python2.4 and later - - If a recursive directory link is detected, emit a warning and skip. - ''' - assert os.path.isdir(path) # only designed for directory argument - walkdirs = set([path]) - targets = set() - for dirpath, dirnames, filenames in os.walk(path): - for dirname in dirnames: - current = os.path.join(dirpath, dirname) - target = os.path.realpath(current) - if os.path.islink(current): - if target in targets: - warning("Skipping recursively symlinked directory %s" % dirname) - else: - walkdirs.add(current) - targets.add(target) - for walkdir in walkdirs: - for value in os.walk(walkdir): - yield value + ''' + Walk filesystem, following symbolic links (but without recursion), on python2.4 and later + + If a recursive directory link is detected, emit a warning and skip. + ''' + assert os.path.isdir(path) # only designed for directory argument + walkdirs = set([path]) + targets = set() + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + current = os.path.join(dirpath, dirname) + target = os.path.realpath(current) + if os.path.islink(current): + if target in targets: + warning("Skipping recursively symlinked directory %s" % dirname) + else: + walkdirs.add(current) + targets.add(target) + for walkdir in walkdirs: + for value in os.walk(walkdir): + yield value + def fswalk(path, follow_symlinks): - ''' - Directory tree generator + ''' + Directory tree generator + + path (str) is the root of the directory tree to walk - path (str) is the root of the directory tree to walk + follow_symlinks (bool) indicates whether to descend into symbolically linked directories + ''' + if follow_symlinks: + return _fswalk_follow_symlinks(path) + return os.walk(path) - follow_symlinks (bool) indicates whether to descend into symbolically linked directories - ''' - if follow_symlinks: - return _fswalk_follow_symlinks(path) - return os.walk(path) def cmd_du(args): - s3 = S3(Config()) - if len(args) > 0: - uri = S3Uri(args[0]) - if uri.type == "s3" and uri.has_bucket(): - subcmd_bucket_usage(s3, uri) - return - subcmd_bucket_usage_all(s3) + s3 = S3(Config()) + if len(args) > 0: + uri = S3Uri(args[0]) + if uri.type == "s3" and uri.has_bucket(): + subcmd_bucket_usage(s3, uri) + return + subcmd_bucket_usage_all(s3) + def subcmd_bucket_usage_all(s3): - response = s3.list_all_buckets() - - buckets_size = 0 - for bucket in response["list"]: - size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"])) - if size != None: - buckets_size += size - total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes) - total_size_str = str(total_size) + size_coeff - output(u"".rjust(8, "-")) - output(u"%s Total" % (total_size_str.ljust(8))) + response = s3.list_all_buckets() + + buckets_size = 0 + for bucket in response["list"]: + size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"])) + if size != None: + buckets_size += size + total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes) + total_size_str = str(total_size) + size_coeff + output(u"".rjust(8, "-")) + output(u"%s Total" % (total_size_str.ljust(8))) + def subcmd_bucket_usage(s3, uri): - bucket = uri.bucket() - object = uri.object() - - if object.endswith('*'): - object = object[:-1] - try: - response = s3.bucket_list(bucket, prefix = object, recursive = True) - except S3Error, e: - if S3.codes.has_key(e.Code): - error(S3.codes[e.Code] % bucket) - return - else: - raise - bucket_size = 0 - for object in response["list"]: - size, size_coeff = formatSize(object["Size"], False) - bucket_size += size - total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes) - total_size_str = str(total_size) + size_coeff - output(u"%s %s" % (total_size_str.ljust(8), uri)) - return bucket_size + bucket = uri.bucket() + object = uri.object() + + if object.endswith('*'): + object = object[:-1] + try: + response = s3.bucket_list(bucket, prefix=object, recursive=True) + except S3Error, e: + if S3.codes.has_key(e.Code): + error(S3.codes[e.Code] % bucket) + return + else: + raise + bucket_size = 0 + for object in response["list"]: + size, size_coeff = formatSize(object["Size"], False) + bucket_size += size + total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes) + total_size_str = str(total_size) + size_coeff + output(u"%s %s" % (total_size_str.ljust(8), uri)) + return bucket_size + def cmd_ls(args): - s3 = S3(Config()) - if len(args) > 0: - uri = S3Uri(args[0]) - if uri.type == "s3" and uri.has_bucket(): - subcmd_bucket_list(s3, uri, cfg.select_dir) - return - subcmd_buckets_list_all(s3) + s3 = S3(Config()) + if len(args) > 0: + uri = S3Uri(args[0]) + if uri.type == "s3" and uri.has_bucket(): + subcmd_bucket_list(s3, uri, cfg.select_dir) + return + subcmd_buckets_list_all(s3) + def cmd_buckets_list_all_all(args): - s3 = S3(Config()) + s3 = S3(Config()) - response = s3.list_all_buckets() + response = s3.list_all_buckets() - for bucket in response["list"]: - subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"])) - output(u"") + for bucket in response["list"]: + subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"])) + output(u"") def subcmd_buckets_list_all(s3): - response = s3.list_all_buckets() - for bucket in response["list"]: - output(u"%s s3://%s" % ( - formatDateTime(bucket["CreationDate"]), - bucket["Name"], - )) - -def subcmd_bucket_list(s3, uri, select_dir = False): - bucket = uri.bucket() - prefix = uri.object() - - debug(u"Bucket 's3://%s':" % bucket) - if prefix.endswith('*'): - prefix = prefix[:-1] - try: - response = s3.bucket_list(bucket, prefix = prefix) - except S3Error, e: - if S3.codes.has_key(e.info["Code"]): - error(S3.codes[e.info["Code"]] % bucket) - return - else: - raise - - if select_dir: - format_string = u"%(uri)s" - elif cfg.list_md5: - format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s" - else: - format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s" - - if select_dir: - for prefix in response['common_prefixes']: - output(format_string % { - "timestamp": "", - "size": "DIR", - "coeff": "", - "md5": "", - "uri": uri.compose_uri(bucket, prefix["Prefix"])}) - else: - for prefix in response['common_prefixes']: - output(format_string % { - "timestamp": "", - "size": "DIR", - "coeff": "", - "md5": "", - "uri": uri.compose_uri(bucket, prefix["Prefix"])}) - - for object in response["list"]: - size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) - output(format_string % { - "timestamp": formatDateTime(object["LastModified"]), - "size" : str(size), - "coeff": size_coeff, - "md5" : object['ETag'].strip('"'), - "uri": uri.compose_uri(bucket, object["Key"]), - }) + response = s3.list_all_buckets() + for bucket in response["list"]: + output(u"%s s3://%s" % ( + formatDateTime(bucket["CreationDate"]), + bucket["Name"], + )) + + +def subcmd_bucket_list(s3, uri, select_dir=False): + bucket = uri.bucket() + prefix = uri.object() + + debug(u"Bucket 's3://%s':" % bucket) + if prefix.endswith('*'): + prefix = prefix[:-1] + try: + response = s3.bucket_list(bucket, prefix=prefix) + except S3Error, e: + if S3.codes.has_key(e.info["Code"]): + error(S3.codes[e.info["Code"]] % bucket) + return + else: + raise + + if select_dir: + format_string = u"%(uri)s" + elif cfg.list_md5: + format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s" + else: + format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s" + + if select_dir: + for prefix in response['common_prefixes']: + output(format_string % { + "timestamp": "", + "size": "DIR", + "coeff": "", + "md5": "", + "uri": uri.compose_uri(bucket, prefix["Prefix"])}) + else: + for prefix in response['common_prefixes']: + output(format_string % { + "timestamp": "", + "size": "DIR", + "coeff": "", + "md5": "", + "uri": uri.compose_uri(bucket, prefix["Prefix"])}) + + for object in response["list"]: + size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) + output(format_string % { + "timestamp": formatDateTime(object["LastModified"]), + "size": str(size), + "coeff": size_coeff, + "md5": object['ETag'].strip('"'), + "uri": uri.compose_uri(bucket, object["Key"]), + }) + def cmd_bucket_create(args): - s3 = S3(Config()) - for arg in args: - uri = S3Uri(arg) - if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): - raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) - try: - response = s3.bucket_create(uri.bucket(), cfg.bucket_location) - output(u"Bucket '%s' created" % uri.uri()) - except S3Error, e: - if S3.codes.has_key(e.info["Code"]): - error(S3.codes[e.info["Code"]] % uri.bucket()) - return - else: - raise + s3 = S3(Config()) + for arg in args: + uri = S3Uri(arg) + if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): + raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) + try: + response = s3.bucket_create(uri.bucket(), cfg.bucket_location) + output(u"Bucket '%s' created" % uri.uri()) + except S3Error, e: + if S3.codes.has_key(e.info["Code"]): + error(S3.codes[e.info["Code"]] % uri.bucket()) + return + else: + raise + def cmd_bucket_delete(args): - def _bucket_delete_one(uri): - try: - response = s3.bucket_delete(uri.bucket()) - except S3Error, e: - if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive): - warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...") - subcmd_object_del_uri(uri.uri(), recursive = True) - return _bucket_delete_one(uri) - elif S3.codes.has_key(e.info["Code"]): - error(S3.codes[e.info["Code"]] % uri.bucket()) - return - else: - raise - - s3 = S3(Config()) - for arg in args: - uri = S3Uri(arg) - if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): - raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) - _bucket_delete_one(uri) - output(u"Bucket '%s' removed" % uri.uri()) - -def fetch_local_list(args, recursive = None): - local_uris = [] - local_list = SortedDict(ignore_case = False) - single_file = False - - if type(args) not in (list, tuple): - args = [args] - - if recursive == None: - recursive = cfg.recursive - - for arg in args: - uri = S3Uri(arg) - if not uri.type == 'file': - raise ParameterError("Expecting filename or directory instead of: %s" % arg) - if uri.isdir() and not recursive: - raise ParameterError("Use --recursive to upload a directory: %s" % arg) - local_uris.append(uri) - - for uri in local_uris: - list_for_uri, single_file = _get_filelist_local(uri) - local_list.update(list_for_uri) - - ## Single file is True if and only if the user - ## specified one local URI and that URI represents - ## a FILE. Ie it is False if the URI was of a DIR - ## and that dir contained only one FILE. That's not - ## a case of single_file==True. - if len(local_list) > 1: - single_file = False - - return local_list, single_file - -def fetch_remote_list(args, require_attribs = False, recursive = None): - remote_uris = [] - remote_list = SortedDict(ignore_case = False) - - if type(args) not in (list, tuple): - args = [args] - - if recursive == None: - recursive = cfg.recursive - - for arg in args: - uri = S3Uri(arg) - if not uri.type == 's3': - raise ParameterError("Expecting S3 URI instead of '%s'" % arg) - remote_uris.append(uri) - - if recursive: - for uri in remote_uris: - objectlist = _get_filelist_remote(uri) - for key in objectlist: - remote_list[key] = objectlist[key] - else: - for uri in remote_uris: - uri_str = str(uri) - ## Wildcards used in remote URI? - ## If yes we'll need a bucket listing... - if uri_str.find('*') > -1 or uri_str.find('?') > -1: - first_wildcard = uri_str.find('*') - first_questionmark = uri_str.find('?') - if first_questionmark > -1 and first_questionmark < first_wildcard: - first_wildcard = first_questionmark - prefix = uri_str[:first_wildcard] - rest = uri_str[first_wildcard+1:] - ## Only request recursive listing if the 'rest' of the URI, - ## i.e. the part after first wildcard, contains '/' - need_recursion = rest.find('/') > -1 - objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion) - for key in objectlist: - ## Check whether the 'key' matches the requested wildcards - if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): - remote_list[key] = objectlist[key] - else: - ## No wildcards - simply append the given URI to the list - key = os.path.basename(uri.object()) - if not key: - raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) - remote_item = { - 'base_uri': uri, - 'object_uri_str': unicode(uri), - 'object_key': uri.object() - } - if require_attribs: - response = S3(cfg).object_info(uri) - remote_item.update({ - 'size': int(response['headers']['content-length']), - 'md5': response['headers']['etag'].strip('"\''), - 'timestamp' : Utils.dateRFC822toUnix(response['headers']['date']) - }) - remote_list[key] = remote_item - return remote_list + def _bucket_delete_one(uri): + try: + response = s3.bucket_delete(uri.bucket()) + except S3Error, e: + if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive): + warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...") + subcmd_object_del_uri(uri.uri(), recursive=True) + return _bucket_delete_one(uri) + elif S3.codes.has_key(e.info["Code"]): + error(S3.codes[e.info["Code"]] % uri.bucket()) + return + else: + raise + + s3 = S3(Config()) + for arg in args: + uri = S3Uri(arg) + if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): + raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) + _bucket_delete_one(uri) + output(u"Bucket '%s' removed" % uri.uri()) + + +def fetch_local_list(args, recursive=None): + local_uris = [] + local_list = SortedDict(ignore_case=False) + single_file = False + + if type(args) not in (list, tuple): + args = [args] + + if recursive == None: + recursive = cfg.recursive + + for arg in args: + uri = S3Uri(arg) + if not uri.type == 'file': + raise ParameterError("Expecting filename or directory instead of: %s" % arg) + if uri.isdir() and not recursive: + raise ParameterError("Use --recursive to upload a directory: %s" % arg) + local_uris.append(uri) + + for uri in local_uris: + list_for_uri, single_file = _get_filelist_local(uri) + local_list.update(list_for_uri) + + ## Single file is True if and only if the user + ## specified one local URI and that URI represents + ## a FILE. Ie it is False if the URI was of a DIR + ## and that dir contained only one FILE. That's not + ## a case of single_file==True. + if len(local_list) > 1: + single_file = False + + return local_list, single_file + + +def fetch_remote_list(args, require_attribs=False, recursive=None): + remote_uris = [] + remote_list = SortedDict(ignore_case=False) + + if type(args) not in (list, tuple): + args = [args] + + if recursive == None: + recursive = cfg.recursive + + for arg in args: + uri = S3Uri(arg) + if not uri.type == 's3': + raise ParameterError("Expecting S3 URI instead of '%s'" % arg) + remote_uris.append(uri) + + if recursive: + for uri in remote_uris: + objectlist = _get_filelist_remote(uri) + for key in objectlist: + remote_list[key] = objectlist[key] + else: + for uri in remote_uris: + uri_str = str(uri) + ## Wildcards used in remote URI? + ## If yes we'll need a bucket listing... + if uri_str.find('*') > -1 or uri_str.find('?') > -1: + first_wildcard = uri_str.find('*') + first_questionmark = uri_str.find('?') + if first_questionmark > -1 and first_questionmark < first_wildcard: + first_wildcard = first_questionmark + prefix = uri_str[:first_wildcard] + rest = uri_str[first_wildcard + 1:] + ## Only request recursive listing if the 'rest' of the URI, + ## i.e. the part after first wildcard, contains '/' + need_recursion = rest.find('/') > -1 + objectlist = _get_filelist_remote(S3Uri(prefix), recursive=need_recursion) + for key in objectlist: + ## Check whether the 'key' matches the requested wildcards + if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): + remote_list[key] = objectlist[key] + else: + ## No wildcards - simply append the given URI to the list + key = os.path.basename(uri.object()) + if not key: + raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) + remote_item = { + 'base_uri': uri, + 'object_uri_str': unicode(uri), + 'object_key': uri.object() + } + if require_attribs: + response = S3(cfg).object_info(uri) + remote_item.update({ + 'size': int(response['headers']['content-length']), + 'md5': response['headers']['etag'].strip('"\''), + 'timestamp': Utils.dateRFC822toUnix(response['headers']['date']) + }) + remote_list[key] = remote_item + return remote_list + def cmd_object_put(args): - cfg = Config() - - if len(args) == 0: - raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") - - ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) - if destination_base_uri.type != 's3': - raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) - destination_base = str(destination_base_uri) - - if len(args) == 0: - raise ParameterError("Nothing to upload. Expecting a local file or directory.") - - local_list, single_file_local = fetch_local_list(args) - - local_list, exclude_list = _filelist_filter_exclude_include(local_list) - - local_count = len(local_list) - - info(u"Summary: %d local files to upload" % local_count) - - if local_count > 0: - if not destination_base.endswith("/"): - if not single_file_local: - raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") - local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) - else: - for key in local_list: - local_list[key]['remote_uri'] = unicodise(destination_base + key) - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - for key in local_list: - output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) - - warning(u"Exitting now because of --dry-run") - return - - if cfg.parallel and len(local_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for key in local_list: - seq += 1 - q.put([local_list[key],seq,len(local_list)]) - - for i in range(cfg.workers): - t = threading.Thread(target=put_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() - else: - seq = 0 - for key in local_list: - seq += 1 - do_put_work(local_list[key],seq,len(local_list)) + cfg = Config() + + if len(args) == 0: + raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + if len(args) == 0: + raise ParameterError("Nothing to upload. Expecting a local file or directory.") + + local_list, single_file_local = fetch_local_list(args) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + local_count = len(local_list) + + info(u"Summary: %d local files to upload" % local_count) + + if local_count > 0: + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + local_list[key]['remote_uri'] = unicodise(destination_base + key) + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in local_list: + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.parallel and len(local_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in local_list: + seq += 1 + q.put([local_list[key], seq, len(local_list)]) + + for i in range(cfg.workers): + t = threading.Thread(target=put_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in local_list: + seq += 1 + do_put_work(local_list[key], seq, len(local_list)) + def put_worker(): - while True: - try: - (item,seq,total) = q.get_nowait() - except Queue.Empty: - return - try: - do_put_work(item,seq,total) - except Exception, e: - report_exception(e) - exit - q.task_done() - -def do_put_work(item,seq,total): - cfg = Config() - s3 = S3(cfg) - uri_final = S3Uri(item['remote_uri']) - - extra_headers = copy(cfg.extra_headers) - full_name_orig = item['full_name'] - full_name = full_name_orig - seq_label = "[%d of %d]" % (seq, total) - if Config().encrypt: - exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig) - if not Config().progress_meter: - output(u"File '%s' started %s" % - (unicodise(full_name_orig), seq_label)) - try: - response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label) - except S3UploadError, e: - error(u"Upload of '%s' failed too many times. Skipping that file." % full_name_orig) - return - except InvalidFileError, e: - warning(u"File can not be uploaded: %s" % e) - return - speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - if not Config().progress_meter: - output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % - (unicodise(full_name_orig), uri_final, response["size"], response["elapsed"], - speed_fmt[0], speed_fmt[1], seq_label)) - if Config().acl_public: - output(u"Public URL of the object is: %s" % - (uri_final.public_url())) - if Config().encrypt and full_name != full_name_orig: - debug(u"Removing temporary encrypted file: %s" % unicodise(full_name)) - os.remove(full_name) + while True: + try: + (item, seq, total) = q.get_nowait() + except Queue.Empty: + return + try: + do_put_work(item, seq, total) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def do_put_work(item, seq, total): + cfg = Config() + s3 = S3(cfg) + uri_final = S3Uri(item['remote_uri']) + + extra_headers = copy(cfg.extra_headers) + full_name_orig = item['full_name'] + full_name = full_name_orig + seq_label = "[%d of %d]" % (seq, total) + if Config().encrypt: + exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig) + if not Config().progress_meter: + output(u"File '%s' started %s" % + (unicodise(full_name_orig), seq_label)) + try: + response = s3.object_put(full_name, uri_final, extra_headers, extra_label=seq_label) + except S3UploadError, e: + error(u"Upload of '%s' failed too many times. Skipping that file." % full_name_orig) + return + except InvalidFileError, e: + warning(u"File can not be uploaded: %s" % e) + return + speed_fmt = formatSize(response["speed"], human_readable=True, floating_point=True) + if not Config().progress_meter: + output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + (unicodise(full_name_orig), uri_final, response["size"], response["elapsed"], + speed_fmt[0], speed_fmt[1], seq_label)) + if Config().acl_public: + output(u"Public URL of the object is: %s" % + (uri_final.public_url())) + if Config().encrypt and full_name != full_name_orig: + debug(u"Removing temporary encrypted file: %s" % unicodise(full_name)) + os.remove(full_name) + def cmd_object_get(args): - cfg = Config() - - ## Check arguments: - ## if not --recursive: - ## - first N arguments must be S3Uri - ## - if the last one is S3 make current dir the destination_base - ## - if the last one is a directory: - ## - take all 'basenames' of the remote objects and - ## make the destination name be 'destination_base'+'basename' - ## - if the last one is a file or not existing: - ## - if the number of sources (N, above) == 1 treat it - ## as a filename and save the object there. - ## - if there's more sources -> Error - ## if --recursive: - ## - first N arguments must be S3Uri - ## - for each Uri get a list of remote objects with that Uri as a prefix - ## - apply exclude/include rules - ## - each list item will have MD5sum, Timestamp and pointer to S3Uri - ## used as a prefix. - ## - the last arg may be a local directory - destination_base - ## - if the last one is S3 make current dir the destination_base - ## - if the last one doesn't exist check remote list: - ## - if there is only one item and its_prefix==its_name - ## download that item to the name given in last arg. - ## - if there are more remote items use the last arg as a destination_base - ## and try to create the directory (incl. all parents). - ## - ## In both cases we end up with a list mapping remote object names (keys) to local file names. - - ## Each item will be a dict with the following attributes - # {'remote_uri', 'local_filename'} - download_list = [] - - if len(args) == 0: - raise ParameterError("Nothing to download. Expecting S3 URI.") - - if S3Uri(args[-1]).type == 'file': - destination_base = args.pop() - else: - destination_base = "." - - if len(args) == 0: - raise ParameterError("Nothing to download. Expecting S3 URI.") - - remote_list = fetch_remote_list(args, require_attribs = False) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - - remote_count = len(remote_list) - - info(u"Summary: %d remote files to download" % remote_count) - - if remote_count > 0: - if not os.path.isdir(destination_base) or destination_base == '-': - ## We were either given a file name (existing or not) or want STDOUT - if remote_count > 1: - raise ParameterError("Destination must be a directory when downloading multiple sources.") - remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base) - elif os.path.isdir(destination_base): - if destination_base[-1] != os.path.sep: - destination_base += os.path.sep - for key in remote_list: - remote_list[key]['local_filename'] = destination_base + key - else: - raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base) - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - for key in remote_list: - output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename'])) - - warning(u"Exitting now because of --dry-run") - return - - if cfg.parallel and len(remote_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for key in remote_list: - seq += 1 - q.put([remote_list[key],seq,len(remote_list)]) - - for i in range(cfg.workers): - t = threading.Thread(target=get_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() - else: - seq = 0 - for key in remote_list: - seq += 1 - do_get_work(remote_list[key],seq,len(remote_list)) + cfg = Config() + + ## Check arguments: + ## if not --recursive: + ## - first N arguments must be S3Uri + ## - if the last one is S3 make current dir the destination_base + ## - if the last one is a directory: + ## - take all 'basenames' of the remote objects and + ## make the destination name be 'destination_base'+'basename' + ## - if the last one is a file or not existing: + ## - if the number of sources (N, above) == 1 treat it + ## as a filename and save the object there. + ## - if there's more sources -> Error + ## if --recursive: + ## - first N arguments must be S3Uri + ## - for each Uri get a list of remote objects with that Uri as a prefix + ## - apply exclude/include rules + ## - each list item will have MD5sum, Timestamp and pointer to S3Uri + ## used as a prefix. + ## - the last arg may be a local directory - destination_base + ## - if the last one is S3 make current dir the destination_base + ## - if the last one doesn't exist check remote list: + ## - if there is only one item and its_prefix==its_name + ## download that item to the name given in last arg. + ## - if there are more remote items use the last arg as a destination_base + ## and try to create the directory (incl. all parents). + ## + ## In both cases we end up with a list mapping remote object names (keys) to local file names. + + ## Each item will be a dict with the following attributes + # {'remote_uri', 'local_filename'} + download_list = [] + + if len(args) == 0: + raise ParameterError("Nothing to download. Expecting S3 URI.") + + if S3Uri(args[-1]).type == 'file': + destination_base = args.pop() + else: + destination_base = "." + + if len(args) == 0: + raise ParameterError("Nothing to download. Expecting S3 URI.") + + remote_list = fetch_remote_list(args, require_attribs=False) + remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + + remote_count = len(remote_list) + + info(u"Summary: %d remote files to download" % remote_count) + + if remote_count > 0: + if not os.path.isdir(destination_base) or destination_base == '-': + ## We were either given a file name (existing or not) or want STDOUT + if remote_count > 1: + raise ParameterError("Destination must be a directory when downloading multiple sources.") + remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base) + elif os.path.isdir(destination_base): + if destination_base[-1] != os.path.sep: + destination_base += os.path.sep + for key in remote_list: + remote_list[key]['local_filename'] = destination_base + key + else: + raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base) + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in remote_list: + output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename'])) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.parallel and len(remote_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in remote_list: + seq += 1 + q.put([remote_list[key], seq, len(remote_list)]) + + for i in range(cfg.workers): + t = threading.Thread(target=get_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in remote_list: + seq += 1 + do_get_work(remote_list[key], seq, len(remote_list)) + def get_worker(): - while True: - try: - (item,seq,total) = q.get_nowait() - except Queue.Empty: - return - try: - do_get_work(item,seq,total) - except ParameterError, e: - error(u"Parameter problem: %s" % e) - except S3Error, e: - error(u"S3 error: %s" % e) - exit - except Exception, e: - report_exception(e) - exit - q.task_done() - -def do_get_work(item,seq,total): - cfg = Config() - s3 = S3(cfg) - uri = S3Uri(item['object_uri_str']) - ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding - destination = unicodise_safe(item['local_filename']) - seq_label = "[%d of %d]" % (seq, total) - start_position = 0 - - if destination == "-": - ## stdout - dst_stream = sys.__stdout__ - else: - ## File - try: - file_exists = os.path.exists(destination) - try: - dst_stream = open(destination, "ab") - except IOError, e: - if e.errno == errno.ENOENT: - basename = destination[:destination.rindex(os.path.sep)] - info(u"Creating directory: %s" % basename) - os.makedirs(basename) - dst_stream = open(destination, "ab") - else: - raise - if file_exists: - if Config().get_continue: - start_position = dst_stream.tell() - elif Config().force: - start_position = 0L - dst_stream.seek(0L) - dst_stream.truncate() - elif Config().skip_existing: - info(u"Skipping over existing file: %s" % (destination)) - return - else: - dst_stream.close() - raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) - except IOError, e: - error(u"Skipping %s: %s" % (destination, e.strerror)) - return - if not Config().progress_meter and destination != "-": - output(u"File %s started %s" % - (uri, seq_label)) - response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label) - if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): - gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"]) - response["size"] = os.stat(destination)[6] - if not Config().progress_meter and destination != "-": - speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % - (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1])) - + while True: + try: + (item, seq, total) = q.get_nowait() + except Queue.Empty: + return + try: + do_get_work(item, seq, total) + except ParameterError, e: + error(u"Parameter problem: %s" % e) + except S3Error, e: + error(u"S3 error: %s" % e) + exit + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def do_get_work(item, seq, total): + cfg = Config() + s3 = S3(cfg) + uri = S3Uri(item['object_uri_str']) + ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding + destination = unicodise_safe(item['local_filename']) + seq_label = "[%d of %d]" % (seq, total) + start_position = 0 + + if destination == "-": + ## stdout + dst_stream = sys.__stdout__ + else: + ## File + try: + file_exists = os.path.exists(destination) + try: + dst_stream = open(destination, "ab") + except IOError, e: + if e.errno == errno.ENOENT: + basename = destination[:destination.rindex(os.path.sep)] + info(u"Creating directory: %s" % basename) + os.makedirs(basename) + dst_stream = open(destination, "ab") + else: + raise + if file_exists: + if Config().get_continue: + start_position = dst_stream.tell() + elif Config().force: + start_position = 0L + dst_stream.seek(0L) + dst_stream.truncate() + elif Config().skip_existing: + info(u"Skipping over existing file: %s" % (destination)) + return + else: + dst_stream.close() + raise ParameterError( + u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) + except IOError, e: + error(u"Skipping %s: %s" % (destination, e.strerror)) + return + if not Config().progress_meter and destination != "-": + output(u"File %s started %s" % + (uri, seq_label)) + response = s3.object_get(uri, dst_stream, start_position=start_position, extra_label=seq_label) + if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): + gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"]) + response["size"] = os.stat(destination)[6] + if not Config().progress_meter and destination != "-": + speed_fmt = formatSize(response["speed"], human_readable=True, floating_point=True) + output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % + (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1])) + + def cmd_object_del(args): - for uri_str in args: - uri = S3Uri(uri_str) - if uri.type != "s3": - raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str) - if not uri.has_object(): - if Config().recursive and not Config().force: - raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str) - elif not Config().recursive: - raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive") - subcmd_object_del_uri(uri_str) + for uri_str in args: + uri = S3Uri(uri_str) + if uri.type != "s3": + raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str) + if not uri.has_object(): + if Config().recursive and not Config().force: + raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str) + elif not Config().recursive: + raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive") + subcmd_object_del_uri(uri_str) + -def subcmd_object_del_uri(uri_str, recursive = None): - s3 = S3(cfg) +def subcmd_object_del_uri(uri_str, recursive=None): + s3 = S3(cfg) - if recursive is None: - recursive = cfg.recursive + if recursive is None: + recursive = cfg.recursive - remote_list = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list = fetch_remote_list(uri_str, require_attribs=False, recursive=recursive) + remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - remote_count = len(remote_list) + remote_count = len(remote_list) - info(u"Summary: %d remote files to delete" % remote_count) + info(u"Summary: %d remote files to delete" % remote_count) - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - for key in remote_list: - output(u"delete: %s" % remote_list[key]['object_uri_str']) + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in remote_list: + output(u"delete: %s" % remote_list[key]['object_uri_str']) - warning(u"Exitting now because of --dry-run") - return + warning(u"Exitting now because of --dry-run") + return + + for key in remote_list: + item = remote_list[key] + response = s3.object_delete(S3Uri(item['object_uri_str'])) + output(u"File %s deleted" % item['object_uri_str']) - for key in remote_list: - item = remote_list[key] - response = s3.object_delete(S3Uri(item['object_uri_str'])) - output(u"File %s deleted" % item['object_uri_str']) def subcmd_cp_mv(args, process_fce, action_str, message): - if len(args) < 2: - raise ParameterError("Expecting two or more S3 URIs for " + action_str) - dst_base_uri = S3Uri(args.pop()) - if dst_base_uri.type != "s3": - raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.") - destination_base = dst_base_uri.uri() - - remote_list = fetch_remote_list(args, require_attribs = False) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - - remote_count = len(remote_list) - - info(u"Summary: %d remote files to %s" % (remote_count, action_str)) - - if cfg.recursive: - if not destination_base.endswith("/"): - destination_base += "/" - for key in remote_list: - remote_list[key]['dest_name'] = destination_base + key - else: - key = remote_list.keys()[0] - if destination_base.endswith("/"): - remote_list[key]['dest_name'] = destination_base + key - else: - remote_list[key]['dest_name'] = destination_base - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - for key in remote_list: - output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name'])) - - warning(u"Exitting now because of --dry-run") - return - - if cfg.parallel and len(remote_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for key in remote_list: - seq += 1 - seq_label = "[%d of %d]" % (seq, remote_count) - - item = remote_list[key] - src_uri = S3Uri(item['object_uri_str']) - dst_uri = S3Uri(item['dest_name']) - extra_headers = copy(cfg.extra_headers) - q.put([src_uri,dst_uri,extra_headers,process_fce,message,seq_label]) - - for i in range(cfg.workers): - t = threading.Thread(target=cp_mv_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() + if len(args) < 2: + raise ParameterError("Expecting two or more S3 URIs for " + action_str) + dst_base_uri = S3Uri(args.pop()) + if dst_base_uri.type != "s3": + raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.") + destination_base = dst_base_uri.uri() + + remote_list = fetch_remote_list(args, require_attribs=False) + remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + + remote_count = len(remote_list) + + info(u"Summary: %d remote files to %s" % (remote_count, action_str)) + + if cfg.recursive: + if not destination_base.endswith("/"): + destination_base += "/" + for key in remote_list: + remote_list[key]['dest_name'] = destination_base + key + else: + key = remote_list.keys()[0] + if destination_base.endswith("/"): + remote_list[key]['dest_name'] = destination_base + key else: - seq = 0 - for key in remote_list: - seq += 1 - seq_label = "[%d of %d]" % (seq, remote_count) - - item = remote_list[key] - src_uri = S3Uri(item['object_uri_str']) - dst_uri = S3Uri(item['dest_name']) - - extra_headers = copy(cfg.extra_headers) - response = process_fce(src_uri, dst_uri, extra_headers) - output(message % { "src" : src_uri, "dst" : dst_uri, "seq_label" : seq_label}) - if Config().acl_public: - info(u"Public URL is: %s" % dst_uri.public_url()) + remote_list[key]['dest_name'] = destination_base + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in remote_list: + output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name'])) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.parallel and len(remote_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in remote_list: + seq += 1 + seq_label = "[%d of %d]" % (seq, remote_count) + + item = remote_list[key] + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['dest_name']) + extra_headers = copy(cfg.extra_headers) + q.put([src_uri, dst_uri, extra_headers, process_fce, message, seq_label]) + + for i in range(cfg.workers): + t = threading.Thread(target=cp_mv_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in remote_list: + seq += 1 + seq_label = "[%d of %d]" % (seq, remote_count) + + item = remote_list[key] + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['dest_name']) + + extra_headers = copy(cfg.extra_headers) + response = process_fce(src_uri, dst_uri, extra_headers) + output(message % {"src": src_uri, "dst": dst_uri, "seq_label": seq_label}) + if Config().acl_public: + info(u"Public URL is: %s" % dst_uri.public_url()) + def cp_mv_worker(): - while True: - try: - (src_uri,dst_uri,extra_headers,process_fce,message,seq_label) = q.get_nowait() - except Queue.Empty: - return - try: - response = process_fce(src_uri, dst_uri, extra_headers) - output(message % { "src" : src_uri, "dst" : dst_uri , "seq_label" : seq_label}) - if Config().acl_public: - info(u"Public URL is: %s" % dst_uri.public_url()) - except Exception, e: - report_exception(e) - exit - q.task_done() + while True: + try: + (src_uri, dst_uri, extra_headers, process_fce, message, seq_label) = q.get_nowait() + except Queue.Empty: + return + try: + response = process_fce(src_uri, dst_uri, extra_headers) + output(message % {"src": src_uri, "dst": dst_uri, "seq_label": seq_label}) + if Config().acl_public: + info(u"Public URL is: %s" % dst_uri.public_url()) + except Exception, e: + report_exception(e) + exit + q.task_done() + def cmd_cp(args): - s3 = S3(Config()) - subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s %(seq_label)s") + s3 = S3(Config()) + subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s %(seq_label)s") + def cmd_mv(args): - s3 = S3(Config()) - subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s %(seq_label)s") + s3 = S3(Config()) + subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s %(seq_label)s") + def cmd_info(args): - s3 = S3(Config()) - - while (len(args)): - uri_arg = args.pop(0) - uri = S3Uri(uri_arg) - if uri.type != "s3" or not uri.has_bucket(): - raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) - - try: - if uri.has_object(): - info = s3.object_info(uri) - output(u"%s (object):" % uri.uri()) - output(u" File size: %s" % info['headers']['content-length']) - output(u" Last mod: %s" % info['headers']['last-modified']) - output(u" MIME type: %s" % info['headers']['content-type']) - output(u" MD5 sum: %s" % info['headers']['etag'].strip('"')) - else: - info = s3.bucket_info(uri) - output(u"%s (bucket):" % uri.uri()) - output(u" Location: %s" % info['bucket-location']) - acl = s3.get_acl(uri) - acl_grant_list = acl.getGrantList() - for grant in acl_grant_list: - output(u" ACL: %s: %s" % (grant['grantee'], grant['permission'])) - if acl.isAnonRead(): - output(u" URL: %s" % uri.public_url()) - except S3Error, e: - if S3.codes.has_key(e.info["Code"]): - error(S3.codes[e.info["Code"]] % uri.bucket()) - return - else: - raise + s3 = S3(Config()) + + while (len(args)): + uri_arg = args.pop(0) + uri = S3Uri(uri_arg) + if uri.type != "s3" or not uri.has_bucket(): + raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) + + try: + if uri.has_object(): + info = s3.object_info(uri) + output(u"%s (object):" % uri.uri()) + output(u" File size: %s" % info['headers']['content-length']) + output(u" Last mod: %s" % info['headers']['last-modified']) + output(u" MIME type: %s" % info['headers']['content-type']) + output(u" MD5 sum: %s" % info['headers']['etag'].strip('"')) + else: + info = s3.bucket_info(uri) + output(u"%s (bucket):" % uri.uri()) + output(u" Location: %s" % info['bucket-location']) + acl = s3.get_acl(uri) + acl_grant_list = acl.getGrantList() + for grant in acl_grant_list: + output(u" ACL: %s: %s" % (grant['grantee'], grant['permission'])) + if acl.isAnonRead(): + output(u" URL: %s" % uri.public_url()) + except S3Error, e: + if S3.codes.has_key(e.info["Code"]): + error(S3.codes[e.info["Code"]] % uri.bucket()) + return + else: + raise + def _get_filelist_local(local_uri): - info(u"Compiling list of local files...") - if local_uri.isdir(): - local_base = deunicodise(local_uri.basename()) - local_path = deunicodise(local_uri.path()) - filelist = fswalk(local_path, cfg.follow_symlinks) - single_file = False - else: - local_base = "" - local_path = deunicodise(local_uri.dirname()) - filelist = [( local_path, [], [deunicodise(local_uri.basename())] )] - single_file = True - loc_list = SortedDict(ignore_case = False) - for root, dirs, files in filelist: - rel_root = root.replace(local_path, local_base, 1) - for f in files: - full_name = os.path.join(root, f) - if not os.path.isfile(full_name): - continue - if os.path.islink(full_name): - if not cfg.follow_symlinks: - continue - relative_file = unicodise(os.path.join(rel_root, f)) - if os.path.sep != "/": - # Convert non-unix dir separators to '/' - relative_file = "/".join(relative_file.split(os.path.sep)) - if cfg.urlencoding_mode == "normal": - relative_file = replace_nonprintables(relative_file) - if relative_file.startswith('./'): - relative_file = relative_file[2:] - sr = os.stat_result(os.lstat(full_name)) - loc_list[relative_file] = { - 'full_name_unicode' : unicodise(full_name), - 'full_name' : full_name, - 'size' : sr.st_size, - 'mtime' : sr.st_mtime, - ## TODO: Possibly more to save here... - } - return loc_list, single_file - -def _get_filelist_remote(remote_uri, recursive = True): - ## If remote_uri ends with '/' then all remote files will have - ## the remote_uri prefix removed in the relative path. - ## If, on the other hand, the remote_uri ends with something else - ## (probably alphanumeric symbol) we'll use the last path part - ## in the relative path. - ## - ## Complicated, eh? See an example: - ## _get_filelist_remote("s3://bckt/abc/def") may yield: - ## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} } - ## _get_filelist_remote("s3://bckt/abc/def/") will yield: - ## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} } - ## Furthermore a prefix-magic can restrict the return list: - ## _get_filelist_remote("s3://bckt/abc/def/x") yields: - ## { 'xyz/blah.txt' : {} } - - info(u"Retrieving list of remote files for %s ..." % remote_uri) - - s3 = S3(Config()) - response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive) - - rem_base_original = rem_base = remote_uri.object() - remote_uri_original = remote_uri - if rem_base != '' and rem_base[-1] != '/': - rem_base = rem_base[:rem_base.rfind('/')+1] - remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) - rem_base_len = len(rem_base) - rem_list = SortedDict(ignore_case = False) - break_now = False - for object in response['list']: - if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: - ## We asked for one file and we got that file :-) - key = os.path.basename(object['Key']) - object_uri_str = remote_uri_original.uri() - break_now = True - rem_list = {} ## Remove whatever has already been put to rem_list - else: - key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! - object_uri_str = remote_uri.uri() + key - rem_list[key] = { - 'size' : int(object['Size']), - 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( - 'md5' : object['ETag'][1:-1], - 'object_key' : object['Key'], - 'object_uri_str' : object_uri_str, - 'base_uri' : remote_uri, - } - if break_now: - break - return rem_list + info(u"Compiling list of local files...") + if local_uri.isdir(): + local_base = deunicodise(local_uri.basename()) + local_path = deunicodise(local_uri.path()) + filelist = fswalk(local_path, cfg.follow_symlinks) + single_file = False + else: + local_base = "" + local_path = deunicodise(local_uri.dirname()) + filelist = [( local_path, [], [deunicodise(local_uri.basename())] )] + single_file = True + loc_list = SortedDict(ignore_case=False) + for root, dirs, files in filelist: + rel_root = root.replace(local_path, local_base, 1) + for f in files: + full_name = os.path.join(root, f) + if not os.path.isfile(full_name): + continue + if os.path.islink(full_name): + if not cfg.follow_symlinks: + continue + relative_file = unicodise(os.path.join(rel_root, f)) + if os.path.sep != "/": + # Convert non-unix dir separators to '/' + relative_file = "/".join(relative_file.split(os.path.sep)) + if cfg.urlencoding_mode == "normal": + relative_file = replace_nonprintables(relative_file) + if relative_file.startswith('./'): + relative_file = relative_file[2:] + sr = os.stat_result(os.lstat(full_name)) + loc_list[relative_file] = { + 'full_name_unicode': unicodise(full_name), + 'full_name': full_name, + 'size': sr.st_size, + 'mtime': sr.st_mtime, + ## TODO: Possibly more to save here... + } + return loc_list, single_file + + +def _get_filelist_remote(remote_uri, recursive=True): + ## If remote_uri ends with '/' then all remote files will have + ## the remote_uri prefix removed in the relative path. + ## If, on the other hand, the remote_uri ends with something else + ## (probably alphanumeric symbol) we'll use the last path part + ## in the relative path. + ## + ## Complicated, eh? See an example: + ## _get_filelist_remote("s3://bckt/abc/def") may yield: + ## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} } + ## _get_filelist_remote("s3://bckt/abc/def/") will yield: + ## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} } + ## Furthermore a prefix-magic can restrict the return list: + ## _get_filelist_remote("s3://bckt/abc/def/x") yields: + ## { 'xyz/blah.txt' : {} } + + info(u"Retrieving list of remote files for %s ..." % remote_uri) + + s3 = S3(Config()) + response = s3.bucket_list(remote_uri.bucket(), prefix=remote_uri.object(), recursive=recursive) + + rem_base_original = rem_base = remote_uri.object() + remote_uri_original = remote_uri + if rem_base != '' and rem_base[-1] != '/': + rem_base = rem_base[:rem_base.rfind('/') + 1] + remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) + rem_base_len = len(rem_base) + rem_list = SortedDict(ignore_case=False) + break_now = False + for object in response['list']: + if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: + ## We asked for one file and we got that file :-) + key = os.path.basename(object['Key']) + object_uri_str = remote_uri_original.uri() + break_now = True + rem_list = {} ## Remove whatever has already been put to rem_list + else: + key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! + object_uri_str = remote_uri.uri() + key + rem_list[key] = { + 'size': int(object['Size']), + 'timestamp': dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( + 'md5': object['ETag'][1:-1], + 'object_key': object['Key'], + 'object_uri_str': object_uri_str, + 'base_uri': remote_uri, + } + if break_now: + break + return rem_list + def _filelist_filter_exclude_include(src_list): - info(u"Applying --exclude/--include") - cfg = Config() - exclude_list = SortedDict(ignore_case = False) - for file in src_list.keys(): - debug(u"CHECK: %s" % file) - excluded = False - for r in cfg.exclude: - if r.search(file): - excluded = True - debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r])) - break - if excluded: - ## No need to check for --include if not excluded - for r in cfg.include: - if r.search(file): - excluded = False - debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r])) - break - if excluded: - ## Still excluded - ok, action it - debug(u"EXCLUDE: %s" % file) - exclude_list[file] = src_list[file] - del(src_list[file]) - continue - else: - debug(u"PASS: %s" % (file)) - return src_list, exclude_list - -def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote, src_and_dst_remote = False): - info(u"Verifying attributes...") - cfg = Config() - exists_list = SortedDict(ignore_case = False) - - debug("Comparing filelists (src_is_local_and_dst_is_remote=%s)" % src_is_local_and_dst_is_remote) - debug("src_list.keys: %s" % src_list.keys()) - debug("dst_list.keys: %s" % dst_list.keys()) - - for file in src_list.keys(): - debug(u"CHECK: %s" % file) - if dst_list.has_key(file): - ## Was --skip-existing requested? - if cfg.skip_existing: - debug(u"IGNR: %s (used --skip-existing)" % (file)) - exists_list[file] = src_list[file] - del(src_list[file]) - ## Remove from destination-list, all that is left there will be deleted - del(dst_list[file]) - continue - - attribs_match = True - ## Check size first - if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: - debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) - attribs_match = False - - if attribs_match and 'md5' in cfg.sync_checks: - ## ... same size, check MD5 - if src_and_dst_remote: - src_md5 = src_list[file]['md5'] - dst_md5 = dst_list[file]['md5'] - elif src_is_local_and_dst_is_remote: - src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) - dst_md5 = dst_list[file]['md5'] - else: - src_md5 = src_list[file]['md5'] - dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) - if src_md5 != dst_md5: - ## Checksums are different. - attribs_match = False - debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) - - if attribs_match: - ## Remove from source-list, all that is left there will be transferred - debug(u"IGNR: %s (transfer not needed)" % file) - exists_list[file] = src_list[file] - del(src_list[file]) - - ## Remove from destination-list, all that is left there will be deleted - del(dst_list[file]) - - return src_list, dst_list, exists_list + info(u"Applying --exclude/--include") + cfg = Config() + exclude_list = SortedDict(ignore_case=False) + for file in src_list.keys(): + debug(u"CHECK: %s" % file) + excluded = False + for r in cfg.exclude: + if r.search(file): + excluded = True + debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r])) + break + if excluded: + ## No need to check for --include if not excluded + for r in cfg.include: + if r.search(file): + excluded = False + debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r])) + break + if excluded: + ## Still excluded - ok, action it + debug(u"EXCLUDE: %s" % file) + exclude_list[file] = src_list[file] + del(src_list[file]) + continue + else: + debug(u"PASS: %s" % (file)) + return src_list, exclude_list + + +def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote, src_and_dst_remote=False): + info(u"Verifying attributes...") + cfg = Config() + exists_list = SortedDict(ignore_case=False) + + debug("Comparing filelists (src_is_local_and_dst_is_remote=%s)" % src_is_local_and_dst_is_remote) + debug("src_list.keys: %s" % src_list.keys()) + debug("dst_list.keys: %s" % dst_list.keys()) + + for file in src_list.keys(): + debug(u"CHECK: %s" % file) + if dst_list.has_key(file): + ## Was --skip-existing requested? + if cfg.skip_existing: + debug(u"IGNR: %s (used --skip-existing)" % (file)) + exists_list[file] = src_list[file] + del(src_list[file]) + ## Remove from destination-list, all that is left there will be deleted + del(dst_list[file]) + continue + + attribs_match = True + ## Check size first + if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: + debug( + u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) + attribs_match = False + + if attribs_match and 'md5' in cfg.sync_checks: + ## ... same size, check MD5 + if src_and_dst_remote: + src_md5 = src_list[file]['md5'] + dst_md5 = dst_list[file]['md5'] + elif src_is_local_and_dst_is_remote: + src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) + dst_md5 = dst_list[file]['md5'] + else: + src_md5 = src_list[file]['md5'] + dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) + if src_md5 != dst_md5: + ## Checksums are different. + attribs_match = False + debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) + + if attribs_match: + ## Remove from source-list, all that is left there will be transferred + debug(u"IGNR: %s (transfer not needed)" % file) + exists_list[file] = src_list[file] + del(src_list[file]) + + ## Remove from destination-list, all that is left there will be deleted + del(dst_list[file]) + + return src_list, dst_list, exists_list + def cmd_sync_remote2remote(args): - s3 = S3(Config()) - - destination_base = args[-1] - dest_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True) - source_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True) - - dest_count = len(dest_list) - source_count = len(source_list) - - info(u"Found %d source files, %d destination files" % (dest_count, source_count)) - - source_list, exclude_list = _filelist_filter_exclude_include(source_list) - - source_list, dest_list, existing_list = _compare_filelists(source_list, dest_list, False, True) - - dest_count = len(dest_list) - source_count = len(source_list) - - info(u"Summary: %d files to copy, %d files to delete" % (source_count, dest_count)) - - if cfg.recursive: - if not destination_base.endswith("/"): - destination_base += "/" - for key in source_list: - source_list[key]['dest_name'] = destination_base + key - else: - key = source_list.keys()[0] - if destination_base.endswith("/"): - source_list[key]['dest_name'] = destination_base + key - else: - source_list[key]['dest_name'] = destination_base - - if cfg.parallel and len(source_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for key in source_list: - seq += 1 - seq_label = "[%d of %d]" % (seq, source_count) - - item = source_list[key] - src_uri = S3Uri(item['object_uri_str']) - dst_uri = S3Uri(item['dest_name']) - extra_headers = copy(cfg.extra_headers) - q.put([src_uri,dst_uri,extra_headers,s3.object_copy,"File %(src)s copied to %(dst)s %(seq_label)s",seq_label]) - - for i in range(cfg.workers): - t = threading.Thread(target=cp_mv_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() - else: - seq = 0 - for key in source_list: - seq += 1 - seq_label = "[%d of %d]" % (seq, remote_count) + s3 = S3(Config()) + + destination_base = args[-1] + dest_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + source_list = fetch_remote_list(args[:-1], recursive=True, require_attribs=True) + + dest_count = len(dest_list) + source_count = len(source_list) - item = source_list[key] - src_uri = S3Uri(item['object_uri_str']) - dst_uri = S3Uri(item['dest_name']) + info(u"Found %d source files, %d destination files" % (dest_count, source_count)) + + source_list, exclude_list = _filelist_filter_exclude_include(source_list) + + source_list, dest_list, existing_list = _compare_filelists(source_list, dest_list, False, True) + + dest_count = len(dest_list) + source_count = len(source_list) + + info(u"Summary: %d files to copy, %d files to delete" % (source_count, dest_count)) + + if cfg.recursive: + if not destination_base.endswith("/"): + destination_base += "/" + for key in source_list: + source_list[key]['dest_name'] = destination_base + key + else: + key = source_list.keys()[0] + if destination_base.endswith("/"): + source_list[key]['dest_name'] = destination_base + key + else: + source_list[key]['dest_name'] = destination_base + + if cfg.parallel and len(source_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in source_list: + seq += 1 + seq_label = "[%d of %d]" % (seq, source_count) + + item = source_list[key] + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['dest_name']) + extra_headers = copy(cfg.extra_headers) + q.put([src_uri, dst_uri, extra_headers, s3.object_copy, "File %(src)s copied to %(dst)s %(seq_label)s", + seq_label]) + + for i in range(cfg.workers): + t = threading.Thread(target=cp_mv_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in source_list: + seq += 1 + seq_label = "[%d of %d]" % (seq, remote_count) + + item = source_list[key] + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['dest_name']) + + extra_headers = copy(cfg.extra_headers) + response = s3.object_copy(src_uri, dst_uri, extra_headers) + output("File %(src)s copied to %(dst)s" % {"src": src_uri, "dst": dst_uri}) + if Config().acl_public: + info(u"Public URL is: %s" % dst_uri.public_url()) - extra_headers = copy(cfg.extra_headers) - response = s3.object_copy(src_uri, dst_uri, extra_headers) - output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri }) - if Config().acl_public: - info(u"Public URL is: %s" % dst_uri.public_url()) def cmd_sync_remote2local(args): - s3 = S3(Config()) - - destination_base = args[-1] - local_list, single_file_local = fetch_local_list(destination_base, recursive = True) - remote_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True) - - local_count = len(local_list) - remote_count = len(remote_list) - - info(u"Found %d remote files, %d local files" % (remote_count, local_count)) - - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - - remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, False) - - local_count = len(local_list) - remote_count = len(remote_list) - - info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count)) - - if not os.path.isdir(destination_base): - ## We were either given a file name (existing or not) or want STDOUT - if remote_count > 1: - raise ParameterError("Destination must be a directory when downloading multiple sources.") - remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base) - else: - if destination_base[-1] != os.path.sep: - destination_base += os.path.sep - for key in remote_list: - local_filename = destination_base + key - if os.path.sep != "/": - local_filename = os.path.sep.join(local_filename.split("/")) - remote_list[key]['local_filename'] = deunicodise(local_filename) - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - if cfg.delete_removed: - for key in local_list: - output(u"delete: %s" % local_list[key]['full_name_unicode']) - for key in remote_list: - output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename'])) - - warning(u"Exitting now because of --dry-run") - return - - if cfg.delete_removed: - for key in local_list: - os.unlink(local_list[key]['full_name']) - output(u"deleted: %s" % local_list[key]['full_name_unicode']) - - total_size = 0 - total_elapsed = 0.0 - timestamp_start = time.time() - seq = 0 - file_list = remote_list.keys() - file_list.sort() - - build_dir_structure(remote_list) - - if cfg.parallel and len(file_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for file in file_list: - seq += 1 - q.put([remote_list[file],seq,len(file_list)]) - - for i in range(cfg.workers): - t = threading.Thread(target=sync_remote2local_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() - else: - for file in file_list: - seq += 1 - do_remote2local_work(remote_list[file],seq,len(file_list)) - - #Omitted due to threading - + s3 = S3(Config()) + + destination_base = args[-1] + local_list, single_file_local = fetch_local_list(destination_base, recursive=True) + remote_list = fetch_remote_list(args[:-1], recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Found %d remote files, %d local files" % (remote_count, local_count)) + + remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + + remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, False) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count)) + + if not os.path.isdir(destination_base): + ## We were either given a file name (existing or not) or want STDOUT + if remote_count > 1: + raise ParameterError("Destination must be a directory when downloading multiple sources.") + remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base) + else: + if destination_base[-1] != os.path.sep: + destination_base += os.path.sep + for key in remote_list: + local_filename = destination_base + key + if os.path.sep != "/": + local_filename = os.path.sep.join(local_filename.split("/")) + remote_list[key]['local_filename'] = deunicodise(local_filename) + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + if cfg.delete_removed: + for key in local_list: + output(u"delete: %s" % local_list[key]['full_name_unicode']) + for key in remote_list: + output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename'])) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.delete_removed: + for key in local_list: + os.unlink(local_list[key]['full_name']) + output(u"deleted: %s" % local_list[key]['full_name_unicode']) + + total_size = 0 + total_elapsed = 0.0 + timestamp_start = time.time() + seq = 0 + file_list = remote_list.keys() + file_list.sort() + + build_dir_structure(remote_list) + + if cfg.parallel and len(file_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for file in file_list: + seq += 1 + q.put([remote_list[file], seq, len(file_list)]) + + for i in range(cfg.workers): + t = threading.Thread(target=sync_remote2local_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + for file in file_list: + seq += 1 + do_remote2local_work(remote_list[file], seq, len(file_list)) + + #Omitted due to threading + + def sync_remote2local_worker(): - while True: - try: - (item,seq,total) = q.get_nowait() - except Queue.Empty: - return - try: - do_remote2local_work(item,seq,total) - except Exception, e: - report_exception(e) - exit - q.task_done() + while True: + try: + (item, seq, total) = q.get_nowait() + except Queue.Empty: + return + try: + do_remote2local_work(item, seq, total) + except Exception, e: + report_exception(e) + exit + q.task_done() + def build_dir_structure(file_list): - #Builds directory structure first to avoid race conditions with threading - dir_cache = {} - for item in file_list: - dst_file = file_list[item]['local_filename'] - dst_dir = os.path.dirname(dst_file) - if not dir_cache.has_key(dst_dir): - dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir) - if dir_cache[dst_dir] == False: - warning(u"%s: destination directory not writable: %s" % (file, dst_dir)) - continue - -def do_remote2local_work(item,seq,total): - def _parse_attrs_header(attrs_header): - attrs = {} - for attr in attrs_header.split("/"): - key, val = attr.split(":") - attrs[key] = val - return attrs - - s3 = S3(Config()) - uri = S3Uri(item['object_uri_str']) - dst_file = item['local_filename'] - seq_label = "[%d of %d]" % (seq, total) - try: - dst_dir = os.path.dirname(dst_file) - try: - open_flags = os.O_CREAT - open_flags |= os.O_TRUNC - # open_flags |= os.O_EXCL - - debug(u"dst_file=%s" % unicodise(dst_file)) - # This will have failed should the file exist - os.close(os.open(dst_file, open_flags)) - # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode. - dst_stream = open(dst_file, "wb") - if not cfg.progress_meter: - output(u"File '%s' started %s" % - (uri, seq_label)) - response = s3.object_get(uri, dst_stream, extra_label = seq_label) - dst_stream.close() - if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs: - attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) - if attrs.has_key('mode'): - os.chmod(dst_file, int(attrs['mode'])) - if attrs.has_key('mtime') or attrs.has_key('atime'): - mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time()) - atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time()) - os.utime(dst_file, (atime, mtime)) - ## FIXME: uid/gid / uname/gname handling comes here! TODO - except OSError, e: - try: dst_stream.close() - except: pass - if e.errno == errno.EEXIST: - warning(u"%s exists - not overwriting" % (dst_file)) - return - if e.errno in (errno.EPERM, errno.EACCES): - warning(u"%s not writable: %s" % (dst_file, e.strerror)) - return - raise e - - # We have to keep repeating this call because - # Python 2.4 doesn't support try/except/finally - # construction :-( - try: dst_stream.close() - except: pass - except S3DownloadError, e: - error(u"%s: download failed too many times. Skipping that file." % file) - return - speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - if not Config().progress_meter: - output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % - (uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq_label)) + #Builds directory structure first to avoid race conditions with threading + dir_cache = {} + for item in file_list: + dst_file = file_list[item]['local_filename'] + dst_dir = os.path.dirname(dst_file) + if not dir_cache.has_key(dst_dir): + dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir) + if dir_cache[dst_dir] == False: + warning(u"%s: destination directory not writable: %s" % (file, dst_dir)) + continue + + +def do_remote2local_work(item, seq, total): + def _parse_attrs_header(attrs_header): + attrs = {} + for attr in attrs_header.split("/"): + key, val = attr.split(":") + attrs[key] = val + return attrs + + s3 = S3(Config()) + uri = S3Uri(item['object_uri_str']) + dst_file = item['local_filename'] + seq_label = "[%d of %d]" % (seq, total) + try: + dst_dir = os.path.dirname(dst_file) + try: + open_flags = os.O_CREAT + open_flags |= os.O_TRUNC + # open_flags |= os.O_EXCL + + debug(u"dst_file=%s" % unicodise(dst_file)) + # This will have failed should the file exist + os.close(os.open(dst_file, open_flags)) + # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode. + dst_stream = open(dst_file, "wb") + if not cfg.progress_meter: + output(u"File '%s' started %s" % + (uri, seq_label)) + response = s3.object_get(uri, dst_stream, extra_label=seq_label) + dst_stream.close() + if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs: + attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) + if attrs.has_key('mode'): + os.chmod(dst_file, int(attrs['mode'])) + if attrs.has_key('mtime') or attrs.has_key('atime'): + mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time()) + atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time()) + os.utime(dst_file, (atime, mtime)) + ## FIXME: uid/gid / uname/gname handling comes here! TODO + except OSError, e: + try: dst_stream.close() + except: pass + if e.errno == errno.EEXIST: + warning(u"%s exists - not overwriting" % (dst_file)) + return + if e.errno in (errno.EPERM, errno.EACCES): + warning(u"%s not writable: %s" % (dst_file, e.strerror)) + return + raise e + + # We have to keep repeating this call because + # Python 2.4 doesn't support try/except/finally + # construction :-( + try: dst_stream.close() + except: pass + except S3DownloadError, e: + error(u"%s: download failed too many times. Skipping that file." % file) + return + speed_fmt = formatSize(response["speed"], human_readable=True, floating_point=True) + if not Config().progress_meter: + output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + (uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], + seq_label)) def cmd_sync_local2remote(args): + s3 = S3(cfg) + + if cfg.encrypt: + error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.") + error(u"Either use unconditional 's3cmd put --recursive'") + error(u"or disable encryption with --no-encrypt parameter.") + sys.exit(1) + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args[-1]) + if destination_base_uri.type != 's3': + raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + local_list, single_file_local = fetch_local_list(args[:-1], recursive=True) + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Found %d local files, %d remote files" % (local_count, remote_count)) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + if single_file_local and len(local_list) == 1 and len(remote_list) == 1: + ## Make remote_key same as local_key for comparison if we're dealing with only one file + remote_list_entry = remote_list[remote_list.keys()[0]] + # Flush remote_list, by the way + remote_list = {local_list.keys()[0]: remote_list_entry} + + local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count)) + + if local_count > 0: + ## Populate 'remote_uri' only if we've got something to upload + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + local_list[key]['remote_uri'] = unicodise(destination_base + key) + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + if cfg.delete_removed: + for key in remote_list: + output(u"delete: %s" % remote_list[key]['object_uri_str']) + for key in local_list: + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.delete_removed: + for key in remote_list: + uri = S3Uri(remote_list[key]['object_uri_str']) + s3.object_delete(uri) + output(u"deleted: '%s'" % uri) + + total_size = 0 + total_elapsed = 0.0 + timestamp_start = time.time() + seq = 0 + file_list = local_list.keys() + file_list.sort() + + if cfg.parallel and len(file_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for file in file_list: + seq += 1 + q.put([local_list[file], seq, len(file_list)]) + + for i in range(cfg.workers): + t = threading.Thread(target=sync_local2remote_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for file in file_list: + seq += 1 + do_local2remote_work(local_list[file], seq, len(file_list)) - s3 = S3(cfg) - - if cfg.encrypt: - error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.") - error(u"Either use unconditional 's3cmd put --recursive'") - error(u"or disable encryption with --no-encrypt parameter.") - sys.exit(1) - - ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args[-1]) - if destination_base_uri.type != 's3': - raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) - destination_base = str(destination_base_uri) - - local_list, single_file_local = fetch_local_list(args[:-1], recursive = True) - remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True) - - local_count = len(local_list) - remote_count = len(remote_list) - - info(u"Found %d local files, %d remote files" % (local_count, remote_count)) - - local_list, exclude_list = _filelist_filter_exclude_include(local_list) - - if single_file_local and len(local_list) == 1 and len(remote_list) == 1: - ## Make remote_key same as local_key for comparison if we're dealing with only one file - remote_list_entry = remote_list[remote_list.keys()[0]] - # Flush remote_list, by the way - remote_list = { local_list.keys()[0] : remote_list_entry } - - local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, True) - - local_count = len(local_list) - remote_count = len(remote_list) - - info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count)) - - if local_count > 0: - ## Populate 'remote_uri' only if we've got something to upload - if not destination_base.endswith("/"): - if not single_file_local: - raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") - local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) - else: - for key in local_list: - local_list[key]['remote_uri'] = unicodise(destination_base + key) - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - if cfg.delete_removed: - for key in remote_list: - output(u"delete: %s" % remote_list[key]['object_uri_str']) - for key in local_list: - output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri'])) - - warning(u"Exitting now because of --dry-run") - return - - if cfg.delete_removed: - for key in remote_list: - uri = S3Uri(remote_list[key]['object_uri_str']) - s3.object_delete(uri) - output(u"deleted: '%s'" % uri) - - total_size = 0 - total_elapsed = 0.0 - timestamp_start = time.time() - seq = 0 - file_list = local_list.keys() - file_list.sort() - - if cfg.parallel and len(file_list) > 1: - #Disabling progress metter for parallel downloads. - cfg.progress_meter = False - #Initialize Queue - global q - q = Queue.Queue() - - seq = 0 - for file in file_list: - seq += 1 - q.put([local_list[file],seq,len(file_list)]) - - for i in range(cfg.workers): - t = threading.Thread(target=sync_local2remote_worker) - t.daemon = True - t.start() - - #Necessary to ensure KeyboardInterrupt can actually kill - #Otherwise Queue.join() blocks until all queue elements have completed - while threading.activeCount() > 1: - time.sleep(.1) - - q.join() - else: - seq = 0 - for file in file_list: - seq += 1 - do_local2remote_work(local_list[file],seq,len(file_list)) - #Omitted summary data, difficult to handle with threading. - + def sync_local2remote_worker(): - while True: - try: - (item,seq,total) = q.get_nowait() - except Queue.Empty: - return - try: - do_local2remote_work(item,seq,total) - except Exception, e: - report_exception(e) - exit - q.task_done() - -def do_local2remote_work(item,seq,total): - def _build_attr_header(src): - import pwd, grp - attrs = {} - src = deunicodise(src) - st = os.stat_result(os.stat(src)) - for attr in cfg.preserve_attrs_list: - if attr == 'uname': - try: - val = pwd.getpwuid(st.st_uid).pw_name - except KeyError: - attr = "uid" - val = st.st_uid - warning(u"%s: Owner username not known. Storing UID=%d instead." % (unicodise(src), val)) - elif attr == 'gname': - try: - val = grp.getgrgid(st.st_gid).gr_name - except KeyError: - attr = "gid" - val = st.st_gid - warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (unicodise(src), val)) - else: - val = getattr(st, 'st_' + attr) - attrs[attr] = val - result = "" - for k in attrs: result += "%s:%s/" % (k, attrs[k]) - return { 'x-amz-meta-s3cmd-attrs' : result[:-1] } - - s3 = S3(cfg) - src = item['full_name'] - uri = S3Uri(item['remote_uri']) - seq_label = "[%d of %d]" % (seq, total) - extra_headers = copy(cfg.extra_headers) - if cfg.preserve_attrs: - attr_header = _build_attr_header(src) - debug(u"attr_header: %s" % attr_header) - extra_headers.update(attr_header) - if not cfg.progress_meter: - output(u"File '%s' started %s" % - (item['full_name_unicode'], seq_label)) - try: - response = s3.object_put(src, uri, extra_headers, extra_label = seq_label) - except S3UploadError, e: - error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode']) - return - except InvalidFileError, e: - warning(u"File can not be uploaded: %s" % e) - return - speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - if not cfg.progress_meter: - output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % - (item['full_name_unicode'], uri, response["size"], response["elapsed"], - speed_fmt[0], speed_fmt[1], seq_label)) + while True: + try: + (item, seq, total) = q.get_nowait() + except Queue.Empty: + return + try: + do_local2remote_work(item, seq, total) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def do_local2remote_work(item, seq, total): + def _build_attr_header(src): + import pwd, grp + + attrs = {} + src = deunicodise(src) + st = os.stat_result(os.stat(src)) + for attr in cfg.preserve_attrs_list: + if attr == 'uname': + try: + val = pwd.getpwuid(st.st_uid).pw_name + except KeyError: + attr = "uid" + val = st.st_uid + warning(u"%s: Owner username not known. Storing UID=%d instead." % (unicodise(src), val)) + elif attr == 'gname': + try: + val = grp.getgrgid(st.st_gid).gr_name + except KeyError: + attr = "gid" + val = st.st_gid + warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (unicodise(src), val)) + else: + val = getattr(st, 'st_' + attr) + attrs[attr] = val + result = "" + for k in attrs: result += "%s:%s/" % (k, attrs[k]) + return {'x-amz-meta-s3cmd-attrs': result[:-1]} + + s3 = S3(cfg) + src = item['full_name'] + uri = S3Uri(item['remote_uri']) + seq_label = "[%d of %d]" % (seq, total) + extra_headers = copy(cfg.extra_headers) + if cfg.preserve_attrs: + attr_header = _build_attr_header(src) + debug(u"attr_header: %s" % attr_header) + extra_headers.update(attr_header) + if not cfg.progress_meter: + output(u"File '%s' started %s" % + (item['full_name_unicode'], seq_label)) + try: + response = s3.object_put(src, uri, extra_headers, extra_label=seq_label) + except S3UploadError, e: + error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode']) + return + except InvalidFileError, e: + warning(u"File can not be uploaded: %s" % e) + return + speed_fmt = formatSize(response["speed"], human_readable=True, floating_point=True) + if not cfg.progress_meter: + output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + (item['full_name_unicode'], uri, response["size"], response["elapsed"], + speed_fmt[0], speed_fmt[1], seq_label)) + def cmd_sync(args): - if (len(args) < 2): - raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param']) + if (len(args) < 2): + raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param']) + + if S3Uri(args[0]).type == "file" and S3Uri(args[-1]).type == "s3": + return cmd_sync_local2remote(args) + if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file": + return cmd_sync_remote2local(args) + if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3": + return cmd_sync_remote2remote(args) + raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args)) - if S3Uri(args[0]).type == "file" and S3Uri(args[-1]).type == "s3": - return cmd_sync_local2remote(args) - if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file": - return cmd_sync_remote2local(args) - if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3": - return cmd_sync_remote2remote(args) - raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args)) def cmd_setacl(args): - def _update_acl(uri, seq_label = ""): - something_changed = False - acl = s3.get_acl(uri) - debug(u"acl: %s - %r" % (uri, acl.grantees)) - if cfg.acl_public == True: - if acl.isAnonRead(): - info(u"%s: already Public, skipping %s" % (uri, seq_label)) - else: - acl.grantAnonRead() - something_changed = True - elif cfg.acl_public == False: # we explicitely check for False, because it could be None - if not acl.isAnonRead(): - info(u"%s: already Private, skipping %s" % (uri, seq_label)) - else: - acl.revokeAnonRead() - something_changed = True - - # update acl with arguments - # grant first and revoke later, because revoke has priority - if cfg.acl_grants: - something_changed = True - for grant in cfg.acl_grants: - acl.grant(**grant); - - if cfg.acl_revokes: - something_changed = True - for revoke in cfg.acl_revokes: - acl.revoke(**revoke); - - if not something_changed: - return - - retsponse = s3.set_acl(uri, acl) - if retsponse['status'] == 200: - if cfg.acl_public in (True, False): - output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label)) - else: - output(u"%s: ACL updated" % uri) - - s3 = S3(cfg) - - set_to_acl = cfg.acl_public and "Public" or "Private" - - if not cfg.recursive: - old_args = args - args = [] - for arg in old_args: - uri = S3Uri(arg) - if not uri.has_object(): - if cfg.acl_public != None: - info("Setting bucket-level ACL for %s to %s" % (uri.uri(), set_to_acl)) - else: - info("Setting bucket-level ACL for %s" % (uri.uri())) - if not cfg.dry_run: - _update_acl(uri) - else: - args.append(arg) - - remote_list = fetch_remote_list(args) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - - remote_count = len(remote_list) - - info(u"Summary: %d remote files to update" % remote_count) - - if cfg.dry_run: - for key in exclude_list: - output(u"exclude: %s" % unicodise(key)) - for key in remote_list: - output(u"setacl: %s" % remote_list[key]['object_uri_str']) - - warning(u"Exitting now because of --dry-run") - return - - seq = 0 - for key in remote_list: - seq += 1 - seq_label = "[%d of %d]" % (seq, remote_count) - uri = S3Uri(remote_list[key]['object_uri_str']) - _update_acl(uri, seq_label) + def _update_acl(uri, seq_label=""): + something_changed = False + acl = s3.get_acl(uri) + debug(u"acl: %s - %r" % (uri, acl.grantees)) + if cfg.acl_public == True: + if acl.isAnonRead(): + info(u"%s: already Public, skipping %s" % (uri, seq_label)) + else: + acl.grantAnonRead() + something_changed = True + elif cfg.acl_public == False: # we explicitely check for False, because it could be None + if not acl.isAnonRead(): + info(u"%s: already Private, skipping %s" % (uri, seq_label)) + else: + acl.revokeAnonRead() + something_changed = True + + # update acl with arguments + # grant first and revoke later, because revoke has priority + if cfg.acl_grants: + something_changed = True + for grant in cfg.acl_grants: + acl.grant(**grant); + + if cfg.acl_revokes: + something_changed = True + for revoke in cfg.acl_revokes: + acl.revoke(**revoke); + + if not something_changed: + return + + retsponse = s3.set_acl(uri, acl) + if retsponse['status'] == 200: + if cfg.acl_public in (True, False): + output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label)) + else: + output(u"%s: ACL updated" % uri) + + s3 = S3(cfg) + + set_to_acl = cfg.acl_public and "Public" or "Private" + + if not cfg.recursive: + old_args = args + args = [] + for arg in old_args: + uri = S3Uri(arg) + if not uri.has_object(): + if cfg.acl_public != None: + info("Setting bucket-level ACL for %s to %s" % (uri.uri(), set_to_acl)) + else: + info("Setting bucket-level ACL for %s" % (uri.uri())) + if not cfg.dry_run: + _update_acl(uri) + else: + args.append(arg) + + remote_list = fetch_remote_list(args) + remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + + remote_count = len(remote_list) + + info(u"Summary: %d remote files to update" % remote_count) + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in remote_list: + output(u"setacl: %s" % remote_list[key]['object_uri_str']) + + warning(u"Exitting now because of --dry-run") + return + + seq = 0 + for key in remote_list: + seq += 1 + seq_label = "[%d of %d]" % (seq, remote_count) + uri = S3Uri(remote_list[key]['object_uri_str']) + _update_acl(uri, seq_label) + def cmd_accesslog(args): - s3 = S3(cfg) - bucket_uri = S3Uri(args.pop()) - if bucket_uri.object(): - raise ParameterError("Only bucket name is required for [accesslog] command") - if cfg.log_target_prefix == False: - accesslog, response = s3.set_accesslog(bucket_uri, enable = False) - elif cfg.log_target_prefix: - log_target_prefix_uri = S3Uri(cfg.log_target_prefix) - if log_target_prefix_uri.type != "s3": - raise ParameterError("--log-target-prefix must be a S3 URI") - accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public) - else: # cfg.log_target_prefix == None - accesslog = s3.get_accesslog(bucket_uri) - - output(u"Access logging for: %s" % bucket_uri.uri()) - output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled()) - if accesslog.isLoggingEnabled(): - output(u" Target prefix: %s" % accesslog.targetPrefix().uri()) - #output(u" Public Access: %s" % accesslog.isAclPublic()) - + s3 = S3(cfg) + bucket_uri = S3Uri(args.pop()) + if bucket_uri.object(): + raise ParameterError("Only bucket name is required for [accesslog] command") + if cfg.log_target_prefix == False: + accesslog, response = s3.set_accesslog(bucket_uri, enable=False) + elif cfg.log_target_prefix: + log_target_prefix_uri = S3Uri(cfg.log_target_prefix) + if log_target_prefix_uri.type != "s3": + raise ParameterError("--log-target-prefix must be a S3 URI") + accesslog, response = s3.set_accesslog(bucket_uri, enable=True, log_target_prefix_uri=log_target_prefix_uri, + acl_public=cfg.acl_public) + else: # cfg.log_target_prefix == None + accesslog = s3.get_accesslog(bucket_uri) + + output(u"Access logging for: %s" % bucket_uri.uri()) + output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled()) + if accesslog.isLoggingEnabled(): + output(u" Target prefix: %s" % accesslog.targetPrefix().uri()) + #output(u" Public Access: %s" % accesslog.isAclPublic()) + + def cmd_sign(args): - string_to_sign = args.pop() - debug("string-to-sign: %r" % string_to_sign) - signature = Utils.sign_string(string_to_sign) - output("Signature: %s" % signature) + string_to_sign = args.pop() + debug("string-to-sign: %r" % string_to_sign) + signature = Utils.sign_string(string_to_sign) + output("Signature: %s" % signature) + def cmd_fixbucket(args): - def _unescape(text): - ## - # Removes HTML or XML character references and entities from a text string. - # - # @param text The HTML (or XML) source text. - # @return The plain text, as a Unicode string, if necessary. - # - # From: http://effbot.org/zone/re-sub.htm#unescape-html - def _unescape_fixup(m): - text = m.group(0) - if not htmlentitydefs.name2codepoint.has_key('apos'): - htmlentitydefs.name2codepoint['apos'] = ord("'") - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) - except KeyError: - pass - return text # leave as is - return re.sub("&#?\w+;", _unescape_fixup, text) - - cfg.urlencoding_mode = "fixbucket" - s3 = S3(cfg) - - count = 0 - for arg in args: - culprit = S3Uri(arg) - if culprit.type != "s3": - raise ParameterError("Expecting S3Uri instead of: %s" % arg) - response = s3.bucket_list_noparse(culprit.bucket(), culprit.object(), recursive = True) - r_xent = re.compile("&#x[\da-fA-F]+;") - keys = re.findall("(.*?)", response['data'], re.MULTILINE) - debug("Keys: %r" % keys) - for key in keys: - if r_xent.search(key): - info("Fixing: %s" % key) - debug("Step 1: Transforming %s" % key) - key_bin = _unescape(key) - debug("Step 2: ... to %s" % key_bin) - key_new = replace_nonprintables(key_bin) - debug("Step 3: ... then to %s" % key_new) - src = S3Uri("s3://%s/%s" % (culprit.bucket(), key_bin)) - dst = S3Uri("s3://%s/%s" % (culprit.bucket(), key_new)) - resp_move = s3.object_move(src, dst) - if resp_move['status'] == 200: - output("File %r renamed to %s" % (key_bin, key_new)) - count += 1 - else: - error("Something went wrong for: %r" % key) - error("Please report the problem to s3tools-bugs@lists.sourceforge.net") - if count > 0: - warning("Fixed %d files' names. Their ACL were reset to Private." % count) - warning("Use 's3cmd setacl --acl-public s3://...' to make") - warning("them publicly readable if required.") + def _unescape(text): + ## + # Removes HTML or XML character references and entities from a text string. + # + # @param text The HTML (or XML) source text. + # @return The plain text, as a Unicode string, if necessary. + # + # From: http://effbot.org/zone/re-sub.htm#unescape-html + def _unescape_fixup(m): + text = m.group(0) + if not htmlentitydefs.name2codepoint.has_key('apos'): + htmlentitydefs.name2codepoint['apos'] = ord("'") + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + except KeyError: + pass + return text # leave as is + + return re.sub("&#?\w+;", _unescape_fixup, text) + + cfg.urlencoding_mode = "fixbucket" + s3 = S3(cfg) + + count = 0 + for arg in args: + culprit = S3Uri(arg) + if culprit.type != "s3": + raise ParameterError("Expecting S3Uri instead of: %s" % arg) + response = s3.bucket_list_noparse(culprit.bucket(), culprit.object(), recursive=True) + r_xent = re.compile("&#x[\da-fA-F]+;") + keys = re.findall("(.*?)", response['data'], re.MULTILINE) + debug("Keys: %r" % keys) + for key in keys: + if r_xent.search(key): + info("Fixing: %s" % key) + debug("Step 1: Transforming %s" % key) + key_bin = _unescape(key) + debug("Step 2: ... to %s" % key_bin) + key_new = replace_nonprintables(key_bin) + debug("Step 3: ... then to %s" % key_new) + src = S3Uri("s3://%s/%s" % (culprit.bucket(), key_bin)) + dst = S3Uri("s3://%s/%s" % (culprit.bucket(), key_new)) + resp_move = s3.object_move(src, dst) + if resp_move['status'] == 200: + output("File %r renamed to %s" % (key_bin, key_new)) + count += 1 + else: + error("Something went wrong for: %r" % key) + error("Please report the problem to s3tools-bugs@lists.sourceforge.net") + if count > 0: + warning("Fixed %d files' names. Their ACL were reset to Private." % count) + warning("Use 's3cmd setacl --acl-public s3://...' to make") + warning("them publicly readable if required.") + def resolve_list(lst, args): - retval = [] - for item in lst: - retval.append(item % args) - return retval - -def gpg_command(command, passphrase = ""): - debug("GPG command: " + " ".join(command)) - p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - p_stdout, p_stderr = p.communicate(passphrase + "\n") - debug("GPG output:") - for line in p_stdout.split("\n"): - debug("GPG: " + line) - p_exitcode = p.wait() - return p_exitcode + retval = [] + for item in lst: + retval.append(item % args) + return retval + + +def gpg_command(command, passphrase=""): + debug("GPG command: " + " ".join(command)) + p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p_stdout, p_stderr = p.communicate(passphrase + "\n") + debug("GPG output:") + for line in p_stdout.split("\n"): + debug("GPG: " + line) + p_exitcode = p.wait() + return p_exitcode + def gpg_encrypt(filename): - tmp_filename = Utils.mktmpfile() - args = { - "gpg_command" : cfg.gpg_command, - "passphrase_fd" : "0", - "input_file" : filename, - "output_file" : tmp_filename, - } - info(u"Encrypting file %(input_file)s to %(output_file)s..." % args) - command = resolve_list(cfg.gpg_encrypt.split(" "), args) - code = gpg_command(command, cfg.gpg_passphrase) - return (code, tmp_filename, "gpg") - -def gpg_decrypt(filename, gpgenc_header = "", in_place = True): - tmp_filename = Utils.mktmpfile(filename) - args = { - "gpg_command" : cfg.gpg_command, - "passphrase_fd" : "0", - "input_file" : filename, - "output_file" : tmp_filename, - } - info(u"Decrypting file %(input_file)s to %(output_file)s..." % args) - command = resolve_list(cfg.gpg_decrypt.split(" "), args) - code = gpg_command(command, cfg.gpg_passphrase) - if code == 0 and in_place: - debug(u"Renaming %s to %s" % (tmp_filename, filename)) - os.unlink(filename) - os.rename(tmp_filename, filename) - tmp_filename = filename - return (code, tmp_filename) + tmp_filename = Utils.mktmpfile() + args = { + "gpg_command": cfg.gpg_command, + "passphrase_fd": "0", + "input_file": filename, + "output_file": tmp_filename, + } + info(u"Encrypting file %(input_file)s to %(output_file)s..." % args) + command = resolve_list(cfg.gpg_encrypt.split(" "), args) + code = gpg_command(command, cfg.gpg_passphrase) + return (code, tmp_filename, "gpg") + + +def gpg_decrypt(filename, gpgenc_header="", in_place=True): + tmp_filename = Utils.mktmpfile(filename) + args = { + "gpg_command": cfg.gpg_command, + "passphrase_fd": "0", + "input_file": filename, + "output_file": tmp_filename, + } + info(u"Decrypting file %(input_file)s to %(output_file)s..." % args) + command = resolve_list(cfg.gpg_decrypt.split(" "), args) + code = gpg_command(command, cfg.gpg_passphrase) + if code == 0 and in_place: + debug(u"Renaming %s to %s" % (tmp_filename, filename)) + os.unlink(filename) + os.rename(tmp_filename, filename) + tmp_filename = filename + return (code, tmp_filename) + def run_configure(config_file): - cfg = Config() - options = [ - ("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3"), - ("secret_key", "Secret Key"), - ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"), - ("gpg_command", "Path to GPG program"), - ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy"), - ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't conect to S3 directly"), - ("proxy_port", "HTTP Proxy server port"), - ] - ## Option-specfic defaults - if getattr(cfg, "gpg_command") == "": - setattr(cfg, "gpg_command", find_executable("gpg")) - - if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"): - re_match=re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy")) - if re_match: - setattr(cfg, "proxy_host", re_match.groups()[1]) - setattr(cfg, "proxy_port", re_match.groups()[2]) - - try: - while 1: - output(u"\nEnter new values or accept defaults in brackets with Enter.") - output(u"Refer to user manual for detailed description of all options.") - for option in options: - prompt = option[1] - ## Option-specific handling - if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True: - setattr(cfg, option[0], "") - continue - if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "": - setattr(cfg, option[0], 0) - continue - - try: - val = getattr(cfg, option[0]) - if type(val) is bool: - val = val and "Yes" or "No" - if val not in (None, ""): - prompt += " [%s]" % val - except AttributeError: - pass - - if len(option) >= 3: - output(u"\n%s" % option[2]) - - val = raw_input(prompt + ": ") - if val != "": - if type(getattr(cfg, option[0])) is bool: - # Turn 'Yes' into True, everything else into False - val = val.lower().startswith('y') - setattr(cfg, option[0], val) - output(u"\nNew settings:") - for option in options: - output(u" %s: %s" % (option[1], getattr(cfg, option[0]))) - val = raw_input("\nTest access with supplied credentials? [Y/n] ") - if val.lower().startswith("y") or val == "": - try: - output(u"Please wait...") - S3(Config()).bucket_list("", "") - output(u"Success. Your access key and secret key worked fine :-)") - - output(u"\nNow verifying that encryption works...") - if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"): - output(u"Not configured. Never mind.") - else: - if not getattr(cfg, "gpg_command"): - raise Exception("Path to GPG program not set") - if not os.path.isfile(getattr(cfg, "gpg_command")): - raise Exception("GPG program not found") - filename = Utils.mktmpfile() - f = open(filename, "w") - f.write(os.sys.copyright) - f.close() - ret_enc = gpg_encrypt(filename) - ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False) - hash = [ - Utils.hash_file_md5(filename), - Utils.hash_file_md5(ret_enc[1]), - Utils.hash_file_md5(ret_dec[1]), - ] - os.unlink(filename) - os.unlink(ret_enc[1]) - os.unlink(ret_dec[1]) - if hash[0] == hash[2] and hash[0] != hash[1]: - output ("Success. Encryption and decryption worked fine :-)") - else: - raise Exception("Encryption verification error.") - - except Exception, e: - error(u"Test failed: %s" % (e)) - val = raw_input("\nRetry configuration? [Y/n] ") - if val.lower().startswith("y") or val == "": - continue - - - val = raw_input("\nSave settings? [y/N] ") - if val.lower().startswith("y"): - break - val = raw_input("Retry configuration? [Y/n] ") - if val.lower().startswith("n"): - raise EOFError() - - ## Overwrite existing config file, make it user-readable only - old_mask = os.umask(0077) - try: - os.remove(config_file) - except OSError, e: - if e.errno != errno.ENOENT: - raise - f = open(config_file, "w") - os.umask(old_mask) - cfg.dump_config(f) - f.close() - output(u"Configuration saved to '%s'" % config_file) - - except (EOFError, KeyboardInterrupt): - output(u"\nConfiguration aborted. Changes were NOT saved.") - return - - except IOError, e: - error(u"Writing config file failed: %s: %s" % (config_file, e.strerror)) - sys.exit(1) + cfg = Config() + options = [ + ("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3"), + ("secret_key", "Secret Key"), + ("gpg_passphrase", "Encryption password", + "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3") + , + ("gpg_command", "Path to GPG program"), + ("use_https", "Use HTTPS protocol", + "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy") + , + ("proxy_host", "HTTP Proxy server name", + "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't conect to S3 directly") + , + ("proxy_port", "HTTP Proxy server port"), + ] + ## Option-specfic defaults + if getattr(cfg, "gpg_command") == "": + setattr(cfg, "gpg_command", find_executable("gpg")) + + if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"): + re_match = re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy")) + if re_match: + setattr(cfg, "proxy_host", re_match.groups()[1]) + setattr(cfg, "proxy_port", re_match.groups()[2]) + + try: + while 1: + output(u"\nEnter new values or accept defaults in brackets with Enter.") + output(u"Refer to user manual for detailed description of all options.") + for option in options: + prompt = option[1] + ## Option-specific handling + if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True: + setattr(cfg, option[0], "") + continue + if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "": + setattr(cfg, option[0], 0) + continue + + try: + val = getattr(cfg, option[0]) + if type(val) is bool: + val = val and "Yes" or "No" + if val not in (None, ""): + prompt += " [%s]" % val + except AttributeError: + pass + + if len(option) >= 3: + output(u"\n%s" % option[2]) + + val = raw_input(prompt + ": ") + if val != "": + if type(getattr(cfg, option[0])) is bool: + # Turn 'Yes' into True, everything else into False + val = val.lower().startswith('y') + setattr(cfg, option[0], val) + output(u"\nNew settings:") + for option in options: + output(u" %s: %s" % (option[1], getattr(cfg, option[0]))) + val = raw_input("\nTest access with supplied credentials? [Y/n] ") + if val.lower().startswith("y") or val == "": + try: + output(u"Please wait...") + S3(Config()).bucket_list("", "") + output(u"Success. Your access key and secret key worked fine :-)") + + output(u"\nNow verifying that encryption works...") + if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"): + output(u"Not configured. Never mind.") + else: + if not getattr(cfg, "gpg_command"): + raise Exception("Path to GPG program not set") + if not os.path.isfile(getattr(cfg, "gpg_command")): + raise Exception("GPG program not found") + filename = Utils.mktmpfile() + f = open(filename, "w") + f.write(os.sys.copyright) + f.close() + ret_enc = gpg_encrypt(filename) + ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False) + hash = [ + Utils.hash_file_md5(filename), + Utils.hash_file_md5(ret_enc[1]), + Utils.hash_file_md5(ret_dec[1]), + ] + os.unlink(filename) + os.unlink(ret_enc[1]) + os.unlink(ret_dec[1]) + if hash[0] == hash[2] and hash[0] != hash[1]: + output("Success. Encryption and decryption worked fine :-)") + else: + raise Exception("Encryption verification error.") + + except Exception, e: + error(u"Test failed: %s" % (e)) + val = raw_input("\nRetry configuration? [Y/n] ") + if val.lower().startswith("y") or val == "": + continue + + val = raw_input("\nSave settings? [y/N] ") + if val.lower().startswith("y"): + break + val = raw_input("Retry configuration? [Y/n] ") + if val.lower().startswith("n"): + raise EOFError() + + ## Overwrite existing config file, make it user-readable only + old_mask = os.umask(0077) + try: + os.remove(config_file) + except OSError, e: + if e.errno != errno.ENOENT: + raise + f = open(config_file, "w") + os.umask(old_mask) + cfg.dump_config(f) + f.close() + output(u"Configuration saved to '%s'" % config_file) + + except (EOFError, KeyboardInterrupt): + output(u"\nConfiguration aborted. Changes were NOT saved.") + return + + except IOError, e: + error(u"Writing config file failed: %s: %s" % (config_file, e.strerror)) + sys.exit(1) + def process_patterns_from_file(fname, patterns_list): - try: - fn = open(fname, "rt") - except IOError, e: - error(e) - sys.exit(1) - for pattern in fn: - pattern = pattern.strip() - if re.match("^#", pattern) or re.match("^\s*$", pattern): - continue - debug(u"%s: adding rule: %s" % (fname, pattern)) - patterns_list.append(pattern) - - return patterns_list - -def process_patterns(patterns_list, patterns_from, is_glob, option_txt = ""): - """ - process_patterns(patterns, patterns_from, is_glob, option_txt = "") - Process --exclude / --include GLOB and REGEXP patterns. - 'option_txt' is 'exclude' / 'include' / 'rexclude' / 'rinclude' - Returns: patterns_compiled, patterns_text - """ - - patterns_compiled = [] - patterns_textual = {} - - if patterns_list is None: - patterns_list = [] - - if patterns_from: - ## Append patterns from glob_from - for fname in patterns_from: - debug(u"processing --%s-from %s" % (option_txt, fname)) - patterns_list = process_patterns_from_file(fname, patterns_list) - - for pattern in patterns_list: - debug(u"processing %s rule: %s" % (option_txt, patterns_list)) - if is_glob: - pattern = glob.fnmatch.translate(pattern) - r = re.compile(pattern) - patterns_compiled.append(r) - patterns_textual[r] = pattern - - return patterns_compiled, patterns_textual + try: + fn = open(fname, "rt") + except IOError, e: + error(e) + sys.exit(1) + for pattern in fn: + pattern = pattern.strip() + if re.match("^#", pattern) or re.match("^\s*$", pattern): + continue + debug(u"%s: adding rule: %s" % (fname, pattern)) + patterns_list.append(pattern) + + return patterns_list + + +def process_patterns(patterns_list, patterns_from, is_glob, option_txt=""): + """ + process_patterns(patterns, patterns_from, is_glob, option_txt = "") + Process --exclude / --include GLOB and REGEXP patterns. + 'option_txt' is 'exclude' / 'include' / 'rexclude' / 'rinclude' + Returns: patterns_compiled, patterns_text + """ + + patterns_compiled = [] + patterns_textual = {} + + if patterns_list is None: + patterns_list = [] + + if patterns_from: + ## Append patterns from glob_from + for fname in patterns_from: + debug(u"processing --%s-from %s" % (option_txt, fname)) + patterns_list = process_patterns_from_file(fname, patterns_list) + + for pattern in patterns_list: + debug(u"processing %s rule: %s" % (option_txt, patterns_list)) + if is_glob: + pattern = glob.fnmatch.translate(pattern) + r = re.compile(pattern) + patterns_compiled.append(r) + patterns_textual[r] = pattern + + return patterns_compiled, patterns_textual + def get_commands_list(): - return [ - {"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1}, - {"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1}, - {"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0}, - {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0}, - {"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2}, - {"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1}, - {"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, - #{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1}, - {"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2}, - {"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0}, - {"cmd":"info", "label":"Get various information about Buckets or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1}, - {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, - {"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2}, - {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1}, - {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1}, - {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1}, - {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1}, - - ## CloudFront commands - {"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.info, "argc":0}, - {"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"[cf://DIST_ID]", "func":CfCmd.info, "argc":0}, - {"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1}, - {"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1}, - {"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1}, - ] + return [ + {"cmd": "mb", "label": "Make bucket", "param": "s3://BUCKET", "func": cmd_bucket_create, "argc": 1}, + {"cmd": "rb", "label": "Remove bucket", "param": "s3://BUCKET", "func": cmd_bucket_delete, "argc": 1}, + {"cmd": "ls", "label": "List objects or buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_ls, + "argc": 0}, + {"cmd": "la", "label": "List all object in all buckets", "param": "", "func": cmd_buckets_list_all_all, + "argc": 0}, + {"cmd": "put", "label": "Put file into bucket", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", + "func": cmd_object_put, "argc": 2}, + {"cmd": "get", "label": "Get file from bucket", "param": "s3://BUCKET/OBJECT LOCAL_FILE", + "func": cmd_object_get, "argc": 1}, + {"cmd": "del", "label": "Delete file from bucket", "param": "s3://BUCKET/OBJECT", "func": cmd_object_del, + "argc": 1}, + #{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1}, + {"cmd": "sync", "label": "Synchronize a directory tree to S3", + "param": "LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func": cmd_sync, "argc": 2}, + {"cmd": "du", "label": "Disk usage by buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_du, + "argc": 0}, + {"cmd": "info", "label": "Get various information about Buckets or Files", "param": "s3://BUCKET[/OBJECT]", + "func": cmd_info, "argc": 1}, + {"cmd": "cp", "label": "Copy object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_cp + , "argc": 2}, + {"cmd": "mv", "label": "Move object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_mv + , "argc": 2}, + {"cmd": "setacl", "label": "Modify Access control list for Bucket or Files", "param": "s3://BUCKET[/OBJECT]" + , "func": cmd_setacl, "argc": 1}, + {"cmd": "accesslog", "label": "Enable/disable bucket access logging", "param": "s3://BUCKET", + "func": cmd_accesslog, "argc": 1}, + {"cmd": "sign", "label": "Sign arbitrary string using the secret key", "param": "STRING-TO-SIGN", + "func": cmd_sign, "argc": 1}, + {"cmd": "fixbucket", "label": "Fix invalid file names in a bucket", "param": "s3://BUCKET[/PREFIX]", + "func": cmd_fixbucket, "argc": 1}, + + ## CloudFront commands + {"cmd": "cflist", "label": "List CloudFront distribution points", "param": "", "func": CfCmd.info, + "argc": 0}, + {"cmd": "cfinfo", "label": "Display CloudFront distribution point parameters", "param": "[cf://DIST_ID]", + "func": CfCmd.info, "argc": 0}, + {"cmd": "cfcreate", "label": "Create CloudFront distribution point", "param": "s3://BUCKET", + "func": CfCmd.create, "argc": 1}, + {"cmd": "cfdelete", "label": "Delete CloudFront distribution point", "param": "cf://DIST_ID", + "func": CfCmd.delete, "argc": 1}, + {"cmd": "cfmodify", "label": "Change CloudFront distribution point parameters", "param": "cf://DIST_ID", + "func": CfCmd.modify, "argc": 1}, + ] + def format_commands(progname, commands_list): - help = "Commands:\n" - for cmd in commands_list: - help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"]) - return help + help = "Commands:\n" + for cmd in commands_list: + help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"]) + return help + class OptionMimeType(Option): - def check_mimetype(option, opt, value): - if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value): - return value - raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value)) + def check_mimetype(option, opt, value): + if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value): + return value + raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value)) + class OptionS3ACL(Option): - def check_s3acl(option, opt, value): - permissions = ('read', 'write', 'read_acp', 'write_acp', 'full_control', 'all') - try: - permission, grantee = re.compile("^(\w+):(.+)$", re.IGNORECASE).match(value).groups() - if not permission or not grantee: - raise - if permission in permissions: - return { 'name' : grantee, 'permission' : permission.upper() } - else: - raise OptionValueError("option %s: invalid S3 ACL permission: %s (valid values: %s)" % - (opt, permission, ", ".join(permissions))) - except: - raise OptionValueError("option %s: invalid S3 ACL format: %r" % (opt, value)) + def check_s3acl(option, opt, value): + permissions = ('read', 'write', 'read_acp', 'write_acp', 'full_control', 'all') + try: + permission, grantee = re.compile("^(\w+):(.+)$", re.IGNORECASE).match(value).groups() + if not permission or not grantee: + raise + if permission in permissions: + return {'name': grantee, 'permission': permission.upper()} + else: + raise OptionValueError("option %s: invalid S3 ACL permission: %s (valid values: %s)" % + (opt, permission, ", ".join(permissions))) + except: + raise OptionValueError("option %s: invalid S3 ACL format: %r" % (opt, value)) + class OptionAll(OptionMimeType, OptionS3ACL): - TYPE_CHECKER = copy(Option.TYPE_CHECKER) - TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype - TYPE_CHECKER["s3acl"] = OptionS3ACL.check_s3acl - TYPES = Option.TYPES + ("mimetype", "s3acl") + TYPE_CHECKER = copy(Option.TYPE_CHECKER) + TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype + TYPE_CHECKER["s3acl"] = OptionS3ACL.check_s3acl + TYPES = Option.TYPES + ("mimetype", "s3acl") + class MyHelpFormatter(IndentedHelpFormatter): - def format_epilog(self, epilog): - if epilog: - return "\n" + epilog + "\n" - else: - return "" + def format_epilog(self, epilog): + if epilog: + return "\n" + epilog + "\n" + else: + return "" + def main(): - global cfg - - commands_list = get_commands_list() - commands = {} - - ## Populate "commands" from "commands_list" - for cmd in commands_list: - if cmd.has_key("cmd"): - commands[cmd["cmd"]] = cmd - - default_verbosity = Config().verbosity - optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter()) - #optparser.disable_interspersed_args() - - config_file = None - if os.getenv("HOME"): - config_file = os.path.join(os.getenv("HOME"), ".s3cfg") - elif os.name == "nt" and os.getenv("USERPROFILE"): - config_file = os.path.join(os.getenv("USERPROFILE"), "Application Data", "s3cmd.ini") - - preferred_encoding = locale.getpreferredencoding() or "UTF-8" - - optparser.set_defaults(encoding = preferred_encoding) - optparser.set_defaults(config = config_file) - optparser.set_defaults(verbosity = default_verbosity) - - optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.") - optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default") - optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.") - - optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)") - - optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.") - optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.") - optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") - optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).") - optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).") - optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") - optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") - optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") - optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all") - optparser.add_option( "--acl-revoke", dest="acl_revokes", type="s3acl", action="append", metavar="PERMISSION:USER_CANONICAL_ID", help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr ite_acp, full_control, all") - - optparser.add_option( "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]") - optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.") - optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.") - optparser.add_option( "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes") - optparser.add_option( "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync") - optparser.add_option( "--exclude-from", dest="exclude_from", action="append", metavar="FILE", help="Read --exclude GLOBs from FILE") - optparser.add_option( "--rexclude", dest="rexclude", action="append", metavar="REGEXP", help="Filenames and paths matching REGEXP (regular expression) will be excluded from sync") - optparser.add_option( "--rexclude-from", dest="rexclude_from", action="append", metavar="FILE", help="Read --rexclude REGEXPs from FILE") - optparser.add_option( "--include", dest="include", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns") - optparser.add_option( "--include-from", dest="include_from", action="append", metavar="FILE", help="Read --include GLOBs from FILE") - optparser.add_option( "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB") - optparser.add_option( "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE") - - optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1") - optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]") - - optparser.add_option( "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)") - optparser.add_option( "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)") - - optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.") - optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option") - - optparser.add_option( "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.") - - optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding) - optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!") - - optparser.add_option( "--list-md5", dest="list_md5", action="store_true", help="Include MD5 sums in bucket listings (only for 'ls' command).") - optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form (eg 1kB instead of 1234).") - - optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).") - optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).") - optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)") - optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)") - optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") - optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)") - optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") - optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") - optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") - optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) - optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files") - - optparser.add_option( "--parallel", dest="parallel", action="store_true", help="Download and upload files in parallel.") - optparser.add_option( "--workers", dest="workers", default=10, help="Sets the number of workers to run for uploading and downloading files (can only be used in conjunction with the --parallel argument)") - - optparser.add_option( "--directory", dest="select_dir", action="store_true", default=False, help="Select directories (only for [ls]).") - optparser.add_option( "--max-retries", dest="max_retries", type="int", action="store", default=5, help="Number of retry before failing GET or PUT.") - optparser.add_option( "--retry-delay", dest="retry_delay", type="int", action="store", default=3, help="Time delay to wait after failing GET or PUT.") - - optparser.set_usage(optparser.usage + " COMMAND [parameters]") - optparser.set_description('S3cmd is a tool for managing objects in '+ - 'Amazon S3 storage. It allows for making and removing '+ - '"buckets" and uploading, downloading and removing '+ - '"objects" from these buckets.') - optparser.epilog = format_commands(optparser.get_prog_name(), commands_list) - optparser.epilog += ("\nSee program homepage for more information at\n%s\n" % PkgInfo.url) - - (options, args) = optparser.parse_args() - - ## Some mucking with logging levels to enable - ## debugging/verbose output for config file parser on request - logging.basicConfig(level=options.verbosity, - format='%(levelname)s: %(message)s', - stream = sys.stderr) - - if options.show_version: - output(u"s3cmd version %s" % PkgInfo.version) - sys.exit(0) - - ## Now finally parse the config file - if not options.config: - error(u"Can't find a config file. Please use --config option.") - sys.exit(1) - - try: - cfg = Config(options.config) - except IOError, e: - if options.run_configure: - cfg = Config() - else: - error(u"%s: %s" % (options.config, e.strerror)) - error(u"Configuration file not available.") - error(u"Consider using --configure parameter to create one.") - sys.exit(1) - - ## And again some logging level adjustments - ## according to configfile and command line parameters - if options.verbosity != default_verbosity: - cfg.verbosity = options.verbosity - logging.root.setLevel(cfg.verbosity) - - ## Default to --progress on TTY devices, --no-progress elsewhere - ## Can be overriden by actual --(no-)progress parameter - cfg.update_option('progress_meter', sys.stdout.isatty()) - - ## Unsupported features on Win32 platform - if os.name == "nt": - if cfg.preserve_attrs: - error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.") - cfg.preserve_attrs = False - if cfg.progress_meter: - error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.") - cfg.progress_meter = False - - ## Pre-process --add-header's and put them to Config.extra_headers SortedDict() - if options.add_header: - for hdr in options.add_header: - try: - key, val = hdr.split(":", 1) - except ValueError: - raise ParameterError("Invalid header format: %s" % hdr) - key_inval = re.sub("[a-zA-Z0-9-.]", "", key) - if key_inval: - key_inval = key_inval.replace(" ", "") - key_inval = key_inval.replace("\t", "") - raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval)) - debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip(), val.strip())) - cfg.extra_headers[key.strip()] = val.strip() - - ## --acl-grant/--acl-revoke arguments are pre-parsed by OptionS3ACL() - if options.acl_grants: - for grant in options.acl_grants: - cfg.acl_grants.append(grant) - - if options.acl_revokes: - for grant in options.acl_revokes: - cfg.acl_revokes.append(grant) - - ## Update Config with other parameters - for option in cfg.option_list(): - try: - if getattr(options, option) != None: - debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option))) - cfg.update_option(option, getattr(options, option)) - except AttributeError: - ## Some Config() options are not settable from command line - pass - - ## Special handling for tri-state options (True, False, None) - cfg.update_option("enable", options.enable) - cfg.update_option("acl_public", options.acl_public) - - ## CloudFront's cf_enable and Config's enable share the same --enable switch - options.cf_enable = options.enable - - ## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch - options.cf_logging = options.log_target_prefix - - ## Update CloudFront options if some were set - for option in CfCmd.options.option_list(): - try: - if getattr(options, option) != None: - debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option))) - CfCmd.options.update_option(option, getattr(options, option)) - except AttributeError: - ## Some CloudFront.Cmd.Options() options are not settable from command line - pass - - ## Set output and filesystem encoding for printing out filenames. - sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace") - sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace") - - ## Process --exclude and --exclude-from - patterns_list, patterns_textual = process_patterns(options.exclude, options.exclude_from, is_glob = True, option_txt = "exclude") - cfg.exclude.extend(patterns_list) - cfg.debug_exclude.update(patterns_textual) - - ## Process --rexclude and --rexclude-from - patterns_list, patterns_textual = process_patterns(options.rexclude, options.rexclude_from, is_glob = False, option_txt = "rexclude") - cfg.exclude.extend(patterns_list) - cfg.debug_exclude.update(patterns_textual) - - ## Process --include and --include-from - patterns_list, patterns_textual = process_patterns(options.include, options.include_from, is_glob = True, option_txt = "include") - cfg.include.extend(patterns_list) - cfg.debug_include.update(patterns_textual) - - ## Process --rinclude and --rinclude-from - patterns_list, patterns_textual = process_patterns(options.rinclude, options.rinclude_from, is_glob = False, option_txt = "rinclude") - cfg.include.extend(patterns_list) - cfg.debug_include.update(patterns_textual) - - ## Process --follow-symlinks - cfg.update_option("follow_symlinks", options.follow_symlinks) - - if cfg.encrypt and cfg.gpg_passphrase == "": - error(u"Encryption requested but no passphrase set in config file.") - error(u"Please re-run 's3cmd --configure' and supply it.") - sys.exit(1) - - if options.dump_config: - cfg.dump_config(sys.stdout) - sys.exit(0) - - if options.run_configure: - run_configure(options.config) - sys.exit(0) - - if len(args) < 1: - error(u"Missing command. Please run with --help for more information.") - sys.exit(1) - - ## Unicodise all remaining arguments: - args = [unicodise(arg) for arg in args] - - command = args.pop(0) - try: - debug(u"Command: %s" % commands[command]["cmd"]) - ## We must do this lookup in extra step to - ## avoid catching all KeyError exceptions - ## from inner functions. - cmd_func = commands[command]["func"] - except KeyError, e: - error(u"Invalid command: %s" % e) - sys.exit(1) - - if len(args) < commands[command]["argc"]: - error(u"Not enough paramters for command '%s'" % command) - sys.exit(1) - - try: - cmd_func(args) - except S3Error, e: - error(u"S3 error: %s" % e) - sys.exit(1) + global cfg + + commands_list = get_commands_list() + commands = {} + + ## Populate "commands" from "commands_list" + for cmd in commands_list: + if cmd.has_key("cmd"): + commands[cmd["cmd"]] = cmd + + default_verbosity = Config().verbosity + optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter()) + #optparser.disable_interspersed_args() + + config_file = None + if os.getenv("HOME"): + config_file = os.path.join(os.getenv("HOME"), ".s3cfg") + elif os.name == "nt" and os.getenv("USERPROFILE"): + config_file = os.path.join(os.getenv("USERPROFILE"), "Application Data", "s3cmd.ini") + + preferred_encoding = locale.getpreferredencoding() or "UTF-8" + + optparser.set_defaults(encoding=preferred_encoding) + optparser.set_defaults(config=config_file) + optparser.set_defaults(verbosity=default_verbosity) + + optparser.add_option("--configure", dest="run_configure", action="store_true", + help="Invoke interactive (re)configuration tool.") + optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default") + optparser.add_option("--dump-config", dest="dump_config", action="store_true", + help="Dump current configuration after parsing config files and command line options and exit.") + + optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", + help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)") + + optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", + help="Encrypt files before uploading to S3.") + optparser.add_option("--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.") + optparser.add_option("-f", "--force", dest="force", action="store_true", + help="Force overwrite and other dangerous operations.") + optparser.add_option("--continue", dest="get_continue", action="store_true", + help="Continue getting a partially downloaded file (only for [get] command).") + optparser.add_option("--skip-existing", dest="skip_existing", action="store_true", + help="Skip over files that exist at the destination (only for [get] and [sync] commands).") + optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", + help="Recursive upload, download or removal.") + optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", + help="Store objects with ACL allowing read for anyone.") + optparser.add_option("--acl-private", dest="acl_public", action="store_false", + help="Store objects with default ACL allowing access for you only.") + optparser.add_option("--acl-grant", dest="acl_grants", type="s3acl", action="append", + metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", + help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all") + optparser.add_option("--acl-revoke", dest="acl_revokes", type="s3acl", action="append", + metavar="PERMISSION:USER_CANONICAL_ID", + help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr ite_acp, full_control, all") + + optparser.add_option("--delete-removed", dest="delete_removed", action="store_true", + help="Delete remote objects with no corresponding local file [sync]") + optparser.add_option("--no-delete-removed", dest="delete_removed", action="store_false", + help="Don't delete remote objects.") + optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", + help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.") + optparser.add_option("--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes") + optparser.add_option("--exclude", dest="exclude", action="append", metavar="GLOB", + help="Filenames and paths matching GLOB will be excluded from sync") + optparser.add_option("--exclude-from", dest="exclude_from", action="append", metavar="FILE", + help="Read --exclude GLOBs from FILE") + optparser.add_option("--rexclude", dest="rexclude", action="append", metavar="REGEXP", + help="Filenames and paths matching REGEXP (regular expression) will be excluded from sync") + optparser.add_option("--rexclude-from", dest="rexclude_from", action="append", metavar="FILE", + help="Read --rexclude REGEXPs from FILE") + optparser.add_option("--include", dest="include", action="append", metavar="GLOB", + help="Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns") + optparser.add_option("--include-from", dest="include_from", action="append", metavar="FILE", + help="Read --include GLOBs from FILE") + optparser.add_option("--rinclude", dest="rinclude", action="append", metavar="REGEXP", + help="Same as --include but uses REGEXP (regular expression) instead of GLOB") + optparser.add_option("--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", + help="Read --rinclude REGEXPs from FILE") + + optparser.add_option("--bucket-location", dest="bucket_location", + help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1") + optparser.add_option("--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", + help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]") + + optparser.add_option("--access-logging-target-prefix", dest="log_target_prefix", + help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)") + optparser.add_option("--no-access-logging", dest="log_target_prefix", action="store_false", + help="Disable access logging (for [cfmodify] and [accesslog] commands)") + + optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", + help="Default MIME-type to be set for objects stored.") + optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", + help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option") + + optparser.add_option("--add-header", dest="add_header", action="append", metavar="NAME:VALUE", + help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.") + + optparser.add_option("--encoding", dest="encoding", metavar="ENCODING", + help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding) + optparser.add_option("--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", + help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!") + + optparser.add_option("--list-md5", dest="list_md5", action="store_true", + help="Include MD5 sums in bucket listings (only for 'ls' command).") + optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", + help="Print sizes in human readable form (eg 1kB instead of 1234).") + + optparser.add_option("--progress", dest="progress_meter", action="store_true", + help="Display progress meter (default on TTY).") + optparser.add_option("--no-progress", dest="progress_meter", action="store_false", + help="Don't display progress meter (default on non-TTY).") + optparser.add_option("--enable", dest="enable", action="store_true", + help="Enable given CloudFront distribution (only for [cfmodify] command)") + optparser.add_option("--disable", dest="enable", action="store_false", + help="Enable given CloudFront distribution (only for [cfmodify] command)") + optparser.add_option("--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", + help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") + optparser.add_option("--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", + help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)") + optparser.add_option("--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", + help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") + optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, + help="Enable verbose output.") + optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, + help="Enable debug output.") + optparser.add_option("--version", dest="show_version", action="store_true", + help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) + optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, + help="Follow symbolic links as if they are regular files") + + optparser.add_option("--parallel", dest="parallel", action="store_true", + help="Download and upload files in parallel.") + optparser.add_option("--workers", dest="workers", default=10, + help="Sets the number of workers to run for uploading and downloading files (can only be used in conjunction with the --parallel argument)") + + optparser.add_option("--directory", dest="select_dir", action="store_true", default=False, + help="Select directories (only for [ls]).") + optparser.add_option("--max-retries", dest="max_retries", type="int", action="store", default=5, + help="Number of retry before failing GET or PUT.") + optparser.add_option("--retry-delay", dest="retry_delay", type="int", action="store", default=3, + help="Time delay to wait after failing GET or PUT.") + + optparser.set_usage(optparser.usage + " COMMAND [parameters]") + optparser.set_description('S3cmd is a tool for managing objects in ' + + 'Amazon S3 storage. It allows for making and removing ' + + '"buckets" and uploading, downloading and removing ' + + '"objects" from these buckets.') + optparser.epilog = format_commands(optparser.get_prog_name(), commands_list) + optparser.epilog += ("\nSee program homepage for more information at\n%s\n" % PkgInfo.url) + + (options, args) = optparser.parse_args() + + ## Some mucking with logging levels to enable + ## debugging/verbose output for config file parser on request + logging.basicConfig(level=options.verbosity, + format='%(levelname)s: %(message)s', + stream=sys.stderr) + + if options.show_version: + output(u"s3cmd version %s" % PkgInfo.version) + sys.exit(0) + + ## Now finally parse the config file + if not options.config: + error(u"Can't find a config file. Please use --config option.") + sys.exit(1) + + try: + cfg = Config(options.config) + except IOError, e: + if options.run_configure: + cfg = Config() + else: + error(u"%s: %s" % (options.config, e.strerror)) + error(u"Configuration file not available.") + error(u"Consider using --configure parameter to create one.") + sys.exit(1) + + ## And again some logging level adjustments + ## according to configfile and command line parameters + if options.verbosity != default_verbosity: + cfg.verbosity = options.verbosity + logging.root.setLevel(cfg.verbosity) + + ## Default to --progress on TTY devices, --no-progress elsewhere + ## Can be overriden by actual --(no-)progress parameter + cfg.update_option('progress_meter', sys.stdout.isatty()) + + ## Unsupported features on Win32 platform + if os.name == "nt": + if cfg.preserve_attrs: + error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.") + cfg.preserve_attrs = False + if cfg.progress_meter: + error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.") + cfg.progress_meter = False + + ## Pre-process --add-header's and put them to Config.extra_headers SortedDict() + if options.add_header: + for hdr in options.add_header: + try: + key, val = hdr.split(":", 1) + except ValueError: + raise ParameterError("Invalid header format: %s" % hdr) + key_inval = re.sub("[a-zA-Z0-9-.]", "", key) + if key_inval: + key_inval = key_inval.replace(" ", "") + key_inval = key_inval.replace("\t", "") + raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval)) + debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip(), val.strip())) + cfg.extra_headers[key.strip()] = val.strip() + + ## --acl-grant/--acl-revoke arguments are pre-parsed by OptionS3ACL() + if options.acl_grants: + for grant in options.acl_grants: + cfg.acl_grants.append(grant) + + if options.acl_revokes: + for grant in options.acl_revokes: + cfg.acl_revokes.append(grant) + + ## Update Config with other parameters + for option in cfg.option_list(): + try: + if getattr(options, option) != None: + debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option))) + cfg.update_option(option, getattr(options, option)) + except AttributeError: + ## Some Config() options are not settable from command line + pass + + ## Special handling for tri-state options (True, False, None) + cfg.update_option("enable", options.enable) + cfg.update_option("acl_public", options.acl_public) + + ## CloudFront's cf_enable and Config's enable share the same --enable switch + options.cf_enable = options.enable + + ## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch + options.cf_logging = options.log_target_prefix + + ## Update CloudFront options if some were set + for option in CfCmd.options.option_list(): + try: + if getattr(options, option) != None: + debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option))) + CfCmd.options.update_option(option, getattr(options, option)) + except AttributeError: + ## Some CloudFront.Cmd.Options() options are not settable from command line + pass + + ## Set output and filesystem encoding for printing out filenames. + sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace") + sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace") + + ## Process --exclude and --exclude-from + patterns_list, patterns_textual = process_patterns(options.exclude, options.exclude_from, is_glob=True, + option_txt="exclude") + cfg.exclude.extend(patterns_list) + cfg.debug_exclude.update(patterns_textual) + + ## Process --rexclude and --rexclude-from + patterns_list, patterns_textual = process_patterns(options.rexclude, options.rexclude_from, is_glob=False, + option_txt="rexclude") + cfg.exclude.extend(patterns_list) + cfg.debug_exclude.update(patterns_textual) + + ## Process --include and --include-from + patterns_list, patterns_textual = process_patterns(options.include, options.include_from, is_glob=True, + option_txt="include") + cfg.include.extend(patterns_list) + cfg.debug_include.update(patterns_textual) + + ## Process --rinclude and --rinclude-from + patterns_list, patterns_textual = process_patterns(options.rinclude, options.rinclude_from, is_glob=False, + option_txt="rinclude") + cfg.include.extend(patterns_list) + cfg.debug_include.update(patterns_textual) + + ## Process --follow-symlinks + cfg.update_option("follow_symlinks", options.follow_symlinks) + + if cfg.encrypt and cfg.gpg_passphrase == "": + error(u"Encryption requested but no passphrase set in config file.") + error(u"Please re-run 's3cmd --configure' and supply it.") + sys.exit(1) + + if options.dump_config: + cfg.dump_config(sys.stdout) + sys.exit(0) + + if options.run_configure: + run_configure(options.config) + sys.exit(0) + + if len(args) < 1: + error(u"Missing command. Please run with --help for more information.") + sys.exit(1) + + ## Unicodise all remaining arguments: + args = [unicodise(arg) for arg in args] + + command = args.pop(0) + try: + debug(u"Command: %s" % commands[command]["cmd"]) + ## We must do this lookup in extra step to + ## avoid catching all KeyError exceptions + ## from inner functions. + cmd_func = commands[command]["func"] + except KeyError, e: + error(u"Invalid command: %s" % e) + sys.exit(1) + + if len(args) < commands[command]["argc"]: + error(u"Not enough paramters for command '%s'" % command) + sys.exit(1) + + try: + cmd_func(args) + except S3Error, e: + error(u"S3 error: %s" % e) + sys.exit(1) + def report_exception(e): - sys.stderr.write(""" + sys.stderr.write(""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! An unexpected error has occurred. Please report the following lines to: @@ -2159,25 +2299,25 @@ def report_exception(e): !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """) - tb = traceback.format_exc(sys.exc_info()) - e_class = str(e.__class__) - e_class = e_class[e_class.rfind(".")+1 : -2] - sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) - try: - sys.stderr.write("S3cmd: %s\n" % PkgInfo.version) - except NameError: - sys.stderr.write("S3cmd: unknown version. Module import problem?\n") - sys.stderr.write("\n") - sys.stderr.write(unicode(tb, errors="replace")) - - if type(e) == ImportError: - sys.stderr.write("\n") - sys.stderr.write("Your sys.path contains these entries:\n") - for path in sys.path: - sys.stderr.write(u"\t%s\n" % path) - sys.stderr.write("Now the question is where have the s3cmd modules been installed?\n") - - sys.stderr.write(""" + tb = traceback.format_exc(sys.exc_info()) + e_class = str(e.__class__) + e_class = e_class[e_class.rfind(".") + 1: -2] + sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) + try: + sys.stderr.write("S3cmd: %s\n" % PkgInfo.version) + except NameError: + sys.stderr.write("S3cmd: unknown version. Module import problem?\n") + sys.stderr.write("\n") + sys.stderr.write(unicode(tb, errors="replace")) + + if type(e) == ImportError: + sys.stderr.write("\n") + sys.stderr.write("Your sys.path contains these entries:\n") + for path in sys.path: + sys.stderr.write(u"\t%s\n" % path) + sys.stderr.write("Now the question is where have the s3cmd modules been installed?\n") + + sys.stderr.write(""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! An unexpected error has occurred. Please report the above lines to: @@ -2186,40 +2326,40 @@ def report_exception(e): """) if __name__ == '__main__': - try: - ## Our modules - ## Keep them in try/except block to - ## detect any syntax errors in there - from S3.Exceptions import * - from S3 import PkgInfo - from S3.S3 import S3 - from S3.Config import Config - from S3.SortedDict import SortedDict - from S3.S3Uri import S3Uri - from S3 import Utils - from S3.Utils import * - from S3.Progress import Progress - from S3.CloudFront import Cmd as CfCmd - - main() - sys.exit(0) - - except ImportError, e: - report_exception(e) - sys.exit(1) - - except ParameterError, e: - error(u"Parameter problem: %s" % e) - sys.exit(1) - - except SystemExit, e: - sys.exit(e.code) - - except KeyboardInterrupt: - sys.stderr.write("See ya!\n") - - sys.exit(1) - - except Exception, e: - report_exception(e) - sys.exit(1) + try: + ## Our modules + ## Keep them in try/except block to + ## detect any syntax errors in there + from S3.Exceptions import * + from S3 import PkgInfo + from S3.S3 import S3 + from S3.Config import Config + from S3.SortedDict import SortedDict + from S3.S3Uri import S3Uri + from S3 import Utils + from S3.Utils import * + from S3.Progress import Progress + from S3.CloudFront import Cmd as CfCmd + + main() + sys.exit(0) + + except ImportError, e: + report_exception(e) + sys.exit(1) + + except ParameterError, e: + error(u"Parameter problem: %s" % e) + sys.exit(1) + + except SystemExit, e: + sys.exit(e.code) + + except KeyboardInterrupt: + sys.stderr.write("See ya!\n") + + sys.exit(1) + + except Exception, e: + report_exception(e) + sys.exit(1) From 5b351ccf788686fc72212894e07a839edc9e3ec1 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Tue, 14 Feb 2012 18:21:33 -0800 Subject: [PATCH 02/16] Quick reformat to get to modern Python (2) --- S3/ACL.py | 366 +++++------ S3/AccessLog.py | 135 +++-- S3/BidirMap.py | 68 +-- S3/CloudFront.py | 1002 ++++++++++++++++--------------- S3/Config.py | 366 +++++------ S3/Exceptions.py | 128 ++-- S3/Progress.py | 283 ++++----- S3/S3.py | 1499 +++++++++++++++++++++++----------------------- S3/S3Uri.py | 362 +++++------ S3/SimpleDB.py | 303 +++++----- S3/SortedDict.py | 97 +-- S3/Utils.py | 565 +++++++++-------- 12 files changed, 2628 insertions(+), 2546 deletions(-) diff --git a/S3/ACL.py b/S3/ACL.py index 8543ae3..eabdf02 100644 --- a/S3/ACL.py +++ b/S3/ACL.py @@ -6,191 +6,195 @@ from Utils import getTreeFromXml try: - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as ET except ImportError: - import elementtree.ElementTree as ET + import elementtree.ElementTree as ET class Grantee(object): - ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers" - LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery" - - def __init__(self): - self.xsi_type = None - self.tag = None - self.name = None - self.display_name = None - self.permission = None - - def __repr__(self): - return 'Grantee("%(tag)s", "%(name)s", "%(permission)s")' % { - "tag" : self.tag, - "name" : self.name, - "permission" : self.permission - } - - def isAllUsers(self): - return self.tag == "URI" and self.name == Grantee.ALL_USERS_URI - - def isAnonRead(self): - return self.isAllUsers() and (self.permission == "READ" or self.permission == "FULL_CONTROL") - - def getElement(self): - el = ET.Element("Grant") - grantee = ET.SubElement(el, "Grantee", { - 'xmlns:xsi' : 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:type' : self.xsi_type - }) - name = ET.SubElement(grantee, self.tag) - name.text = self.name - permission = ET.SubElement(el, "Permission") - permission.text = self.permission - return el + ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers" + LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery" + + def __init__(self): + self.xsi_type = None + self.tag = None + self.name = None + self.display_name = None + self.permission = None + + def __repr__(self): + return 'Grantee("%(tag)s", "%(name)s", "%(permission)s")' % { + "tag": self.tag, + "name": self.name, + "permission": self.permission + } + + def isAllUsers(self): + return self.tag == "URI" and self.name == Grantee.ALL_USERS_URI + + def isAnonRead(self): + return self.isAllUsers() and (self.permission == "READ" or self.permission == "FULL_CONTROL") + + def getElement(self): + el = ET.Element("Grant") + grantee = ET.SubElement(el, "Grantee", { + 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + 'xsi:type': self.xsi_type + }) + name = ET.SubElement(grantee, self.tag) + name.text = self.name + permission = ET.SubElement(el, "Permission") + permission.text = self.permission + return el + class GranteeAnonRead(Grantee): - def __init__(self): - Grantee.__init__(self) - self.xsi_type = "Group" - self.tag = "URI" - self.name = Grantee.ALL_USERS_URI - self.permission = "READ" + def __init__(self): + Grantee.__init__(self) + self.xsi_type = "Group" + self.tag = "URI" + self.name = Grantee.ALL_USERS_URI + self.permission = "READ" + class GranteeLogDelivery(Grantee): - def __init__(self, permission): - """ - permission must be either READ_ACP or WRITE - """ - Grantee.__init__(self) - self.xsi_type = "Group" - self.tag = "URI" - self.name = Grantee.LOG_DELIVERY_URI - self.permission = permission + def __init__(self, permission): + """ + permission must be either READ_ACP or WRITE + """ + Grantee.__init__(self) + self.xsi_type = "Group" + self.tag = "URI" + self.name = Grantee.LOG_DELIVERY_URI + self.permission = permission + class ACL(object): - EMPTY_ACL = "" - - def __init__(self, xml = None): - if not xml: - xml = ACL.EMPTY_ACL - - self.grantees = [] - self.owner_id = "" - self.owner_nick = "" - - tree = getTreeFromXml(xml) - self.parseOwner(tree) - self.parseGrants(tree) - - def parseOwner(self, tree): - self.owner_id = tree.findtext(".//Owner//ID") - self.owner_nick = tree.findtext(".//Owner//DisplayName") - - def parseGrants(self, tree): - for grant in tree.findall(".//Grant"): - grantee = Grantee() - g = grant.find(".//Grantee") - grantee.xsi_type = g.attrib['{http://www.w3.org/2001/XMLSchema-instance}type'] - grantee.permission = grant.find('Permission').text - for el in g: - if el.tag == "DisplayName": - grantee.display_name = el.text - else: - grantee.tag = el.tag - grantee.name = el.text - self.grantees.append(grantee) - - def getGrantList(self): - acl = [] - for grantee in self.grantees: - if grantee.display_name: - user = grantee.display_name - elif grantee.isAllUsers(): - user = "*anon*" - else: - user = grantee.name - acl.append({'grantee': user, 'permission': grantee.permission}) - return acl - - def getOwner(self): - return { 'id' : self.owner_id, 'nick' : self.owner_nick } - - def isAnonRead(self): - for grantee in self.grantees: - if grantee.isAnonRead(): - return True - return False - - def grantAnonRead(self): - if not self.isAnonRead(): - self.appendGrantee(GranteeAnonRead()) - - def revokeAnonRead(self): - self.grantees = [g for g in self.grantees if not g.isAnonRead()] - - def appendGrantee(self, grantee): - self.grantees.append(grantee) - - def hasGrant(self, name, permission): - name = name.lower() - permission = permission.upper() - - for grantee in self.grantees: - if grantee.name.lower() == name: - if grantee.permission == "FULL_CONTROL": - return True - elif grantee.permission.upper() == permission: - return True - - return False; - - def grant(self, name, permission): - if self.hasGrant(name, permission): - return - - name = name.lower() - permission = permission.upper() - - if "ALL" == permission: - permission = "FULL_CONTROL" - - if "FULL_CONTROL" == permission: - self.revoke(name, "ALL") - - grantee = Grantee() - grantee.name = name - grantee.permission = permission - - if name.find('@') <= -1: # ultra lame attempt to differenciate emails id from canonical ids - grantee.xsi_type = "CanonicalUser" - grantee.tag = "ID" - else: - grantee.xsi_type = "AmazonCustomerByEmail" - grantee.tag = "EmailAddress" - - self.appendGrantee(grantee) - - - def revoke(self, name, permission): - name = name.lower() - permission = permission.upper() - - if "ALL" == permission: - self.grantees = [g for g in self.grantees if not g.name.lower() == name] - else: - self.grantees = [g for g in self.grantees if not (g.name.lower() == name and g.permission.upper() == permission)] - - - def __str__(self): - tree = getTreeFromXml(ACL.EMPTY_ACL) - tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/" - owner = tree.find(".//Owner//ID") - owner.text = self.owner_id - acl = tree.find(".//AccessControlList") - for grantee in self.grantees: - acl.append(grantee.getElement()) - return ET.tostring(tree) + EMPTY_ACL = "" + + def __init__(self, xml=None): + if not xml: + xml = ACL.EMPTY_ACL + + self.grantees = [] + self.owner_id = "" + self.owner_nick = "" + + tree = getTreeFromXml(xml) + self.parseOwner(tree) + self.parseGrants(tree) + + def parseOwner(self, tree): + self.owner_id = tree.findtext(".//Owner//ID") + self.owner_nick = tree.findtext(".//Owner//DisplayName") + + def parseGrants(self, tree): + for grant in tree.findall(".//Grant"): + grantee = Grantee() + g = grant.find(".//Grantee") + grantee.xsi_type = g.attrib['{http://www.w3.org/2001/XMLSchema-instance}type'] + grantee.permission = grant.find('Permission').text + for el in g: + if el.tag == "DisplayName": + grantee.display_name = el.text + else: + grantee.tag = el.tag + grantee.name = el.text + self.grantees.append(grantee) + + def getGrantList(self): + acl = [] + for grantee in self.grantees: + if grantee.display_name: + user = grantee.display_name + elif grantee.isAllUsers(): + user = "*anon*" + else: + user = grantee.name + acl.append({'grantee': user, 'permission': grantee.permission}) + return acl + + def getOwner(self): + return {'id': self.owner_id, 'nick': self.owner_nick} + + def isAnonRead(self): + for grantee in self.grantees: + if grantee.isAnonRead(): + return True + return False + + def grantAnonRead(self): + if not self.isAnonRead(): + self.appendGrantee(GranteeAnonRead()) + + def revokeAnonRead(self): + self.grantees = [g for g in self.grantees if not g.isAnonRead()] + + def appendGrantee(self, grantee): + self.grantees.append(grantee) + + def hasGrant(self, name, permission): + name = name.lower() + permission = permission.upper() + + for grantee in self.grantees: + if grantee.name.lower() == name: + if grantee.permission == "FULL_CONTROL": + return True + elif grantee.permission.upper() == permission: + return True + + return False; + + def grant(self, name, permission): + if self.hasGrant(name, permission): + return + + name = name.lower() + permission = permission.upper() + + if "ALL" == permission: + permission = "FULL_CONTROL" + + if "FULL_CONTROL" == permission: + self.revoke(name, "ALL") + + grantee = Grantee() + grantee.name = name + grantee.permission = permission + + if name.find('@') <= -1: # ultra lame attempt to differenciate emails id from canonical ids + grantee.xsi_type = "CanonicalUser" + grantee.tag = "ID" + else: + grantee.xsi_type = "AmazonCustomerByEmail" + grantee.tag = "EmailAddress" + + self.appendGrantee(grantee) + + + def revoke(self, name, permission): + name = name.lower() + permission = permission.upper() + + if "ALL" == permission: + self.grantees = [g for g in self.grantees if not g.name.lower() == name] + else: + self.grantees = [g for g in self.grantees if + not (g.name.lower() == name and g.permission.upper() == permission)] + + + def __str__(self): + tree = getTreeFromXml(ACL.EMPTY_ACL) + tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/" + owner = tree.find(".//Owner//ID") + owner.text = self.owner_id + acl = tree.find(".//AccessControlList") + for grantee in self.grantees: + acl.append(grantee.getElement()) + return ET.tostring(tree) if __name__ == "__main__": - xml = """ + xml = """ 12345678901234567890 @@ -213,10 +217,10 @@ def __str__(self): """ - acl = ACL(xml) - print "Grants:", acl.getGrantList() - acl.revokeAnonRead() - print "Grants:", acl.getGrantList() - acl.grantAnonRead() - print "Grants:", acl.getGrantList() - print acl + acl = ACL(xml) + print "Grants:", acl.getGrantList() + acl.revokeAnonRead() + print "Grants:", acl.getGrantList() + acl.grantAnonRead() + print "Grants:", acl.getGrantList() + print acl diff --git a/S3/AccessLog.py b/S3/AccessLog.py index 6718298..b85bfe9 100644 --- a/S3/AccessLog.py +++ b/S3/AccessLog.py @@ -9,82 +9,85 @@ from ACL import GranteeAnonRead try: - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as ET except ImportError: - import elementtree.ElementTree as ET + import elementtree.ElementTree as ET __all__ = [] + class AccessLog(object): - LOG_DISABLED = "" - LOG_TEMPLATE = "" + LOG_DISABLED = "" + LOG_TEMPLATE = "" + + def __init__(self, xml=None): + if not xml: + xml = self.LOG_DISABLED + self.tree = getTreeFromXml(xml) + self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01" + + def isLoggingEnabled(self): + return bool(self.tree.find(".//LoggingEnabled")) - def __init__(self, xml = None): - if not xml: - xml = self.LOG_DISABLED - self.tree = getTreeFromXml(xml) - self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01" - - def isLoggingEnabled(self): - return bool(self.tree.find(".//LoggingEnabled")) + def disableLogging(self): + el = self.tree.find(".//LoggingEnabled") + if el: + self.tree.remove(el) - def disableLogging(self): - el = self.tree.find(".//LoggingEnabled") - if el: - self.tree.remove(el) - - def enableLogging(self, target_prefix_uri): - el = self.tree.find(".//LoggingEnabled") - if not el: - el = getTreeFromXml(self.LOG_TEMPLATE) - self.tree.append(el) - el.find(".//TargetBucket").text = target_prefix_uri.bucket() - el.find(".//TargetPrefix").text = target_prefix_uri.object() + def enableLogging(self, target_prefix_uri): + el = self.tree.find(".//LoggingEnabled") + if not el: + el = getTreeFromXml(self.LOG_TEMPLATE) + self.tree.append(el) + el.find(".//TargetBucket").text = target_prefix_uri.bucket() + el.find(".//TargetPrefix").text = target_prefix_uri.object() - def targetPrefix(self): - if self.isLoggingEnabled(): - el = self.tree.find(".//LoggingEnabled") - target_prefix = "s3://%s/%s" % ( - self.tree.find(".//LoggingEnabled//TargetBucket").text, - self.tree.find(".//LoggingEnabled//TargetPrefix").text) - return S3Uri.S3Uri(target_prefix) - else: - return "" + def targetPrefix(self): + if self.isLoggingEnabled(): + el = self.tree.find(".//LoggingEnabled") + target_prefix = "s3://%s/%s" % ( + self.tree.find(".//LoggingEnabled//TargetBucket").text, + self.tree.find(".//LoggingEnabled//TargetPrefix").text) + return S3Uri.S3Uri(target_prefix) + else: + return "" - def setAclPublic(self, acl_public): - le = self.tree.find(".//LoggingEnabled") - if not le: - raise ParameterError("Logging not enabled, can't set default ACL for logs") - tg = le.find(".//TargetGrants") - if not acl_public: - if not tg: - ## All good, it's not been there - return - else: - le.remove(tg) - else: # acl_public == True - anon_read = GranteeAnonRead().getElement() - if not tg: - tg = ET.SubElement(le, "TargetGrants") - ## What if TargetGrants already exists? We should check if - ## AnonRead is there before appending a new one. Later... - tg.append(anon_read) + def setAclPublic(self, acl_public): + le = self.tree.find(".//LoggingEnabled") + if not le: + raise ParameterError("Logging not enabled, can't set default ACL for logs") + tg = le.find(".//TargetGrants") + if not acl_public: + if not tg: + ## All good, it's not been there + return + else: + le.remove(tg) + else: # acl_public == True + anon_read = GranteeAnonRead().getElement() + if not tg: + tg = ET.SubElement(le, "TargetGrants") + ## What if TargetGrants already exists? We should check if + ## AnonRead is there before appending a new one. Later... + tg.append(anon_read) - def isAclPublic(self): - raise NotImplementedError() + def isAclPublic(self): + raise NotImplementedError() + + def __str__(self): + return ET.tostring(self.tree) - def __str__(self): - return ET.tostring(self.tree) __all__.append("AccessLog") if __name__ == "__main__": - from S3Uri import S3Uri - log = AccessLog() - print log - log.enableLogging(S3Uri("s3://targetbucket/prefix/log-")) - print log - log.setAclPublic(True) - print log - log.setAclPublic(False) - print log - log.disableLogging() - print log + from S3Uri import S3Uri + + log = AccessLog() + print log + log.enableLogging(S3Uri("s3://targetbucket/prefix/log-")) + print log + log.setAclPublic(True) + print log + log.setAclPublic(False) + print log + log.disableLogging() + print log diff --git a/S3/BidirMap.py b/S3/BidirMap.py index 7d1f477..26ef5bf 100644 --- a/S3/BidirMap.py +++ b/S3/BidirMap.py @@ -4,37 +4,37 @@ ## License: GPL Version 2 class BidirMap(object): - def __init__(self, **map): - self.k2v = {} - self.v2k = {} - for key in map: - self.__setitem__(key, map[key]) - - def __setitem__(self, key, value): - if self.v2k.has_key(value): - if self.v2k[value] != key: - raise KeyError("Value '"+str(value)+"' already in use with key '"+str(self.v2k[value])+"'") - try: - del(self.v2k[self.k2v[key]]) - except KeyError: - pass - self.k2v[key] = value - self.v2k[value] = key - - def __getitem__(self, key): - return self.k2v[key] - - def __str__(self): - return self.v2k.__str__() - - def getkey(self, value): - return self.v2k[value] - - def getvalue(self, key): - return self.k2v[key] - - def keys(self): - return [key for key in self.k2v] - - def values(self): - return [value for value in self.v2k] + def __init__(self, **map): + self.k2v = {} + self.v2k = {} + for key in map: + self.__setitem__(key, map[key]) + + def __setitem__(self, key, value): + if self.v2k.has_key(value): + if self.v2k[value] != key: + raise KeyError("Value '" + str(value) + "' already in use with key '" + str(self.v2k[value]) + "'") + try: + del(self.v2k[self.k2v[key]]) + except KeyError: + pass + self.k2v[key] = value + self.v2k[value] = key + + def __getitem__(self, key): + return self.k2v[key] + + def __str__(self): + return self.v2k.__str__() + + def getkey(self, value): + return self.v2k[value] + + def getvalue(self, key): + return self.k2v[key] + + def keys(self): + return [key for key in self.k2v] + + def values(self): + return [value for value in self.v2k] diff --git a/S3/CloudFront.py b/S3/CloudFront.py index c2ebe39..b7c6029 100644 --- a/S3/CloudFront.py +++ b/S3/CloudFront.py @@ -9,9 +9,9 @@ from logging import debug, info, warning, error try: - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as ET except ImportError: - import elementtree.ElementTree as ET + import elementtree.ElementTree as ET from Config import Config from Exceptions import * @@ -19,512 +19,520 @@ from S3Uri import S3Uri, S3UriS3 def output(message): - sys.stdout.write(message + "\n") + sys.stdout.write(message + "\n") + def pretty_output(label, message): - #label = ("%s " % label).ljust(20, ".") - label = ("%s:" % label).ljust(15) - output("%s %s" % (label, message)) + #label = ("%s " % label).ljust(20, ".") + label = ("%s:" % label).ljust(15) + output("%s %s" % (label, message)) + class DistributionSummary(object): - ## Example: - ## - ## - ## 1234567890ABC - ## Deployed - ## 2009-01-16T11:49:02.189Z - ## blahblahblah.cloudfront.net - ## example.bucket.s3.amazonaws.com - ## true - ## - - def __init__(self, tree): - if tree.tag != "DistributionSummary": - raise ValueError("Expected xml, got: <%s />" % tree.tag) - self.parse(tree) - - def parse(self, tree): - self.info = getDictFromTree(tree) - self.info['Enabled'] = (self.info['Enabled'].lower() == "true") - - def uri(self): - return S3Uri("cf://%s" % self.info['Id']) + ## Example: + ## + ## + ## 1234567890ABC + ## Deployed + ## 2009-01-16T11:49:02.189Z + ## blahblahblah.cloudfront.net + ## example.bucket.s3.amazonaws.com + ## true + ## + + def __init__(self, tree): + if tree.tag != "DistributionSummary": + raise ValueError("Expected xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + self.info['Enabled'] = (self.info['Enabled'].lower() == "true") + + def uri(self): + return S3Uri("cf://%s" % self.info['Id']) + class DistributionList(object): - ## Example: - ## - ## - ## - ## 100 - ## false - ## - ## ... handled by DistributionSummary() class ... - ## - ## - - def __init__(self, xml): - tree = getTreeFromXml(xml) - if tree.tag != "DistributionList": - raise ValueError("Expected xml, got: <%s />" % tree.tag) - self.parse(tree) - - def parse(self, tree): - self.info = getDictFromTree(tree) - ## Normalise some items - self.info['IsTruncated'] = (self.info['IsTruncated'].lower() == "true") - - self.dist_summs = [] - for dist_summ in tree.findall(".//DistributionSummary"): - self.dist_summs.append(DistributionSummary(dist_summ)) + ## Example: + ## + ## + ## + ## 100 + ## false + ## + ## ... handled by DistributionSummary() class ... + ## + ## + + def __init__(self, xml): + tree = getTreeFromXml(xml) + if tree.tag != "DistributionList": + raise ValueError("Expected xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + ## Normalise some items + self.info['IsTruncated'] = (self.info['IsTruncated'].lower() == "true") + + self.dist_summs = [] + for dist_summ in tree.findall(".//DistributionSummary"): + self.dist_summs.append(DistributionSummary(dist_summ)) + class Distribution(object): - ## Example: - ## - ## - ## 1234567890ABC - ## InProgress - ## 2009-01-16T13:07:11.319Z - ## blahblahblah.cloudfront.net - ## - ## ... handled by DistributionConfig() class ... - ## - ## - - def __init__(self, xml): - tree = getTreeFromXml(xml) - if tree.tag != "Distribution": - raise ValueError("Expected xml, got: <%s />" % tree.tag) - self.parse(tree) - - def parse(self, tree): - self.info = getDictFromTree(tree) - ## Normalise some items - self.info['LastModifiedTime'] = dateS3toPython(self.info['LastModifiedTime']) - - self.info['DistributionConfig'] = DistributionConfig(tree = tree.find(".//DistributionConfig")) - - def uri(self): - return S3Uri("cf://%s" % self.info['Id']) + ## Example: + ## + ## + ## 1234567890ABC + ## InProgress + ## 2009-01-16T13:07:11.319Z + ## blahblahblah.cloudfront.net + ## + ## ... handled by DistributionConfig() class ... + ## + ## + + def __init__(self, xml): + tree = getTreeFromXml(xml) + if tree.tag != "Distribution": + raise ValueError("Expected xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + ## Normalise some items + self.info['LastModifiedTime'] = dateS3toPython(self.info['LastModifiedTime']) + + self.info['DistributionConfig'] = DistributionConfig(tree=tree.find(".//DistributionConfig")) + + def uri(self): + return S3Uri("cf://%s" % self.info['Id']) + class DistributionConfig(object): - ## Example: - ## - ## - ## somebucket.s3.amazonaws.com - ## s3://somebucket/ - ## http://somebucket.s3.amazonaws.com/ - ## true - ## - ## bu.ck.et - ## /cf-somebucket/ - ## - ## - - EMPTY_CONFIG = "true" - xmlns = "http://cloudfront.amazonaws.com/doc/2010-06-01/" - def __init__(self, xml = None, tree = None): - if not xml: - xml = DistributionConfig.EMPTY_CONFIG - - if not tree: - tree = getTreeFromXml(xml) - - if tree.tag != "DistributionConfig": - raise ValueError("Expected xml, got: <%s />" % tree.tag) - self.parse(tree) - - def parse(self, tree): - self.info = getDictFromTree(tree) - self.info['Enabled'] = (self.info['Enabled'].lower() == "true") - if not self.info.has_key("CNAME"): - self.info['CNAME'] = [] - if type(self.info['CNAME']) != list: - self.info['CNAME'] = [self.info['CNAME']] - self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']] - if not self.info.has_key("Comment"): - self.info['Comment'] = "" - ## Figure out logging - complex node not parsed by getDictFromTree() - logging_nodes = tree.findall(".//Logging") - if logging_nodes: - logging_dict = getDictFromTree(logging_nodes[0]) - logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket']) - if not success: - warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket']) - self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict) - else: - self.info['Logging'] = None - - def __str__(self): - tree = ET.Element("DistributionConfig") - tree.attrib['xmlns'] = DistributionConfig.xmlns - - ## Retain the order of the following calls! - appendXmlTextNode("Origin", self.info['Origin'], tree) - appendXmlTextNode("CallerReference", self.info['CallerReference'], tree) - for cname in self.info['CNAME']: - appendXmlTextNode("CNAME", cname.lower(), tree) - if self.info['Comment']: - appendXmlTextNode("Comment", self.info['Comment'], tree) - appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree) - if self.info['Logging']: - logging_el = ET.Element("Logging") - appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el) - appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el) - tree.append(logging_el) - return ET.tostring(tree) + ## Example: + ## + ## + ## somebucket.s3.amazonaws.com + ## s3://somebucket/ + ## http://somebucket.s3.amazonaws.com/ + ## true + ## + ## bu.ck.et + ## /cf-somebucket/ + ## + ## + + EMPTY_CONFIG = "true" + xmlns = "http://cloudfront.amazonaws.com/doc/2010-06-01/" + + def __init__(self, xml=None, tree=None): + if not xml: + xml = DistributionConfig.EMPTY_CONFIG + + if not tree: + tree = getTreeFromXml(xml) + + if tree.tag != "DistributionConfig": + raise ValueError("Expected xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + self.info['Enabled'] = (self.info['Enabled'].lower() == "true") + if not self.info.has_key("CNAME"): + self.info['CNAME'] = [] + if type(self.info['CNAME']) != list: + self.info['CNAME'] = [self.info['CNAME']] + self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']] + if not self.info.has_key("Comment"): + self.info['Comment'] = "" + ## Figure out logging - complex node not parsed by getDictFromTree() + logging_nodes = tree.findall(".//Logging") + if logging_nodes: + logging_dict = getDictFromTree(logging_nodes[0]) + logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket']) + if not success: + warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket']) + self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict) + else: + self.info['Logging'] = None + + def __str__(self): + tree = ET.Element("DistributionConfig") + tree.attrib['xmlns'] = DistributionConfig.xmlns + + ## Retain the order of the following calls! + appendXmlTextNode("Origin", self.info['Origin'], tree) + appendXmlTextNode("CallerReference", self.info['CallerReference'], tree) + for cname in self.info['CNAME']: + appendXmlTextNode("CNAME", cname.lower(), tree) + if self.info['Comment']: + appendXmlTextNode("Comment", self.info['Comment'], tree) + appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree) + if self.info['Logging']: + logging_el = ET.Element("Logging") + appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el) + appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el) + tree.append(logging_el) + return ET.tostring(tree) + class CloudFront(object): - operations = { - "CreateDist" : { 'method' : "POST", 'resource' : "" }, - "DeleteDist" : { 'method' : "DELETE", 'resource' : "/%(dist_id)s" }, - "GetList" : { 'method' : "GET", 'resource' : "" }, - "GetDistInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s" }, - "GetDistConfig" : { 'method' : "GET", 'resource' : "/%(dist_id)s/config" }, - "SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" }, - } - - ## Maximum attempts of re-issuing failed requests - _max_retries = 5 - - def __init__(self, config): - self.config = config - - ## -------------------------------------------------- - ## Methods implementing CloudFront API - ## -------------------------------------------------- - - def GetList(self): - response = self.send_request("GetList") - response['dist_list'] = DistributionList(response['data']) - if response['dist_list'].info['IsTruncated']: - raise NotImplementedError("List is truncated. Ask s3cmd author to add support.") - ## TODO: handle Truncated - return response - - def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None): - dist_config = DistributionConfig() - dist_config.info['Enabled'] = True - dist_config.info['Origin'] = uri.host_name() - dist_config.info['CallerReference'] = str(uri) - if comment == None: - dist_config.info['Comment'] = uri.public_url() - else: - dist_config.info['Comment'] = comment - for cname in cnames_add: - if dist_config.info['CNAME'].count(cname) == 0: - dist_config.info['CNAME'].append(cname) - if logging: - dist_config.info['Logging'] = S3UriS3(logging) - request_body = str(dist_config) - debug("CreateDistribution(): request_body: %s" % request_body) - response = self.send_request("CreateDist", body = request_body) - response['distribution'] = Distribution(response['data']) - return response - - def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [], - comment = None, enabled = None, logging = None): - if cfuri.type != "cf": - raise ValueError("Expected CFUri instead of: %s" % cfuri) - # Get current dist status (enabled/disabled) and Etag - info("Checking current status of %s" % cfuri) - response = self.GetDistConfig(cfuri) - dc = response['dist_config'] - if enabled != None: - dc.info['Enabled'] = enabled - if comment != None: - dc.info['Comment'] = comment - for cname in cnames_add: - if dc.info['CNAME'].count(cname) == 0: - dc.info['CNAME'].append(cname) - for cname in cnames_remove: - while dc.info['CNAME'].count(cname) > 0: - dc.info['CNAME'].remove(cname) - if logging != None: - if logging == False: - dc.info['Logging'] = False - else: - dc.info['Logging'] = S3UriS3(logging) - response = self.SetDistConfig(cfuri, dc, response['headers']['etag']) - return response - - def DeleteDistribution(self, cfuri): - if cfuri.type != "cf": - raise ValueError("Expected CFUri instead of: %s" % cfuri) - # Get current dist status (enabled/disabled) and Etag - info("Checking current status of %s" % cfuri) - response = self.GetDistConfig(cfuri) - if response['dist_config'].info['Enabled']: - info("Distribution is ENABLED. Disabling first.") - response['dist_config'].info['Enabled'] = False - response = self.SetDistConfig(cfuri, response['dist_config'], - response['headers']['etag']) - warning("Waiting for Distribution to become disabled.") - warning("This may take several minutes, please wait.") - while True: - response = self.GetDistInfo(cfuri) - d = response['distribution'] - if d.info['Status'] == "Deployed" and d.info['Enabled'] == False: - info("Distribution is now disabled") - break - warning("Still waiting...") - time.sleep(10) - headers = {} - headers['if-match'] = response['headers']['etag'] - response = self.send_request("DeleteDist", dist_id = cfuri.dist_id(), - headers = headers) - return response - - def GetDistInfo(self, cfuri): - if cfuri.type != "cf": - raise ValueError("Expected CFUri instead of: %s" % cfuri) - response = self.send_request("GetDistInfo", dist_id = cfuri.dist_id()) - response['distribution'] = Distribution(response['data']) - return response - - def GetDistConfig(self, cfuri): - if cfuri.type != "cf": - raise ValueError("Expected CFUri instead of: %s" % cfuri) - response = self.send_request("GetDistConfig", dist_id = cfuri.dist_id()) - response['dist_config'] = DistributionConfig(response['data']) - return response - - def SetDistConfig(self, cfuri, dist_config, etag = None): - if etag == None: - debug("SetDistConfig(): Etag not set. Fetching it first.") - etag = self.GetDistConfig(cfuri)['headers']['etag'] - debug("SetDistConfig(): Etag = %s" % etag) - request_body = str(dist_config) - debug("SetDistConfig(): request_body: %s" % request_body) - headers = {} - headers['if-match'] = etag - response = self.send_request("SetDistConfig", dist_id = cfuri.dist_id(), - body = request_body, headers = headers) - return response - - ## -------------------------------------------------- - ## Low-level methods for handling CloudFront requests - ## -------------------------------------------------- - - def send_request(self, op_name, dist_id = None, body = None, headers = {}, retries = _max_retries): - operation = self.operations[op_name] - if body: - headers['content-type'] = 'text/plain' - request = self.create_request(operation, dist_id, headers) - conn = self.get_connection() - debug("send_request(): %s %s" % (request['method'], request['resource'])) - conn.request(request['method'], request['resource'], body, request['headers']) - http_response = conn.getresponse() - response = {} - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = dict(http_response.getheaders()) - response["data"] = http_response.read() - conn.close() - - debug("CloudFront: response: %r" % response) - - if response["status"] >= 500: - e = CloudFrontError(response) - if retries: - warning(u"Retrying failed request: %s" % op_name) - warning(unicode(e)) - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - return self.send_request(op_name, dist_id, body, retries - 1) - else: - raise e - - if response["status"] < 200 or response["status"] > 299: - raise CloudFrontError(response) - - return response - - def create_request(self, operation, dist_id = None, headers = None): - resource = self.config.cloudfront_resource + ( - operation['resource'] % { 'dist_id' : dist_id }) - - if not headers: - headers = {} - - if headers.has_key("date"): - if not headers.has_key("x-amz-date"): - headers["x-amz-date"] = headers["date"] - del(headers["date"]) - - if not headers.has_key("x-amz-date"): - headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) - - signature = self.sign_request(headers) - headers["Authorization"] = "AWS "+self.config.access_key+":"+signature - - request = {} - request['resource'] = resource - request['headers'] = headers - request['method'] = operation['method'] - - return request - - def sign_request(self, headers): - string_to_sign = headers['x-amz-date'] - signature = sign_string(string_to_sign) - debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature)) - return signature - - def get_connection(self): - if self.config.proxy_host != "": - raise ParameterError("CloudFront commands don't work from behind a HTTP proxy") - return httplib.HTTPSConnection(self.config.cloudfront_host) - - def _fail_wait(self, retries): - # Wait a few seconds. The more it fails the more we wait. - return (self._max_retries - retries + 1) * 3 + operations = { + "CreateDist": {'method': "POST", 'resource': ""}, + "DeleteDist": {'method': "DELETE", 'resource': "/%(dist_id)s"}, + "GetList": {'method': "GET", 'resource': ""}, + "GetDistInfo": {'method': "GET", 'resource': "/%(dist_id)s"}, + "GetDistConfig": {'method': "GET", 'resource': "/%(dist_id)s/config"}, + "SetDistConfig": {'method': "PUT", 'resource': "/%(dist_id)s/config"}, + } + + ## Maximum attempts of re-issuing failed requests + _max_retries = 5 + + def __init__(self, config): + self.config = config + + ## -------------------------------------------------- + ## Methods implementing CloudFront API + ## -------------------------------------------------- + + def GetList(self): + response = self.send_request("GetList") + response['dist_list'] = DistributionList(response['data']) + if response['dist_list'].info['IsTruncated']: + raise NotImplementedError("List is truncated. Ask s3cmd author to add support.") + ## TODO: handle Truncated + return response + + def CreateDistribution(self, uri, cnames_add=[], comment=None, logging=None): + dist_config = DistributionConfig() + dist_config.info['Enabled'] = True + dist_config.info['Origin'] = uri.host_name() + dist_config.info['CallerReference'] = str(uri) + if comment == None: + dist_config.info['Comment'] = uri.public_url() + else: + dist_config.info['Comment'] = comment + for cname in cnames_add: + if dist_config.info['CNAME'].count(cname) == 0: + dist_config.info['CNAME'].append(cname) + if logging: + dist_config.info['Logging'] = S3UriS3(logging) + request_body = str(dist_config) + debug("CreateDistribution(): request_body: %s" % request_body) + response = self.send_request("CreateDist", body=request_body) + response['distribution'] = Distribution(response['data']) + return response + + def ModifyDistribution(self, cfuri, cnames_add=[], cnames_remove=[], + comment=None, enabled=None, logging=None): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + # Get current dist status (enabled/disabled) and Etag + info("Checking current status of %s" % cfuri) + response = self.GetDistConfig(cfuri) + dc = response['dist_config'] + if enabled != None: + dc.info['Enabled'] = enabled + if comment != None: + dc.info['Comment'] = comment + for cname in cnames_add: + if dc.info['CNAME'].count(cname) == 0: + dc.info['CNAME'].append(cname) + for cname in cnames_remove: + while dc.info['CNAME'].count(cname) > 0: + dc.info['CNAME'].remove(cname) + if logging != None: + if logging == False: + dc.info['Logging'] = False + else: + dc.info['Logging'] = S3UriS3(logging) + response = self.SetDistConfig(cfuri, dc, response['headers']['etag']) + return response + + def DeleteDistribution(self, cfuri): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + # Get current dist status (enabled/disabled) and Etag + info("Checking current status of %s" % cfuri) + response = self.GetDistConfig(cfuri) + if response['dist_config'].info['Enabled']: + info("Distribution is ENABLED. Disabling first.") + response['dist_config'].info['Enabled'] = False + response = self.SetDistConfig(cfuri, response['dist_config'], + response['headers']['etag']) + warning("Waiting for Distribution to become disabled.") + warning("This may take several minutes, please wait.") + while True: + response = self.GetDistInfo(cfuri) + d = response['distribution'] + if d.info['Status'] == "Deployed" and d.info['Enabled'] == False: + info("Distribution is now disabled") + break + warning("Still waiting...") + time.sleep(10) + headers = {} + headers['if-match'] = response['headers']['etag'] + response = self.send_request("DeleteDist", dist_id=cfuri.dist_id(), + headers=headers) + return response + + def GetDistInfo(self, cfuri): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + response = self.send_request("GetDistInfo", dist_id=cfuri.dist_id()) + response['distribution'] = Distribution(response['data']) + return response + + def GetDistConfig(self, cfuri): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + response = self.send_request("GetDistConfig", dist_id=cfuri.dist_id()) + response['dist_config'] = DistributionConfig(response['data']) + return response + + def SetDistConfig(self, cfuri, dist_config, etag=None): + if etag == None: + debug("SetDistConfig(): Etag not set. Fetching it first.") + etag = self.GetDistConfig(cfuri)['headers']['etag'] + debug("SetDistConfig(): Etag = %s" % etag) + request_body = str(dist_config) + debug("SetDistConfig(): request_body: %s" % request_body) + headers = {} + headers['if-match'] = etag + response = self.send_request("SetDistConfig", dist_id=cfuri.dist_id(), + body=request_body, headers=headers) + return response + + ## -------------------------------------------------- + ## Low-level methods for handling CloudFront requests + ## -------------------------------------------------- + + def send_request(self, op_name, dist_id=None, body=None, headers={}, retries=_max_retries): + operation = self.operations[op_name] + if body: + headers['content-type'] = 'text/plain' + request = self.create_request(operation, dist_id, headers) + conn = self.get_connection() + debug("send_request(): %s %s" % (request['method'], request['resource'])) + conn.request(request['method'], request['resource'], body, request['headers']) + http_response = conn.getresponse() + response = {} + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = dict(http_response.getheaders()) + response["data"] = http_response.read() + conn.close() + + debug("CloudFront: response: %r" % response) + + if response["status"] >= 500: + e = CloudFrontError(response) + if retries: + warning(u"Retrying failed request: %s" % op_name) + warning(unicode(e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + return self.send_request(op_name, dist_id, body, retries - 1) + else: + raise e + + if response["status"] < 200 or response["status"] > 299: + raise CloudFrontError(response) + + return response + + def create_request(self, operation, dist_id=None, headers=None): + resource = self.config.cloudfront_resource + ( + operation['resource'] % {'dist_id': dist_id}) + + if not headers: + headers = {} + + if headers.has_key("date"): + if not headers.has_key("x-amz-date"): + headers["x-amz-date"] = headers["date"] + del(headers["date"]) + + if not headers.has_key("x-amz-date"): + headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) + + signature = self.sign_request(headers) + headers["Authorization"] = "AWS " + self.config.access_key + ":" + signature + + request = {} + request['resource'] = resource + request['headers'] = headers + request['method'] = operation['method'] + + return request + + def sign_request(self, headers): + string_to_sign = headers['x-amz-date'] + signature = sign_string(string_to_sign) + debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature)) + return signature + + def get_connection(self): + if self.config.proxy_host != "": + raise ParameterError("CloudFront commands don't work from behind a HTTP proxy") + return httplib.HTTPSConnection(self.config.cloudfront_host) + + def _fail_wait(self, retries): + # Wait a few seconds. The more it fails the more we wait. + return (self._max_retries - retries + 1) * 3 + class Cmd(object): - """ - Class that implements CloudFront commands - """ - - class Options(object): - cf_cnames_add = [] - cf_cnames_remove = [] - cf_comment = None - cf_enable = None - cf_logging = None - - def option_list(self): - return [opt for opt in dir(self) if opt.startswith("cf_")] - - def update_option(self, option, value): - setattr(Cmd.options, option, value) - - options = Options() - dist_list = None - - @staticmethod - def _get_dist_name_for_bucket(uri): - cf = CloudFront(Config()) - debug("_get_dist_name_for_bucket(%r)" % uri) - assert(uri.type == "s3") - if Cmd.dist_list is None: - response = cf.GetList() - Cmd.dist_list = {} - for d in response['dist_list'].dist_summs: - Cmd.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri() - debug("dist_list: %s" % Cmd.dist_list) - return Cmd.dist_list[uri.bucket()] - - @staticmethod - def _parse_args(args): - cfuris = [] - for arg in args: - uri = S3Uri(arg) - if uri.type == 's3': - try: - uri = Cmd._get_dist_name_for_bucket(uri) - except Exception, e: - debug(e) - raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % uri) - if uri.type != 'cf': - raise ParameterError("CloudFront URI required instead of: %s" % arg) - cfuris.append(uri) - return cfuris - - @staticmethod - def info(args): - cf = CloudFront(Config()) - if not args: - response = cf.GetList() - for d in response['dist_list'].dist_summs: - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(d.info['Origin'])) - pretty_output("DistId", d.uri()) - pretty_output("DomainName", d.info['DomainName']) - pretty_output("Status", d.info['Status']) - pretty_output("Enabled", d.info['Enabled']) - output("") - else: - cfuris = Cmd._parse_args(args) - for cfuri in cfuris: - response = cf.GetDistInfo(cfuri) - d = response['distribution'] - dc = d.info['DistributionConfig'] - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) - pretty_output("DistId", d.uri()) - pretty_output("DomainName", d.info['DomainName']) - pretty_output("Status", d.info['Status']) - pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) - pretty_output("Comment", dc.info['Comment']) - pretty_output("Enabled", dc.info['Enabled']) - pretty_output("Logging", dc.info['Logging'] or "Disabled") - pretty_output("Etag", response['headers']['etag']) - - @staticmethod - def create(args): - cf = CloudFront(Config()) - buckets = [] - for arg in args: - uri = S3Uri(arg) - if uri.type != "s3": - raise ParameterError("Bucket can only be created from a s3:// URI instead of: %s" % arg) - if uri.object(): - raise ParameterError("Use s3:// URI with a bucket name only instead of: %s" % arg) - if not uri.is_dns_compatible(): - raise ParameterError("CloudFront can only handle lowercase-named buckets.") - buckets.append(uri) - if not buckets: - raise ParameterError("No valid bucket names found") - for uri in buckets: - info("Creating distribution from: %s" % uri) - response = cf.CreateDistribution(uri, cnames_add = Cmd.options.cf_cnames_add, - comment = Cmd.options.cf_comment, - logging = Cmd.options.cf_logging) - d = response['distribution'] - dc = d.info['DistributionConfig'] - output("Distribution created:") - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) - pretty_output("DistId", d.uri()) - pretty_output("DomainName", d.info['DomainName']) - pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) - pretty_output("Comment", dc.info['Comment']) - pretty_output("Status", d.info['Status']) - pretty_output("Enabled", dc.info['Enabled']) - pretty_output("Etag", response['headers']['etag']) - - @staticmethod - def delete(args): - cf = CloudFront(Config()) - cfuris = Cmd._parse_args(args) - for cfuri in cfuris: - response = cf.DeleteDistribution(cfuri) - if response['status'] >= 400: - error("Distribution %s could not be deleted: %s" % (cfuri, response['reason'])) - output("Distribution %s deleted" % cfuri) - - @staticmethod - def modify(args): - cf = CloudFront(Config()) - if len(args) > 1: - raise ParameterError("Too many parameters. Modify one Distribution at a time.") - try: - cfuri = Cmd._parse_args(args)[0] - except IndexError, e: - raise ParameterError("No valid Distribution URI found.") - response = cf.ModifyDistribution(cfuri, - cnames_add = Cmd.options.cf_cnames_add, - cnames_remove = Cmd.options.cf_cnames_remove, - comment = Cmd.options.cf_comment, - enabled = Cmd.options.cf_enable, - logging = Cmd.options.cf_logging) - if response['status'] >= 400: - error("Distribution %s could not be modified: %s" % (cfuri, response['reason'])) - output("Distribution modified: %s" % cfuri) - response = cf.GetDistInfo(cfuri) - d = response['distribution'] - dc = d.info['DistributionConfig'] - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) - pretty_output("DistId", d.uri()) - pretty_output("DomainName", d.info['DomainName']) - pretty_output("Status", d.info['Status']) - pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) - pretty_output("Comment", dc.info['Comment']) - pretty_output("Enabled", dc.info['Enabled']) - pretty_output("Etag", response['headers']['etag']) + """ + Class that implements CloudFront commands + """ + + class Options(object): + cf_cnames_add = [] + cf_cnames_remove = [] + cf_comment = None + cf_enable = None + cf_logging = None + + def option_list(self): + return [opt for opt in dir(self) if opt.startswith("cf_")] + + def update_option(self, option, value): + setattr(Cmd.options, option, value) + + options = Options() + dist_list = None + + @staticmethod + def _get_dist_name_for_bucket(uri): + cf = CloudFront(Config()) + debug("_get_dist_name_for_bucket(%r)" % uri) + assert(uri.type == "s3") + if Cmd.dist_list is None: + response = cf.GetList() + Cmd.dist_list = {} + for d in response['dist_list'].dist_summs: + Cmd.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri() + debug("dist_list: %s" % Cmd.dist_list) + return Cmd.dist_list[uri.bucket()] + + @staticmethod + def _parse_args(args): + cfuris = [] + for arg in args: + uri = S3Uri(arg) + if uri.type == 's3': + try: + uri = Cmd._get_dist_name_for_bucket(uri) + except Exception, e: + debug(e) + raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % uri) + if uri.type != 'cf': + raise ParameterError("CloudFront URI required instead of: %s" % arg) + cfuris.append(uri) + return cfuris + + @staticmethod + def info(args): + cf = CloudFront(Config()) + if not args: + response = cf.GetList() + for d in response['dist_list'].dist_summs: + pretty_output("Origin", S3UriS3.httpurl_to_s3uri(d.info['Origin'])) + pretty_output("DistId", d.uri()) + pretty_output("DomainName", d.info['DomainName']) + pretty_output("Status", d.info['Status']) + pretty_output("Enabled", d.info['Enabled']) + output("") + else: + cfuris = Cmd._parse_args(args) + for cfuri in cfuris: + response = cf.GetDistInfo(cfuri) + d = response['distribution'] + dc = d.info['DistributionConfig'] + pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) + pretty_output("DistId", d.uri()) + pretty_output("DomainName", d.info['DomainName']) + pretty_output("Status", d.info['Status']) + pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) + pretty_output("Comment", dc.info['Comment']) + pretty_output("Enabled", dc.info['Enabled']) + pretty_output("Logging", dc.info['Logging'] or "Disabled") + pretty_output("Etag", response['headers']['etag']) + + @staticmethod + def create(args): + cf = CloudFront(Config()) + buckets = [] + for arg in args: + uri = S3Uri(arg) + if uri.type != "s3": + raise ParameterError("Bucket can only be created from a s3:// URI instead of: %s" % arg) + if uri.object(): + raise ParameterError("Use s3:// URI with a bucket name only instead of: %s" % arg) + if not uri.is_dns_compatible(): + raise ParameterError("CloudFront can only handle lowercase-named buckets.") + buckets.append(uri) + if not buckets: + raise ParameterError("No valid bucket names found") + for uri in buckets: + info("Creating distribution from: %s" % uri) + response = cf.CreateDistribution(uri, cnames_add=Cmd.options.cf_cnames_add, + comment=Cmd.options.cf_comment, + logging=Cmd.options.cf_logging) + d = response['distribution'] + dc = d.info['DistributionConfig'] + output("Distribution created:") + pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) + pretty_output("DistId", d.uri()) + pretty_output("DomainName", d.info['DomainName']) + pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) + pretty_output("Comment", dc.info['Comment']) + pretty_output("Status", d.info['Status']) + pretty_output("Enabled", dc.info['Enabled']) + pretty_output("Etag", response['headers']['etag']) + + @staticmethod + def delete(args): + cf = CloudFront(Config()) + cfuris = Cmd._parse_args(args) + for cfuri in cfuris: + response = cf.DeleteDistribution(cfuri) + if response['status'] >= 400: + error("Distribution %s could not be deleted: %s" % (cfuri, response['reason'])) + output("Distribution %s deleted" % cfuri) + + @staticmethod + def modify(args): + cf = CloudFront(Config()) + if len(args) > 1: + raise ParameterError("Too many parameters. Modify one Distribution at a time.") + try: + cfuri = Cmd._parse_args(args)[0] + except IndexError, e: + raise ParameterError("No valid Distribution URI found.") + response = cf.ModifyDistribution(cfuri, + cnames_add=Cmd.options.cf_cnames_add, + cnames_remove=Cmd.options.cf_cnames_remove, + comment=Cmd.options.cf_comment, + enabled=Cmd.options.cf_enable, + logging=Cmd.options.cf_logging) + if response['status'] >= 400: + error("Distribution %s could not be modified: %s" % (cfuri, response['reason'])) + output("Distribution modified: %s" % cfuri) + response = cf.GetDistInfo(cfuri) + d = response['distribution'] + dc = d.info['DistributionConfig'] + pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) + pretty_output("DistId", d.uri()) + pretty_output("DomainName", d.info['DomainName']) + pretty_output("Status", d.info['Status']) + pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) + pretty_output("Comment", dc.info['Comment']) + pretty_output("Enabled", dc.info['Enabled']) + pretty_output("Etag", response['headers']['etag']) diff --git a/S3/Config.py b/S3/Config.py index ebce45c..c25f05c 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -10,191 +10,193 @@ from SortedDict import SortedDict class Config(object): - _instance = None - _parsed_files = [] - _doc = {} - access_key = "" - secret_key = "" - host_base = "s3.amazonaws.com" - host_bucket = "%(bucket)s.s3.amazonaws.com" - simpledb_host = "sdb.amazonaws.com" - cloudfront_host = "cloudfront.amazonaws.com" - cloudfront_resource = "/2010-06-01/distribution" - verbosity = logging.WARNING - progress_meter = True - progress_class = Progress.ProgressCR - send_chunk = 4096 - recv_chunk = 4096 - list_md5 = False - human_readable_sizes = False - extra_headers = SortedDict(ignore_case = True) - force = False - enable = None - get_continue = False - skip_existing = False - recursive = False - acl_public = None - acl_grants = [] - acl_revokes = [] - proxy_host = "" - proxy_port = 3128 - encrypt = False - dry_run = False - preserve_attrs = True - preserve_attrs_list = [ - 'uname', # Verbose owner Name (e.g. 'root') - 'uid', # Numeric user ID (e.g. 0) - 'gname', # Group name (e.g. 'users') - 'gid', # Numeric group ID (e.g. 100) - 'atime', # Last access timestamp - 'mtime', # Modification timestamp - 'ctime', # Creation timestamp - 'mode', # File mode (e.g. rwxr-xr-x = 755) - #'acl', # Full ACL (not yet supported) - ] - delete_removed = False - _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted" - gpg_passphrase = "" - gpg_command = "" - gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" - gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" - use_https = False - bucket_location = "US" - default_mime_type = "binary/octet-stream" - guess_mime_type = True - # List of checks to be performed for 'sync' - sync_checks = ['size', 'md5'] # 'weak-timestamp' - # List of compiled REGEXPs - exclude = [] - include = [] - # Dict mapping compiled REGEXPs back to their textual form - debug_exclude = {} - debug_include = {} - encoding = "utf-8" - urlencoding_mode = "normal" - log_target_prefix = "" - reduced_redundancy = False - parallel = False - workers = 10 - follow_symlinks=False - select_dir = False - max_retries = 5 - retry_delay = 3 - - ## Creating a singleton - def __new__(self, configfile = None): - if self._instance is None: - self._instance = object.__new__(self) - return self._instance - - def __init__(self, configfile = None): - if configfile: - self.read_config_file(configfile) - - def option_list(self): - retval = [] - for option in dir(self): - ## Skip attributes that start with underscore or are not string, int or bool - option_type = type(getattr(Config, option)) - if option.startswith("_") or \ - not (option_type in ( - type("string"), # str - type(42), # int - type(True))): # bool - continue - retval.append(option) - return retval - - def read_config_file(self, configfile): - cp = ConfigParser(configfile) - for option in self.option_list(): - self.update_option(option, cp.get(option)) - self._parsed_files.append(configfile) - - def dump_config(self, stream): - ConfigDumper(stream).dump("default", self) - - def update_option(self, option, value): - if value is None: - return - #### Special treatment of some options - ## verbosity must be known to "logging" module - if option == "verbosity": - try: - setattr(Config, "verbosity", logging._levelNames[value]) - except KeyError: - error("Config: verbosity level '%s' is not valid" % value) - ## allow yes/no, true/false, on/off and 1/0 for boolean options - elif type(getattr(Config, option)) is type(True): # bool - if str(value).lower() in ("true", "yes", "on", "1"): - setattr(Config, option, True) - elif str(value).lower() in ("false", "no", "off", "0"): - setattr(Config, option, False) - else: - error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value)) - elif type(getattr(Config, option)) is type(42): # int - try: - setattr(Config, option, int(value)) - except ValueError, e: - error("Config: value of option '%s' must be an integer, not '%s'" % (option, value)) - else: # string - setattr(Config, option, value) + _instance = None + _parsed_files = [] + _doc = {} + access_key = "" + secret_key = "" + host_base = "s3.amazonaws.com" + host_bucket = "%(bucket)s.s3.amazonaws.com" + simpledb_host = "sdb.amazonaws.com" + cloudfront_host = "cloudfront.amazonaws.com" + cloudfront_resource = "/2010-06-01/distribution" + verbosity = logging.WARNING + progress_meter = True + progress_class = Progress.ProgressCR + send_chunk = 4096 + recv_chunk = 4096 + list_md5 = False + human_readable_sizes = False + extra_headers = SortedDict(ignore_case=True) + force = False + enable = None + get_continue = False + skip_existing = False + recursive = False + acl_public = None + acl_grants = [] + acl_revokes = [] + proxy_host = "" + proxy_port = 3128 + encrypt = False + dry_run = False + preserve_attrs = True + preserve_attrs_list = [ + 'uname', # Verbose owner Name (e.g. 'root') + 'uid', # Numeric user ID (e.g. 0) + 'gname', # Group name (e.g. 'users') + 'gid', # Numeric group ID (e.g. 100) + 'atime', # Last access timestamp + 'mtime', # Modification timestamp + 'ctime', # Creation timestamp + 'mode', # File mode (e.g. rwxr-xr-x = 755) + #'acl', # Full ACL (not yet supported) + ] + delete_removed = False + _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted" + gpg_passphrase = "" + gpg_command = "" + gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" + gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" + use_https = False + bucket_location = "US" + default_mime_type = "binary/octet-stream" + guess_mime_type = True + # List of checks to be performed for 'sync' + sync_checks = ['size', 'md5'] # 'weak-timestamp' + # List of compiled REGEXPs + exclude = [] + include = [] + # Dict mapping compiled REGEXPs back to their textual form + debug_exclude = {} + debug_include = {} + encoding = "utf-8" + urlencoding_mode = "normal" + log_target_prefix = "" + reduced_redundancy = False + parallel = False + workers = 10 + follow_symlinks = False + select_dir = False + max_retries = 5 + retry_delay = 3 + + ## Creating a singleton + def __new__(self, configfile=None): + if self._instance is None: + self._instance = object.__new__(self) + return self._instance + + def __init__(self, configfile=None): + if configfile: + self.read_config_file(configfile) + + def option_list(self): + retval = [] + for option in dir(self): + ## Skip attributes that start with underscore or are not string, int or bool + option_type = type(getattr(Config, option)) + if option.startswith("_") or\ + not (option_type in ( + type("string"), # str + type(42), # int + type(True))): # bool + continue + retval.append(option) + return retval + + def read_config_file(self, configfile): + cp = ConfigParser(configfile) + for option in self.option_list(): + self.update_option(option, cp.get(option)) + self._parsed_files.append(configfile) + + def dump_config(self, stream): + ConfigDumper(stream).dump("default", self) + + def update_option(self, option, value): + if value is None: + return + #### Special treatment of some options + ## verbosity must be known to "logging" module + if option == "verbosity": + try: + setattr(Config, "verbosity", logging._levelNames[value]) + except KeyError: + error("Config: verbosity level '%s' is not valid" % value) + ## allow yes/no, true/false, on/off and 1/0 for boolean options + elif type(getattr(Config, option)) is type(True): # bool + if str(value).lower() in ("true", "yes", "on", "1"): + setattr(Config, option, True) + elif str(value).lower() in ("false", "no", "off", "0"): + setattr(Config, option, False) + else: + error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value)) + elif type(getattr(Config, option)) is type(42): # int + try: + setattr(Config, option, int(value)) + except ValueError, e: + error("Config: value of option '%s' must be an integer, not '%s'" % (option, value)) + else: # string + setattr(Config, option, value) + class ConfigParser(object): - def __init__(self, file, sections = []): - self.cfg = {} - self.parse_file(file, sections) - - def parse_file(self, file, sections = []): - debug("ConfigParser: Reading file '%s'" % file) - if type(sections) != type([]): - sections = [sections] - in_our_section = True - f = open(file, "r") - r_comment = re.compile("^\s*#.*") - r_empty = re.compile("^\s*$") - r_section = re.compile("^\[([^\]]+)\]") - r_data = re.compile("^\s*(?P\w+)\s*=\s*(?P.*)") - r_quotes = re.compile("^\"(.*)\"\s*$") - for line in f: - if r_comment.match(line) or r_empty.match(line): - continue - is_section = r_section.match(line) - if is_section: - section = is_section.groups()[0] - in_our_section = (section in sections) or (len(sections) == 0) - continue - is_data = r_data.match(line) - if is_data and in_our_section: - data = is_data.groupdict() - if r_quotes.match(data["value"]): - data["value"] = data["value"][1:-1] - self.__setitem__(data["key"], data["value"]) - if data["key"] in ("access_key", "secret_key", "gpg_passphrase"): - print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3) - else: - print_value = data["value"] - debug("ConfigParser: %s->%s" % (data["key"], print_value)) - continue - warning("Ignoring invalid line in '%s': %s" % (file, line)) - - def __getitem__(self, name): - return self.cfg[name] - - def __setitem__(self, name, value): - self.cfg[name] = value - - def get(self, name, default = None): - if self.cfg.has_key(name): - return self.cfg[name] - return default + def __init__(self, file, sections=[]): + self.cfg = {} + self.parse_file(file, sections) + + def parse_file(self, file, sections=[]): + debug("ConfigParser: Reading file '%s'" % file) + if type(sections) != type([]): + sections = [sections] + in_our_section = True + f = open(file, "r") + r_comment = re.compile("^\s*#.*") + r_empty = re.compile("^\s*$") + r_section = re.compile("^\[([^\]]+)\]") + r_data = re.compile("^\s*(?P\w+)\s*=\s*(?P.*)") + r_quotes = re.compile("^\"(.*)\"\s*$") + for line in f: + if r_comment.match(line) or r_empty.match(line): + continue + is_section = r_section.match(line) + if is_section: + section = is_section.groups()[0] + in_our_section = (section in sections) or (len(sections) == 0) + continue + is_data = r_data.match(line) + if is_data and in_our_section: + data = is_data.groupdict() + if r_quotes.match(data["value"]): + data["value"] = data["value"][1:-1] + self.__setitem__(data["key"], data["value"]) + if data["key"] in ("access_key", "secret_key", "gpg_passphrase"): + print_value = (data["value"][:2] + "...%d_chars..." + data["value"][-1:]) % (len(data["value"]) - 3) + else: + print_value = data["value"] + debug("ConfigParser: %s->%s" % (data["key"], print_value)) + continue + warning("Ignoring invalid line in '%s': %s" % (file, line)) + + def __getitem__(self, name): + return self.cfg[name] + + def __setitem__(self, name, value): + self.cfg[name] = value + + def get(self, name, default=None): + if self.cfg.has_key(name): + return self.cfg[name] + return default + class ConfigDumper(object): - def __init__(self, stream): - self.stream = stream + def __init__(self, stream): + self.stream = stream - def dump(self, section, config): - self.stream.write("[%s]\n" % section) - for option in config.option_list(): - self.stream.write("%s = %s\n" % (option, getattr(config, option))) + def dump(self, section, config): + self.stream.write("[%s]\n" % section) + for option in config.option_list(): + self.stream.write("%s = %s\n" % (option, getattr(config, option))) diff --git a/S3/Exceptions.py b/S3/Exceptions.py index 9da5e33..38ede0f 100644 --- a/S3/Exceptions.py +++ b/S3/Exceptions.py @@ -7,77 +7,85 @@ from logging import debug, info, warning, error try: - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as ET except ImportError: - import elementtree.ElementTree as ET + import elementtree.ElementTree as ET class S3Exception(Exception): - def __init__(self, message = ""): - self.message = unicodise(message) - - def __str__(self): - ## Call unicode(self) instead of self.message because - ## __unicode__() method could be overriden in subclasses! - return deunicodise(unicode(self)) - - def __unicode__(self): - return self.message - - ## (Base)Exception.message has been deprecated in Python 2.6 - def _get_message(self): - return self._message - def _set_message(self, message): - self._message = message - message = property(_get_message, _set_message) - - -class S3Error (S3Exception): - def __init__(self, response): - self.status = response["status"] - self.reason = response["reason"] - self.info = { - "Code" : "", - "Message" : "", - "Resource" : "" - } - debug("S3Error: %s (%s)" % (self.status, self.reason)) - if response.has_key("headers"): - for header in response["headers"]: - debug("HttpHeader: %s: %s" % (header, response["headers"][header])) - if response.has_key("data"): - tree = getTreeFromXml(response["data"]) - error_node = tree - if not error_node.tag == "Error": - error_node = tree.find(".//Error") - for child in error_node.getchildren(): - if child.text != "": - debug("ErrorXML: " + child.tag + ": " + repr(child.text)) - self.info[child.tag] = child.text - self.code = self.info["Code"] - self.message = self.info["Message"] - self.resource = self.info["Resource"] - - def __unicode__(self): - retval = u"%d " % (self.status) - retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason)) - if self.info.has_key("Message"): - retval += (u": %s" % self.info["Message"]) - return retval + def __init__(self, message=""): + self.message = unicodise(message) + + def __str__(self): + ## Call unicode(self) instead of self.message because + ## __unicode__() method could be overriden in subclasses! + return deunicodise(unicode(self)) + + def __unicode__(self): + return self.message + + ## (Base)Exception.message has been deprecated in Python 2.6 + def _get_message(self): + return self._message + + def _set_message(self, message): + self._message = message + + message = property(_get_message, _set_message) + + +class S3Error(S3Exception): + def __init__(self, response): + self.status = response["status"] + self.reason = response["reason"] + self.info = { + "Code": "", + "Message": "", + "Resource": "" + } + debug("S3Error: %s (%s)" % (self.status, self.reason)) + if response.has_key("headers"): + for header in response["headers"]: + debug("HttpHeader: %s: %s" % (header, response["headers"][header])) + if response.has_key("data"): + tree = getTreeFromXml(response["data"]) + error_node = tree + if not error_node.tag == "Error": + error_node = tree.find(".//Error") + for child in error_node.getchildren(): + if child.text != "": + debug("ErrorXML: " + child.tag + ": " + repr(child.text)) + self.info[child.tag] = child.text + self.code = self.info["Code"] + self.message = self.info["Message"] + self.resource = self.info["Resource"] + + def __unicode__(self): + retval = u"%d " % (self.status) + retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason)) + if self.info.has_key("Message"): + retval += (u": %s" % self.info["Message"]) + return retval + class CloudFrontError(S3Error): - pass - + pass + + class S3UploadError(S3Exception): - pass + pass + class S3DownloadError(S3Exception): - pass + pass + class S3RequestError(S3Exception): - pass + pass + class InvalidFileError(S3Exception): - pass + pass + class ParameterError(S3Exception): - pass + pass diff --git a/S3/Progress.py b/S3/Progress.py index 18e5b27..e085498 100644 --- a/S3/Progress.py +++ b/S3/Progress.py @@ -8,147 +8,150 @@ import Utils class Progress(object): - _stdout = sys.stdout - - def __init__(self, labels, total_size): - self._stdout = sys.stdout - self.new_file(labels, total_size) - - def new_file(self, labels, total_size): - self.labels = labels - self.total_size = total_size - # Set initial_position to something in the - # case we're not counting from 0. For instance - # when appending to a partially downloaded file. - # Setting initial_position will let the speed - # be computed right. - self.initial_position = 0 - self.current_position = self.initial_position - self.time_start = datetime.datetime.now() - self.time_last = self.time_start - self.time_current = self.time_start - - self.display(new_file = True) - - def update(self, current_position = -1, delta_position = -1): - self.time_last = self.time_current - self.time_current = datetime.datetime.now() - if current_position > -1: - self.current_position = current_position - elif delta_position > -1: - self.current_position += delta_position - #else: - # no update, just call display() - self.display() - - def done(self, message): - self.display(done_message = message) - - def output_labels(self): - self._stdout.write(u"%(source)s -> %(destination)s %(extra)s\n" % self.labels) - self._stdout.flush() - - def display(self, new_file = False, done_message = None): - """ - display(new_file = False[/True], done = False[/True]) - - Override this method to provide a nicer output. - """ - if new_file: - self.output_labels() - self.last_milestone = 0 - return - - if self.current_position == self.total_size: - print_size = Utils.formatSize(self.current_position, True) - if print_size[1] != "": print_size[1] += "B" - timedelta = self.time_current - self.time_start - sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 - print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) - self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % - (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) - self._stdout.flush() - return - - rel_position = selfself.current_position * 100 / self.total_size - if rel_position >= self.last_milestone: - self.last_milestone = (int(rel_position) / 5) * 5 - self._stdout.write("%d%% ", self.last_milestone) - self._stdout.flush() - return + _stdout = sys.stdout + + def __init__(self, labels, total_size): + self._stdout = sys.stdout + self.new_file(labels, total_size) + + def new_file(self, labels, total_size): + self.labels = labels + self.total_size = total_size + # Set initial_position to something in the + # case we're not counting from 0. For instance + # when appending to a partially downloaded file. + # Setting initial_position will let the speed + # be computed right. + self.initial_position = 0 + self.current_position = self.initial_position + self.time_start = datetime.datetime.now() + self.time_last = self.time_start + self.time_current = self.time_start + + self.display(new_file=True) + + def update(self, current_position=-1, delta_position=-1): + self.time_last = self.time_current + self.time_current = datetime.datetime.now() + if current_position > -1: + self.current_position = current_position + elif delta_position > -1: + self.current_position += delta_position + #else: + # no update, just call display() + self.display() + + def done(self, message): + self.display(done_message=message) + + def output_labels(self): + self._stdout.write(u"%(source)s -> %(destination)s %(extra)s\n" % self.labels) + self._stdout.flush() + + def display(self, new_file=False, done_message=None): + """ + display(new_file = False[/True], done = False[/True]) + + Override this method to provide a nicer output. + """ + if new_file: + self.output_labels() + self.last_milestone = 0 + return + + if self.current_position == self.total_size: + print_size = Utils.formatSize(self.current_position, True) + if print_size[1] != "": print_size[1] += "B" + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds) / 1000000.0 + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % + (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) + self._stdout.flush() + return + + rel_position = selfself.current_position * 100 / self.total_size + if rel_position >= self.last_milestone: + self.last_milestone = (int(rel_position) / 5) * 5 + self._stdout.write("%d%% ", self.last_milestone) + self._stdout.flush() + return + class ProgressANSI(Progress): - ## http://en.wikipedia.org/wiki/ANSI_escape_code - SCI = '\x1b[' - ANSI_hide_cursor = SCI + "?25l" - ANSI_show_cursor = SCI + "?25h" - ANSI_save_cursor_pos = SCI + "s" - ANSI_restore_cursor_pos = SCI + "u" - ANSI_move_cursor_to_column = SCI + "%uG" - ANSI_erase_to_eol = SCI + "0K" - ANSI_erase_current_line = SCI + "2K" - - def display(self, new_file = False, done_message = None): - """ - display(new_file = False[/True], done_message = None) - """ - if new_file: - self.output_labels() - self._stdout.write(self.ANSI_save_cursor_pos) - self._stdout.flush() - return - - timedelta = self.time_current - self.time_start - sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 - if (sec_elapsed > 0): - print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) - else: - print_speed = (0, "") - self._stdout.write(self.ANSI_restore_cursor_pos) - self._stdout.write(self.ANSI_erase_to_eol) - self._stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { - "current" : str(self.current_position).rjust(len(str(self.total_size))), - "total" : self.total_size, - "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0, - "elapsed" : sec_elapsed, - "speed" : print_speed[0], - "speed_coeff" : print_speed[1] - }) - - if done_message: - self._stdout.write(" %s\n" % done_message) - - self._stdout.flush() +## http://en.wikipedia.org/wiki/ANSI_escape_code + SCI = '\x1b[' + ANSI_hide_cursor = SCI + "?25l" + ANSI_show_cursor = SCI + "?25h" + ANSI_save_cursor_pos = SCI + "s" + ANSI_restore_cursor_pos = SCI + "u" + ANSI_move_cursor_to_column = SCI + "%uG" + ANSI_erase_to_eol = SCI + "0K" + ANSI_erase_current_line = SCI + "2K" + + def display(self, new_file=False, done_message=None): + """ + display(new_file = False[/True], done_message = None) + """ + if new_file: + self.output_labels() + self._stdout.write(self.ANSI_save_cursor_pos) + self._stdout.flush() + return + + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds) / 1000000.0 + if (sec_elapsed > 0): + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + else: + print_speed = (0, "") + self._stdout.write(self.ANSI_restore_cursor_pos) + self._stdout.write(self.ANSI_erase_to_eol) + self._stdout.write( + "%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { + "current": str(self.current_position).rjust(len(str(self.total_size))), + "total": self.total_size, + "percent": self.total_size and (self.current_position * 100 / self.total_size) or 0, + "elapsed": sec_elapsed, + "speed": print_speed[0], + "speed_coeff": print_speed[1] + }) + + if done_message: + self._stdout.write(" %s\n" % done_message) + + self._stdout.flush() + class ProgressCR(Progress): - ## Uses CR char (Carriage Return) just like other progress bars do. - CR_char = chr(13) - - def display(self, new_file = False, done_message = None): - """ - display(new_file = False[/True], done_message = None) - """ - if new_file: - self.output_labels() - return - - timedelta = self.time_current - self.time_start - sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 - if (sec_elapsed > 0): - print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) - else: - print_speed = (0, "") - self._stdout.write(self.CR_char) - output = " %(current)s of %(total)s %(percent)3d%% in %(elapsed)4ds %(speed)7.2f %(speed_coeff)sB/s" % { - "current" : str(self.current_position).rjust(len(str(self.total_size))), - "total" : self.total_size, - "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0, - "elapsed" : sec_elapsed, - "speed" : print_speed[0], - "speed_coeff" : print_speed[1] - } - self._stdout.write(output) - if done_message: - self._stdout.write(" %s\n" % done_message) - - self._stdout.flush() +## Uses CR char (Carriage Return) just like other progress bars do. + CR_char = chr(13) + + def display(self, new_file=False, done_message=None): + """ + display(new_file = False[/True], done_message = None) + """ + if new_file: + self.output_labels() + return + + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds) / 1000000.0 + if (sec_elapsed > 0): + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + else: + print_speed = (0, "") + self._stdout.write(self.CR_char) + output = " %(current)s of %(total)s %(percent)3d%% in %(elapsed)4ds %(speed)7.2f %(speed_coeff)sB/s" % { + "current": str(self.current_position).rjust(len(str(self.total_size))), + "total": self.total_size, + "percent": self.total_size and (self.current_position * 100 / self.total_size) or 0, + "elapsed": sec_elapsed, + "speed": print_speed[0], + "speed_coeff": print_speed[1] + } + self._stdout.write(output) + if done_message: + self._stdout.write(" %s\n" % done_message) + + self._stdout.flush() diff --git a/S3/S3.py b/S3/S3.py index c87493f..a4fa01b 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -14,9 +14,9 @@ from stat import ST_SIZE try: - from hashlib import md5 + from hashlib import md5 except ImportError: - from md5 import md5 + from md5 import md5 from Utils import * from SortedDict import SortedDict @@ -28,753 +28,756 @@ from S3Uri import S3Uri __all__ = [] + class S3Request(object): - def __init__(self, s3, method_string, resource, headers, params = {}): - self.s3 = s3 - self.headers = SortedDict(headers or {}, ignore_case = True) - self.resource = resource - self.method_string = method_string - self.params = params - - self.update_timestamp() - self.sign() - - def update_timestamp(self): - if self.headers.has_key("date"): - del(self.headers["date"]) - self.headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) - - def format_param_str(self): - """ - Format URL parameters from self.params and returns - ?parm1=val1&parm2=val2 or an empty string if there - are no parameters. Output of this function should - be appended directly to self.resource['uri'] - """ - param_str = "" - for param in self.params: - if self.params[param] not in (None, ""): - param_str += "&%s=%s" % (param, self.params[param]) - else: - param_str += "&%s" % param - return param_str and "?" + param_str[1:] - - def sign(self): - h = self.method_string + "\n" - h += self.headers.get("content-md5", "")+"\n" - h += self.headers.get("content-type", "")+"\n" - h += self.headers.get("date", "")+"\n" - for header in self.headers.keys(): - if header.startswith("x-amz-"): - h += header+":"+str(self.headers[header])+"\n" - if self.resource['bucket']: - h += "/" + self.resource['bucket'] - h += self.resource['uri'] - debug("SignHeaders: " + repr(h)) - signature = sign_string(h) - - self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature - - def get_triplet(self): - self.update_timestamp() - self.sign() - resource = dict(self.resource) ## take a copy - resource['uri'] += self.format_param_str() - return (self.method_string, resource, self.headers) + def __init__(self, s3, method_string, resource, headers, params={}): + self.s3 = s3 + self.headers = SortedDict(headers or {}, ignore_case=True) + self.resource = resource + self.method_string = method_string + self.params = params + + self.update_timestamp() + self.sign() + + def update_timestamp(self): + if self.headers.has_key("date"): + del(self.headers["date"]) + self.headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) + + def format_param_str(self): + """ + Format URL parameters from self.params and returns + ?parm1=val1&parm2=val2 or an empty string if there + are no parameters. Output of this function should + be appended directly to self.resource['uri'] + """ + param_str = "" + for param in self.params: + if self.params[param] not in (None, ""): + param_str += "&%s=%s" % (param, self.params[param]) + else: + param_str += "&%s" % param + return param_str and "?" + param_str[1:] + + def sign(self): + h = self.method_string + "\n" + h += self.headers.get("content-md5", "") + "\n" + h += self.headers.get("content-type", "") + "\n" + h += self.headers.get("date", "") + "\n" + for header in self.headers.keys(): + if header.startswith("x-amz-"): + h += header + ":" + str(self.headers[header]) + "\n" + if self.resource['bucket']: + h += "/" + self.resource['bucket'] + h += self.resource['uri'] + debug("SignHeaders: " + repr(h)) + signature = sign_string(h) + + self.headers["Authorization"] = "AWS " + self.s3.config.access_key + ":" + signature + + def get_triplet(self): + self.update_timestamp() + self.sign() + resource = dict(self.resource) ## take a copy + resource['uri'] += self.format_param_str() + return (self.method_string, resource, self.headers) + class S3(object): - http_methods = BidirMap( - GET = 0x01, - PUT = 0x02, - HEAD = 0x04, - DELETE = 0x08, - MASK = 0x0F, - ) - - targets = BidirMap( - SERVICE = 0x0100, - BUCKET = 0x0200, - OBJECT = 0x0400, - MASK = 0x0700, - ) - - operations = BidirMap( - UNDFINED = 0x0000, - LIST_ALL_BUCKETS = targets["SERVICE"] | http_methods["GET"], - BUCKET_CREATE = targets["BUCKET"] | http_methods["PUT"], - BUCKET_LIST = targets["BUCKET"] | http_methods["GET"], - BUCKET_DELETE = targets["BUCKET"] | http_methods["DELETE"], - OBJECT_PUT = targets["OBJECT"] | http_methods["PUT"], - OBJECT_GET = targets["OBJECT"] | http_methods["GET"], - OBJECT_HEAD = targets["OBJECT"] | http_methods["HEAD"], - OBJECT_DELETE = targets["OBJECT"] | http_methods["DELETE"], - ) - - codes = { - "NoSuchBucket" : "Bucket '%s' does not exist", - "AccessDenied" : "Access to bucket '%s' was denied", - "BucketAlreadyExists" : "Bucket '%s' already exists", - } - - ## S3 sometimes sends HTTP-307 response - redir_map = {} - - def __init__(self, config): - self.config = config - - def get_connection(self, bucket): - if self.config.proxy_host != "": - return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port) - else: - if self.config.use_https: - return httplib.HTTPSConnection(self.get_hostname(bucket)) - else: - return httplib.HTTPConnection(self.get_hostname(bucket)) - - def get_hostname(self, bucket): - if bucket and check_bucket_name_dns_conformity(bucket): - if self.redir_map.has_key(bucket): - host = self.redir_map[bucket] - else: - host = getHostnameFromBucket(bucket) - else: - host = self.config.host_base - debug('get_hostname(%s): %s' % (bucket, host)) - return host - - def set_hostname(self, bucket, redir_hostname): - self.redir_map[bucket] = redir_hostname - - def format_uri(self, resource): - if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']): - uri = "/%s%s" % (resource['bucket'], resource['uri']) - else: - uri = resource['uri'] - if self.config.proxy_host != "": - uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri) - debug('format_uri(): ' + uri) - return uri - - ## Commands / Actions - def list_all_buckets(self): - request = self.create_request("LIST_ALL_BUCKETS") - response = self.send_request(request) - response["list"] = getListFromXml(response["data"], "Bucket") - return response - - def bucket_list(self, bucket, prefix = None, recursive = None): - def _list_truncated(data): - ## can either be "true" or "false" or be missing completely - is_truncated = getTextFromXml(data, ".//IsTruncated") or "false" - return is_truncated.lower() != "false" - - def _get_contents(data): - return getListFromXml(data, "Contents") - - def _get_common_prefixes(data): - return getListFromXml(data, "CommonPrefixes") - - uri_params = {} - truncated = True - list = [] - prefixes = [] - - while truncated: - response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params) - current_list = _get_contents(response["data"]) - current_prefixes = _get_common_prefixes(response["data"]) - truncated = _list_truncated(response["data"]) - if truncated: - if current_list: - uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"]) - else: - uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"]) - debug("Listing continues after '%s'" % uri_params['marker']) - - list += current_list - prefixes += current_prefixes - - response['list'] = list - response['common_prefixes'] = prefixes - return response - - def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}): - if prefix: - uri_params['prefix'] = self.urlencode_string(prefix) - if not self.config.recursive and not recursive: - uri_params['delimiter'] = "/" - request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) - response = self.send_request(request) - #debug(response) - return response - - def bucket_create(self, bucket, bucket_location = None): - headers = SortedDict(ignore_case = True) - body = "" - if bucket_location and bucket_location.strip().upper() != "US": - bucket_location = bucket_location.strip() - if bucket_location.upper() == "EU": - bucket_location = bucket_location.upper() - else: - bucket_location = bucket_location.lower() - body = "" - body += bucket_location - body += "" - debug("bucket_location: " + body) - check_bucket_name(bucket, dns_strict = True) - else: - check_bucket_name(bucket, dns_strict = False) - if self.config.acl_public: - headers["x-amz-acl"] = "public-read" - request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers) - response = self.send_request(request, body) - return response - - def bucket_delete(self, bucket): - request = self.create_request("BUCKET_DELETE", bucket = bucket) - response = self.send_request(request) - return response - - def bucket_info(self, uri): - request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location") - response = self.send_request(request) - response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any" - return response - - def object_put(self, filename, uri, extra_headers = None, extra_label = ""): - # TODO TODO - # Make it consistent with stream-oriented object_get() - if uri.type != "s3": - raise ValueError("Expected URI type 's3', got '%s'" % uri.type) - - if not os.path.isfile(filename): - raise InvalidFileError(u"%s is not a regular file" % unicodise(filename)) - try: - file = open(filename, "rb") - size = os.stat(filename)[ST_SIZE] - except IOError, e: - raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror)) - headers = SortedDict(ignore_case = True) - if extra_headers: - headers.update(extra_headers) - headers["content-length"] = size - content_type = None - if self.config.guess_mime_type: - content_type = mimetypes.guess_type(filename)[0] - if not content_type: - content_type = self.config.default_mime_type - debug("Content-Type set to '%s'" % content_type) - headers["content-type"] = content_type - if self.config.acl_public: - headers["x-amz-acl"] = "public-read" - if self.config.reduced_redundancy: - headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY" - request = self.create_request("OBJECT_PUT", uri = uri, headers = headers) - labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label } - response = self.send_file(request, file, labels) - return response - - def object_get(self, uri, stream, start_position = 0, extra_label = ""): - if uri.type != "s3": - raise ValueError("Expected URI type 's3', got '%s'" % uri.type) - request = self.create_request("OBJECT_GET", uri = uri) - labels = { 'source' : unicodise(uri.uri()), 'destination' : unicodise(stream.name), 'extra' : extra_label } - response = self.recv_file(request, stream, labels, start_position) - return response - - def object_delete(self, uri): - if uri.type != "s3": - raise ValueError("Expected URI type 's3', got '%s'" % uri.type) - request = self.create_request("OBJECT_DELETE", uri = uri) - response = self.send_request(request) - return response - - def object_copy(self, src_uri, dst_uri, extra_headers = None): - if src_uri.type != "s3": - raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type) - if dst_uri.type != "s3": - raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type) - headers = SortedDict(ignore_case = True) - headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object())) - ## TODO: For now COPY, later maybe add a switch? - headers['x-amz-metadata-directive'] = "COPY" - if self.config.acl_public: - headers["x-amz-acl"] = "public-read" - if self.config.reduced_redundancy: - headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY" - # if extra_headers: - # headers.update(extra_headers) - request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers) - response = self.send_request(request) - return response - - def object_move(self, src_uri, dst_uri, extra_headers = None): - response_copy = self.object_copy(src_uri, dst_uri, extra_headers) - debug("Object %s copied to %s" % (src_uri, dst_uri)) - if getRootTagName(response_copy["data"]) == "CopyObjectResult": - response_delete = self.object_delete(src_uri) - debug("Object %s deleted" % src_uri) - return response_copy - - def object_info(self, uri): - request = self.create_request("OBJECT_HEAD", uri = uri) - response = self.send_request(request) - return response - - def get_acl(self, uri): - if uri.has_object(): - request = self.create_request("OBJECT_GET", uri = uri, extra = "?acl") - else: - request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl") - - response = self.send_request(request) - acl = ACL(response['data']) - return acl - - def set_acl(self, uri, acl): - if uri.has_object(): - request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl") - else: - request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl") - - body = str(acl) - debug(u"set_acl(%s): acl-xml: %s" % (uri, body)) - response = self.send_request(request, body) - return response - - def get_accesslog(self, uri): - request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging") - response = self.send_request(request) - accesslog = AccessLog(response['data']) - return accesslog - - def set_accesslog_acl(self, uri): - acl = self.get_acl(uri) - debug("Current ACL(%s): %s" % (uri.uri(), str(acl))) - acl.appendGrantee(GranteeLogDelivery("READ_ACP")) - acl.appendGrantee(GranteeLogDelivery("WRITE")) - debug("Updated ACL(%s): %s" % (uri.uri(), str(acl))) - self.set_acl(uri, acl) - - def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False): - request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging") - accesslog = AccessLog() - if enable: - accesslog.enableLogging(log_target_prefix_uri) - accesslog.setAclPublic(acl_public) - else: - accesslog.disableLogging() - body = str(accesslog) - debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body)) - try: - response = self.send_request(request, body) - except S3Error, e: - if e.info['Code'] == "InvalidTargetBucketForLogging": - info("Setting up log-delivery ACL for target bucket.") - self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket())) - response = self.send_request(request, body) - else: - raise - return accesslog, response - - ## Low level methods - def urlencode_string(self, string, urlencoding_mode = None): - if type(string) == unicode: - string = string.encode("utf-8") - - if urlencoding_mode is None: - urlencoding_mode = self.config.urlencoding_mode - - if urlencoding_mode == "verbatim": - ## Don't do any pre-processing - return string - - encoded = "" - ## List of characters that must be escaped for S3 - ## Haven't found this in any official docs - ## but my tests show it's more less correct. - ## If you start getting InvalidSignature errors - ## from S3 check the error headers returned - ## from S3 to see whether the list hasn't - ## changed. - for c in string: # I'm not sure how to know in what encoding - # 'object' is. Apparently "type(object)==str" - # but the contents is a string of unicode - # bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k' - # Don't know what it will do on non-utf8 - # systems. - # [hope that sounds reassuring ;-)] - o = ord(c) - if (o < 0x20 or o == 0x7f): - if urlencoding_mode == "fixbucket": - encoded += "%%%02X" % o - else: - error(u"Non-printable character 0x%02x in: %s" % (o, string)) - error(u"Please report it to s3tools-bugs@lists.sourceforge.net") - encoded += replace_nonprintables(c) - elif (o == 0x20 or # Space and below - o == 0x22 or # " - o == 0x23 or # # - o == 0x25 or # % (escape character) - o == 0x26 or # & - o == 0x2B or # + (or it would become ) - o == 0x3C or # < - o == 0x3E or # > - o == 0x3F or # ? - o == 0x60 or # ` - o >= 123): # { and above, including >= 128 for UTF-8 - encoded += "%%%02X" % o - else: - encoded += c - debug("String '%s' encoded to '%s'" % (string, encoded)) - return encoded - - def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params): - resource = { 'bucket' : None, 'uri' : "/" } - - if uri and (bucket or object): - raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied") - ## If URI is given use that instead of bucket/object parameters - if uri: - bucket = uri.bucket() - object = uri.has_object() and uri.object() or None - - if bucket: - resource['bucket'] = str(bucket) - if object: - resource['uri'] = "/" + self.urlencode_string(object) - if extra: - resource['uri'] += extra - - method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"]) - - request = S3Request(self, method_string, resource, headers, params) - - debug("CreateRequest: resource[uri]=" + resource['uri']) - return request - - def _fail_wait(self, retries): - # Wait a few seconds. The more it fails the more we wait. - return (self.config.max_retries - retries + 1) * self.config.retry_delay - - def send_request(self, request, body = None, retries = -1): - if retries == -1: - retries = self.config.max_retries - method_string, resource, headers = request.get_triplet() - debug("Processing request, please wait...") - if not headers.has_key('content-length'): - headers['content-length'] = body and len(body) or 0 - try: - conn = self.get_connection(resource['bucket']) - conn.request(method_string, self.format_uri(resource), body, headers) - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - response["data"] = http_response.read() - debug("Response: " + str(response)) - conn.close() - except Exception, e: - if retries: - warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - return self.send_request(request, body, retries - 1) - else: - raise S3RequestError("Request failed for: %s" % resource['uri']) - - if response["status"] == 307: - ## RedirectPermanent - redir_bucket = getTextFromXml(response['data'], ".//Bucket") - redir_hostname = getTextFromXml(response['data'], ".//Endpoint") - self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) - return self.send_request(request, body) - - if response["status"] >= 500: - e = S3Error(response) - if retries: - warning(u"Retrying failed request: %s" % resource['uri']) - warning(unicode(e)) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - return self.send_request(request, body, retries - 1) - else: - raise e - - if response["status"] < 200 or response["status"] > 299: - raise S3Error(response) - - return response - - def send_file(self, request, file, labels, throttle = 0, retries = -1): - if retries == -1: - retries = self.config.max_retries - method_string, resource, headers = request.get_triplet() - size_left = size_total = headers.get("content-length") - if self.config.progress_meter: - progress = self.config.progress_class(labels, size_total) - else: - info("Sending file '%s', please wait..." % file.name) - timestamp_start = time.time() - try: - conn = self.get_connection(resource['bucket']) - conn.connect() - conn.putrequest(method_string, self.format_uri(resource)) - for header in headers.keys(): - conn.putheader(header, str(headers[header])) - conn.endheaders() - except Exception, e: - if self.config.progress_meter: - progress.done("failed") - if retries: - warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - # Connection error -> same throttle value - return self.send_file(request, file, labels, throttle, retries - 1) - else: - raise S3UploadError("Upload failed for: %s" % resource['uri']) - file.seek(0) - md5_hash = md5() - try: - while (size_left > 0): - #debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) - data = file.read(self.config.send_chunk) - md5_hash.update(data) - conn.send(data) - if self.config.progress_meter: - progress.update(delta_position = len(data)) - size_left -= len(data) - if throttle: - time.sleep(throttle) - md5_computed = md5_hash.hexdigest() - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - response["data"] = http_response.read() - response["size"] = size_total - conn.close() - debug(u"Response: %s" % response) - except Exception, e: - if self.config.progress_meter: - progress.done("failed") - if retries: - if retries < self.config.max_retries: - throttle = throttle and throttle * 5 or 0.01 - warning("Upload failed: %s (%s)" % (resource['uri'], e)) - warning("Retrying on lower speed (throttle=%0.2f)" % throttle) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - # Connection error -> same throttle value - return self.send_file(request, file, labels, throttle, retries - 1) - else: - debug("Giving up on '%s' %s" % (file.name, e)) - raise S3UploadError("Upload failed for: %s" % resource['uri']) - - timestamp_end = time.time() - response["elapsed"] = timestamp_end - timestamp_start - response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) - - if self.config.progress_meter: - ## The above conn.close() takes some time -> update() progress meter - ## to correct the average speed. Otherwise people will complain that - ## 'progress' and response["speed"] are inconsistent ;-) - progress.update() - progress.done("done") - - if response["status"] == 307: - ## RedirectPermanent - redir_bucket = getTextFromXml(response['data'], ".//Bucket") - redir_hostname = getTextFromXml(response['data'], ".//Endpoint") - self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) - return self.send_file(request, file, labels) - - # S3 from time to time doesn't send ETag back in a response :-( - # Force re-upload here. - if not response['headers'].has_key('etag'): - response['headers']['etag'] = '' - - if response["status"] < 200 or response["status"] > 299: - try_retry = False - if response["status"] >= 500: - ## AWS internal error - retry - try_retry = True - elif response["status"] >= 400: - err = S3Error(response) - ## Retriable client error? - if err.code in [ 'BadDigest', 'OperationAborted', 'TokenRefreshRequired', 'RequestTimeout' ]: - try_retry = True - - if try_retry: - if retries: - warning("Upload failed: %s (%s)" % (resource['uri'], S3Error(response))) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - return self.send_file(request, file, labels, throttle, retries - 1) - else: - warning("Too many failures. Giving up on '%s'" % (file.name)) - raise S3UploadError - - ## Non-recoverable error - raise S3Error(response) - - debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"])) - if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest(): - warning("MD5 Sums don't match!") - if retries: - warning("Retrying upload of %s" % (file.name)) - return self.send_file(request, file, labels, throttle, retries - 1) - else: - warning("Too many failures. Giving up on '%s'" % (file.name)) - raise S3UploadError - - return response - - def recv_file(self, request, stream, labels, start_position = 0, retries = -1): - if retries == -1: - retries = self.config.max_retries - method_string, resource, headers = request.get_triplet() - if self.config.progress_meter: - progress = self.config.progress_class(labels, 0) - else: - info("Receiving file '%s', please wait..." % stream.name) - timestamp_start = time.time() - try: - conn = self.get_connection(resource['bucket']) - conn.connect() - conn.putrequest(method_string, self.format_uri(resource)) - for header in headers.keys(): - conn.putheader(header, str(headers[header])) - if start_position > 0: - debug("Requesting Range: %d .. end" % start_position) - conn.putheader("Range", "bytes=%d-" % start_position) - conn.endheaders() - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - debug("Response: %s" % response) - except Exception, e: - if self.config.progress_meter: - progress.done("failed") - if retries: - warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - # Connection error -> same throttle value - return self.recv_file(request, stream, labels, start_position, retries - 1) - else: - raise S3DownloadError("Download failed for: %s" % resource['uri']) - - if response["status"] == 307: - ## RedirectPermanent - response['data'] = http_response.read() - redir_bucket = getTextFromXml(response['data'], ".//Bucket") - redir_hostname = getTextFromXml(response['data'], ".//Endpoint") - self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) - return self.recv_file(request, stream, labels) - - if response["status"] < 200 or response["status"] > 299: - raise S3Error(response) - - if start_position == 0: - # Only compute MD5 on the fly if we're downloading from beginning - # Otherwise we'd get a nonsense. - md5_hash = md5() - size_left = int(response["headers"]["content-length"]) - size_total = start_position + size_left - current_position = start_position - - if self.config.progress_meter: - progress.total_size = size_total - progress.initial_position = current_position - progress.current_position = current_position - - try: - while (current_position < size_total): - this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left - data = http_response.read(this_chunk) - stream.write(data) - if start_position == 0: - md5_hash.update(data) - current_position += len(data) - ## Call progress meter from here... - if self.config.progress_meter: - progress.update(delta_position = len(data)) - conn.close() - except Exception, e: - if self.config.progress_meter: - progress.done("failed") - if retries: - warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) - - if self._fail_wait(retries) > 0: - warning("Waiting %d sec..." % self._fail_wait(retries)) - time.sleep(self._fail_wait(retries)) - - # Connection error -> same throttle value - return self.recv_file(request, stream, labels, current_position, retries - 1) - else: - raise S3DownloadError("Download failed for: %s" % resource['uri']) - - stream.flush() - timestamp_end = time.time() - - if self.config.progress_meter: - ## The above stream.flush() may take some time -> update() progress meter - ## to correct the average speed. Otherwise people will complain that - ## 'progress' and response["speed"] are inconsistent ;-) - progress.update() - progress.done("done") - - if start_position == 0: - # Only compute MD5 on the fly if we were downloading from the beginning - response["md5"] = md5_hash.hexdigest() - else: - # Otherwise try to compute MD5 of the output file - try: - response["md5"] = hash_file_md5(stream.name) - except IOError, e: - if e.errno != errno.ENOENT: - warning("Unable to open file: %s: %s" % (stream.name, e)) - warning("Unable to verify MD5. Assume it matches.") - response["md5"] = response["headers"]["etag"] - - response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0 - response["elapsed"] = timestamp_end - timestamp_start - response["size"] = current_position - response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) - if response["size"] != start_position + long(response["headers"]["content-length"]): - warning("Reported size (%s) does not match received size (%s)" % ( - start_position + response["headers"]["content-length"], response["size"])) - debug("ReceiveFile: Computed MD5 = %s" % response["md5"]) - if not response["md5match"]: - warning("MD5 signatures do not match: computed=%s, received=%s" % ( - response["md5"], response["headers"]["etag"])) - return response + http_methods = BidirMap( + GET=0x01, + PUT=0x02, + HEAD=0x04, + DELETE=0x08, + MASK=0x0F, + ) + + targets = BidirMap( + SERVICE=0x0100, + BUCKET=0x0200, + OBJECT=0x0400, + MASK=0x0700, + ) + + operations = BidirMap( + UNDFINED=0x0000, + LIST_ALL_BUCKETS=targets["SERVICE"] | http_methods["GET"], + BUCKET_CREATE=targets["BUCKET"] | http_methods["PUT"], + BUCKET_LIST=targets["BUCKET"] | http_methods["GET"], + BUCKET_DELETE=targets["BUCKET"] | http_methods["DELETE"], + OBJECT_PUT=targets["OBJECT"] | http_methods["PUT"], + OBJECT_GET=targets["OBJECT"] | http_methods["GET"], + OBJECT_HEAD=targets["OBJECT"] | http_methods["HEAD"], + OBJECT_DELETE=targets["OBJECT"] | http_methods["DELETE"], + ) + + codes = { + "NoSuchBucket": "Bucket '%s' does not exist", + "AccessDenied": "Access to bucket '%s' was denied", + "BucketAlreadyExists": "Bucket '%s' already exists", + } + + ## S3 sometimes sends HTTP-307 response + redir_map = {} + + def __init__(self, config): + self.config = config + + def get_connection(self, bucket): + if self.config.proxy_host != "": + return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port) + else: + if self.config.use_https: + return httplib.HTTPSConnection(self.get_hostname(bucket)) + else: + return httplib.HTTPConnection(self.get_hostname(bucket)) + + def get_hostname(self, bucket): + if bucket and check_bucket_name_dns_conformity(bucket): + if self.redir_map.has_key(bucket): + host = self.redir_map[bucket] + else: + host = getHostnameFromBucket(bucket) + else: + host = self.config.host_base + debug('get_hostname(%s): %s' % (bucket, host)) + return host + + def set_hostname(self, bucket, redir_hostname): + self.redir_map[bucket] = redir_hostname + + def format_uri(self, resource): + if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']): + uri = "/%s%s" % (resource['bucket'], resource['uri']) + else: + uri = resource['uri'] + if self.config.proxy_host != "": + uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri) + debug('format_uri(): ' + uri) + return uri + + ## Commands / Actions + def list_all_buckets(self): + request = self.create_request("LIST_ALL_BUCKETS") + response = self.send_request(request) + response["list"] = getListFromXml(response["data"], "Bucket") + return response + + def bucket_list(self, bucket, prefix=None, recursive=None): + def _list_truncated(data): + ## can either be "true" or "false" or be missing completely + is_truncated = getTextFromXml(data, ".//IsTruncated") or "false" + return is_truncated.lower() != "false" + + def _get_contents(data): + return getListFromXml(data, "Contents") + + def _get_common_prefixes(data): + return getListFromXml(data, "CommonPrefixes") + + uri_params = {} + truncated = True + list = [] + prefixes = [] + + while truncated: + response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params) + current_list = _get_contents(response["data"]) + current_prefixes = _get_common_prefixes(response["data"]) + truncated = _list_truncated(response["data"]) + if truncated: + if current_list: + uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"]) + else: + uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"]) + debug("Listing continues after '%s'" % uri_params['marker']) + + list += current_list + prefixes += current_prefixes + + response['list'] = list + response['common_prefixes'] = prefixes + return response + + def bucket_list_noparse(self, bucket, prefix=None, recursive=None, uri_params={}): + if prefix: + uri_params['prefix'] = self.urlencode_string(prefix) + if not self.config.recursive and not recursive: + uri_params['delimiter'] = "/" + request = self.create_request("BUCKET_LIST", bucket=bucket, **uri_params) + response = self.send_request(request) + #debug(response) + return response + + def bucket_create(self, bucket, bucket_location=None): + headers = SortedDict(ignore_case=True) + body = "" + if bucket_location and bucket_location.strip().upper() != "US": + bucket_location = bucket_location.strip() + if bucket_location.upper() == "EU": + bucket_location = bucket_location.upper() + else: + bucket_location = bucket_location.lower() + body = "" + body += bucket_location + body += "" + debug("bucket_location: " + body) + check_bucket_name(bucket, dns_strict=True) + else: + check_bucket_name(bucket, dns_strict=False) + if self.config.acl_public: + headers["x-amz-acl"] = "public-read" + request = self.create_request("BUCKET_CREATE", bucket=bucket, headers=headers) + response = self.send_request(request, body) + return response + + def bucket_delete(self, bucket): + request = self.create_request("BUCKET_DELETE", bucket=bucket) + response = self.send_request(request) + return response + + def bucket_info(self, uri): + request = self.create_request("BUCKET_LIST", bucket=uri.bucket(), extra="?location") + response = self.send_request(request) + response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any" + return response + + def object_put(self, filename, uri, extra_headers=None, extra_label=""): + # TODO TODO + # Make it consistent with stream-oriented object_get() + if uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % uri.type) + + if not os.path.isfile(filename): + raise InvalidFileError(u"%s is not a regular file" % unicodise(filename)) + try: + file = open(filename, "rb") + size = os.stat(filename)[ST_SIZE] + except IOError, e: + raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror)) + headers = SortedDict(ignore_case=True) + if extra_headers: + headers.update(extra_headers) + headers["content-length"] = size + content_type = None + if self.config.guess_mime_type: + content_type = mimetypes.guess_type(filename)[0] + if not content_type: + content_type = self.config.default_mime_type + debug("Content-Type set to '%s'" % content_type) + headers["content-type"] = content_type + if self.config.acl_public: + headers["x-amz-acl"] = "public-read" + if self.config.reduced_redundancy: + headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY" + request = self.create_request("OBJECT_PUT", uri=uri, headers=headers) + labels = {'source': unicodise(filename), 'destination': unicodise(uri.uri()), 'extra': extra_label} + response = self.send_file(request, file, labels) + return response + + def object_get(self, uri, stream, start_position=0, extra_label=""): + if uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % uri.type) + request = self.create_request("OBJECT_GET", uri=uri) + labels = {'source': unicodise(uri.uri()), 'destination': unicodise(stream.name), 'extra': extra_label} + response = self.recv_file(request, stream, labels, start_position) + return response + + def object_delete(self, uri): + if uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % uri.type) + request = self.create_request("OBJECT_DELETE", uri=uri) + response = self.send_request(request) + return response + + def object_copy(self, src_uri, dst_uri, extra_headers=None): + if src_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type) + if dst_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type) + headers = SortedDict(ignore_case=True) + headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object())) + ## TODO: For now COPY, later maybe add a switch? + headers['x-amz-metadata-directive'] = "COPY" + if self.config.acl_public: + headers["x-amz-acl"] = "public-read" + if self.config.reduced_redundancy: + headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY" + # if extra_headers: + # headers.update(extra_headers) + request = self.create_request("OBJECT_PUT", uri=dst_uri, headers=headers) + response = self.send_request(request) + return response + + def object_move(self, src_uri, dst_uri, extra_headers=None): + response_copy = self.object_copy(src_uri, dst_uri, extra_headers) + debug("Object %s copied to %s" % (src_uri, dst_uri)) + if getRootTagName(response_copy["data"]) == "CopyObjectResult": + response_delete = self.object_delete(src_uri) + debug("Object %s deleted" % src_uri) + return response_copy + + def object_info(self, uri): + request = self.create_request("OBJECT_HEAD", uri=uri) + response = self.send_request(request) + return response + + def get_acl(self, uri): + if uri.has_object(): + request = self.create_request("OBJECT_GET", uri=uri, extra="?acl") + else: + request = self.create_request("BUCKET_LIST", bucket=uri.bucket(), extra="?acl") + + response = self.send_request(request) + acl = ACL(response['data']) + return acl + + def set_acl(self, uri, acl): + if uri.has_object(): + request = self.create_request("OBJECT_PUT", uri=uri, extra="?acl") + else: + request = self.create_request("BUCKET_CREATE", bucket=uri.bucket(), extra="?acl") + + body = str(acl) + debug(u"set_acl(%s): acl-xml: %s" % (uri, body)) + response = self.send_request(request, body) + return response + + def get_accesslog(self, uri): + request = self.create_request("BUCKET_LIST", bucket=uri.bucket(), extra="?logging") + response = self.send_request(request) + accesslog = AccessLog(response['data']) + return accesslog + + def set_accesslog_acl(self, uri): + acl = self.get_acl(uri) + debug("Current ACL(%s): %s" % (uri.uri(), str(acl))) + acl.appendGrantee(GranteeLogDelivery("READ_ACP")) + acl.appendGrantee(GranteeLogDelivery("WRITE")) + debug("Updated ACL(%s): %s" % (uri.uri(), str(acl))) + self.set_acl(uri, acl) + + def set_accesslog(self, uri, enable, log_target_prefix_uri=None, acl_public=False): + request = self.create_request("BUCKET_CREATE", bucket=uri.bucket(), extra="?logging") + accesslog = AccessLog() + if enable: + accesslog.enableLogging(log_target_prefix_uri) + accesslog.setAclPublic(acl_public) + else: + accesslog.disableLogging() + body = str(accesslog) + debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body)) + try: + response = self.send_request(request, body) + except S3Error, e: + if e.info['Code'] == "InvalidTargetBucketForLogging": + info("Setting up log-delivery ACL for target bucket.") + self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket())) + response = self.send_request(request, body) + else: + raise + return accesslog, response + + ## Low level methods + def urlencode_string(self, string, urlencoding_mode=None): + if type(string) == unicode: + string = string.encode("utf-8") + + if urlencoding_mode is None: + urlencoding_mode = self.config.urlencoding_mode + + if urlencoding_mode == "verbatim": + ## Don't do any pre-processing + return string + + encoded = "" + ## List of characters that must be escaped for S3 + ## Haven't found this in any official docs + ## but my tests show it's more less correct. + ## If you start getting InvalidSignature errors + ## from S3 check the error headers returned + ## from S3 to see whether the list hasn't + ## changed. + for c in string: # I'm not sure how to know in what encoding + # 'object' is. Apparently "type(object)==str" + # but the contents is a string of unicode + # bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k' + # Don't know what it will do on non-utf8 + # systems. + # [hope that sounds reassuring ;-)] + o = ord(c) + if (o < 0x20 or o == 0x7f): + if urlencoding_mode == "fixbucket": + encoded += "%%%02X" % o + else: + error(u"Non-printable character 0x%02x in: %s" % (o, string)) + error(u"Please report it to s3tools-bugs@lists.sourceforge.net") + encoded += replace_nonprintables(c) + elif (o == 0x20 or # Space and below + o == 0x22 or # " + o == 0x23 or # # + o == 0x25 or # % (escape character) + o == 0x26 or # & + o == 0x2B or # + (or it would become ) + o == 0x3C or # < + o == 0x3E or # > + o == 0x3F or # ? + o == 0x60 or # ` + o >= 123): # { and above, including >= 128 for UTF-8 + encoded += "%%%02X" % o + else: + encoded += c + debug("String '%s' encoded to '%s'" % (string, encoded)) + return encoded + + def create_request(self, operation, uri=None, bucket=None, object=None, headers=None, extra=None, **params): + resource = {'bucket': None, 'uri': "/"} + + if uri and (bucket or object): + raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied") + ## If URI is given use that instead of bucket/object parameters + if uri: + bucket = uri.bucket() + object = uri.has_object() and uri.object() or None + + if bucket: + resource['bucket'] = str(bucket) + if object: + resource['uri'] = "/" + self.urlencode_string(object) + if extra: + resource['uri'] += extra + + method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"]) + + request = S3Request(self, method_string, resource, headers, params) + + debug("CreateRequest: resource[uri]=" + resource['uri']) + return request + + def _fail_wait(self, retries): + # Wait a few seconds. The more it fails the more we wait. + return (self.config.max_retries - retries + 1) * self.config.retry_delay + + def send_request(self, request, body=None, retries=-1): + if retries == -1: + retries = self.config.max_retries + method_string, resource, headers = request.get_triplet() + debug("Processing request, please wait...") + if not headers.has_key('content-length'): + headers['content-length'] = body and len(body) or 0 + try: + conn = self.get_connection(resource['bucket']) + conn.request(method_string, self.format_uri(resource), body, headers) + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + response["data"] = http_response.read() + debug("Response: " + str(response)) + conn.close() + except Exception, e: + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + return self.send_request(request, body, retries - 1) + else: + raise S3RequestError("Request failed for: %s" % resource['uri']) + + if response["status"] == 307: + ## RedirectPermanent + redir_bucket = getTextFromXml(response['data'], ".//Bucket") + redir_hostname = getTextFromXml(response['data'], ".//Endpoint") + self.set_hostname(redir_bucket, redir_hostname) + warning("Redirected to: %s" % (redir_hostname)) + return self.send_request(request, body) + + if response["status"] >= 500: + e = S3Error(response) + if retries: + warning(u"Retrying failed request: %s" % resource['uri']) + warning(unicode(e)) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + return self.send_request(request, body, retries - 1) + else: + raise e + + if response["status"] < 200 or response["status"] > 299: + raise S3Error(response) + + return response + + def send_file(self, request, file, labels, throttle=0, retries=-1): + if retries == -1: + retries = self.config.max_retries + method_string, resource, headers = request.get_triplet() + size_left = size_total = headers.get("content-length") + if self.config.progress_meter: + progress = self.config.progress_class(labels, size_total) + else: + info("Sending file '%s', please wait..." % file.name) + timestamp_start = time.time() + try: + conn = self.get_connection(resource['bucket']) + conn.connect() + conn.putrequest(method_string, self.format_uri(resource)) + for header in headers.keys(): + conn.putheader(header, str(headers[header])) + conn.endheaders() + except Exception, e: + if self.config.progress_meter: + progress.done("failed") + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + # Connection error -> same throttle value + return self.send_file(request, file, labels, throttle, retries - 1) + else: + raise S3UploadError("Upload failed for: %s" % resource['uri']) + file.seek(0) + md5_hash = md5() + try: + while (size_left > 0): + #debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) + data = file.read(self.config.send_chunk) + md5_hash.update(data) + conn.send(data) + if self.config.progress_meter: + progress.update(delta_position=len(data)) + size_left -= len(data) + if throttle: + time.sleep(throttle) + md5_computed = md5_hash.hexdigest() + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + response["data"] = http_response.read() + response["size"] = size_total + conn.close() + debug(u"Response: %s" % response) + except Exception, e: + if self.config.progress_meter: + progress.done("failed") + if retries: + if retries < self.config.max_retries: + throttle = throttle and throttle * 5 or 0.01 + warning("Upload failed: %s (%s)" % (resource['uri'], e)) + warning("Retrying on lower speed (throttle=%0.2f)" % throttle) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + # Connection error -> same throttle value + return self.send_file(request, file, labels, throttle, retries - 1) + else: + debug("Giving up on '%s' %s" % (file.name, e)) + raise S3UploadError("Upload failed for: %s" % resource['uri']) + + timestamp_end = time.time() + response["elapsed"] = timestamp_end - timestamp_start + response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) + + if self.config.progress_meter: + ## The above conn.close() takes some time -> update() progress meter + ## to correct the average speed. Otherwise people will complain that + ## 'progress' and response["speed"] are inconsistent ;-) + progress.update() + progress.done("done") + + if response["status"] == 307: + ## RedirectPermanent + redir_bucket = getTextFromXml(response['data'], ".//Bucket") + redir_hostname = getTextFromXml(response['data'], ".//Endpoint") + self.set_hostname(redir_bucket, redir_hostname) + warning("Redirected to: %s" % (redir_hostname)) + return self.send_file(request, file, labels) + + # S3 from time to time doesn't send ETag back in a response :-( + # Force re-upload here. + if not response['headers'].has_key('etag'): + response['headers']['etag'] = '' + + if response["status"] < 200 or response["status"] > 299: + try_retry = False + if response["status"] >= 500: + ## AWS internal error - retry + try_retry = True + elif response["status"] >= 400: + err = S3Error(response) + ## Retriable client error? + if err.code in ['BadDigest', 'OperationAborted', 'TokenRefreshRequired', 'RequestTimeout']: + try_retry = True + + if try_retry: + if retries: + warning("Upload failed: %s (%s)" % (resource['uri'], S3Error(response))) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + return self.send_file(request, file, labels, throttle, retries - 1) + else: + warning("Too many failures. Giving up on '%s'" % (file.name)) + raise S3UploadError + + ## Non-recoverable error + raise S3Error(response) + + debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"])) + if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest(): + warning("MD5 Sums don't match!") + if retries: + warning("Retrying upload of %s" % (file.name)) + return self.send_file(request, file, labels, throttle, retries - 1) + else: + warning("Too many failures. Giving up on '%s'" % (file.name)) + raise S3UploadError + + return response + + def recv_file(self, request, stream, labels, start_position=0, retries=-1): + if retries == -1: + retries = self.config.max_retries + method_string, resource, headers = request.get_triplet() + if self.config.progress_meter: + progress = self.config.progress_class(labels, 0) + else: + info("Receiving file '%s', please wait..." % stream.name) + timestamp_start = time.time() + try: + conn = self.get_connection(resource['bucket']) + conn.connect() + conn.putrequest(method_string, self.format_uri(resource)) + for header in headers.keys(): + conn.putheader(header, str(headers[header])) + if start_position > 0: + debug("Requesting Range: %d .. end" % start_position) + conn.putheader("Range", "bytes=%d-" % start_position) + conn.endheaders() + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + debug("Response: %s" % response) + except Exception, e: + if self.config.progress_meter: + progress.done("failed") + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + # Connection error -> same throttle value + return self.recv_file(request, stream, labels, start_position, retries - 1) + else: + raise S3DownloadError("Download failed for: %s" % resource['uri']) + + if response["status"] == 307: + ## RedirectPermanent + response['data'] = http_response.read() + redir_bucket = getTextFromXml(response['data'], ".//Bucket") + redir_hostname = getTextFromXml(response['data'], ".//Endpoint") + self.set_hostname(redir_bucket, redir_hostname) + warning("Redirected to: %s" % (redir_hostname)) + return self.recv_file(request, stream, labels) + + if response["status"] < 200 or response["status"] > 299: + raise S3Error(response) + + if start_position == 0: + # Only compute MD5 on the fly if we're downloading from beginning + # Otherwise we'd get a nonsense. + md5_hash = md5() + size_left = int(response["headers"]["content-length"]) + size_total = start_position + size_left + current_position = start_position + + if self.config.progress_meter: + progress.total_size = size_total + progress.initial_position = current_position + progress.current_position = current_position + + try: + while (current_position < size_total): + this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left + data = http_response.read(this_chunk) + stream.write(data) + if start_position == 0: + md5_hash.update(data) + current_position += len(data) + ## Call progress meter from here... + if self.config.progress_meter: + progress.update(delta_position=len(data)) + conn.close() + except Exception, e: + if self.config.progress_meter: + progress.done("failed") + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + + if self._fail_wait(retries) > 0: + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + + # Connection error -> same throttle value + return self.recv_file(request, stream, labels, current_position, retries - 1) + else: + raise S3DownloadError("Download failed for: %s" % resource['uri']) + + stream.flush() + timestamp_end = time.time() + + if self.config.progress_meter: + ## The above stream.flush() may take some time -> update() progress meter + ## to correct the average speed. Otherwise people will complain that + ## 'progress' and response["speed"] are inconsistent ;-) + progress.update() + progress.done("done") + + if start_position == 0: + # Only compute MD5 on the fly if we were downloading from the beginning + response["md5"] = md5_hash.hexdigest() + else: + # Otherwise try to compute MD5 of the output file + try: + response["md5"] = hash_file_md5(stream.name) + except IOError, e: + if e.errno != errno.ENOENT: + warning("Unable to open file: %s: %s" % (stream.name, e)) + warning("Unable to verify MD5. Assume it matches.") + response["md5"] = response["headers"]["etag"] + + response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0 + response["elapsed"] = timestamp_end - timestamp_start + response["size"] = current_position + response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) + if response["size"] != start_position + long(response["headers"]["content-length"]): + warning("Reported size (%s) does not match received size (%s)" % ( + start_position + response["headers"]["content-length"], response["size"])) + debug("ReceiveFile: Computed MD5 = %s" % response["md5"]) + if not response["md5match"]: + warning("MD5 signatures do not match: computed=%s, received=%s" % ( + response["md5"], response["headers"]["etag"])) + return response + __all__.append("S3") diff --git a/S3/S3Uri.py b/S3/S3Uri.py index fc5f25b..a1a74a5 100644 --- a/S3/S3Uri.py +++ b/S3/S3Uri.py @@ -12,200 +12,208 @@ from Utils import unicodise, check_bucket_name_dns_conformity class S3Uri(object): - type = None - _subclasses = None - - def __new__(self, string): - if not self._subclasses: - ## Generate a list of all subclasses of S3Uri - self._subclasses = [] - dict = sys.modules[__name__].__dict__ - for something in dict: - if type(dict[something]) is not type(self): - continue - if issubclass(dict[something], self) and dict[something] != self: - self._subclasses.append(dict[something]) - for subclass in self._subclasses: - try: - instance = object.__new__(subclass) - instance.__init__(string) - return instance - except ValueError, e: - continue - raise ValueError("%s: not a recognized URI" % string) - - def __str__(self): - return self.uri() - - def __unicode__(self): - return self.uri() - - def public_url(self): - raise ValueError("This S3 URI does not have Anonymous URL representation") - - def basename(self): - return self.__unicode__().split("/")[-1] + type = None + _subclasses = None + + def __new__(self, string): + if not self._subclasses: + ## Generate a list of all subclasses of S3Uri + self._subclasses = [] + dict = sys.modules[__name__].__dict__ + for something in dict: + if type(dict[something]) is not type(self): + continue + if issubclass(dict[something], self) and dict[something] != self: + self._subclasses.append(dict[something]) + for subclass in self._subclasses: + try: + instance = object.__new__(subclass) + instance.__init__(string) + return instance + except ValueError, e: + continue + raise ValueError("%s: not a recognized URI" % string) + + def __str__(self): + return self.uri() + + def __unicode__(self): + return self.uri() + + def public_url(self): + raise ValueError("This S3 URI does not have Anonymous URL representation") + + def basename(self): + return self.__unicode__().split("/")[-1] + class S3UriS3(S3Uri): - type = "s3" - _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE) - def __init__(self, string): - match = self._re.match(string) - if not match: - raise ValueError("%s: not a S3 URI" % string) - groups = match.groups() - self._bucket = groups[0] - self._object = unicodise(groups[1]) - - def bucket(self): - return self._bucket - - def object(self): - return self._object - - def has_bucket(self): - return bool(self._bucket) - - def has_object(self): - return bool(self._object) - - def uri(self): - return "/".join(["s3:/", self._bucket, self._object]) - - def is_dns_compatible(self): - return check_bucket_name_dns_conformity(self._bucket) - - def public_url(self): - if self.is_dns_compatible(): - return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object) - else: - return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object) - - def host_name(self): - if self.is_dns_compatible(): - return "%s.s3.amazonaws.com" % (self._bucket) - else: - return "s3.amazonaws.com" - - @staticmethod - def compose_uri(bucket, object = ""): - return "s3://%s/%s" % (bucket, object) - - @staticmethod - def httpurl_to_s3uri(http_url): - m=re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE) - hostname, object = m.groups()[1:] - hostname = hostname.lower() - if hostname == "s3.amazonaws.com": - ## old-style url: http://s3.amazonaws.com/bucket/object - if object.count("/") == 0: - ## no object given - bucket = object - object = "" - else: - ## bucket/object - bucket, object = object.split("/", 1) - elif hostname.endswith(".s3.amazonaws.com"): - ## new-style url: http://bucket.s3.amazonaws.com/object - bucket = hostname[:-(len(".s3.amazonaws.com"))] - else: - raise ValueError("Unable to parse URL: %s" % http_url) - return S3Uri("s3://%(bucket)s/%(object)s" % { - 'bucket' : bucket, - 'object' : object }) + type = "s3" + _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE) + + def __init__(self, string): + match = self._re.match(string) + if not match: + raise ValueError("%s: not a S3 URI" % string) + groups = match.groups() + self._bucket = groups[0] + self._object = unicodise(groups[1]) + + def bucket(self): + return self._bucket + + def object(self): + return self._object + + def has_bucket(self): + return bool(self._bucket) + + def has_object(self): + return bool(self._object) + + def uri(self): + return "/".join(["s3:/", self._bucket, self._object]) + + def is_dns_compatible(self): + return check_bucket_name_dns_conformity(self._bucket) + + def public_url(self): + if self.is_dns_compatible(): + return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object) + else: + return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object) + + def host_name(self): + if self.is_dns_compatible(): + return "%s.s3.amazonaws.com" % (self._bucket) + else: + return "s3.amazonaws.com" + + @staticmethod + def compose_uri(bucket, object=""): + return "s3://%s/%s" % (bucket, object) + + @staticmethod + def httpurl_to_s3uri(http_url): + m = re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE) + hostname, object = m.groups()[1:] + hostname = hostname.lower() + if hostname == "s3.amazonaws.com": + ## old-style url: http://s3.amazonaws.com/bucket/object + if object.count("/") == 0: + ## no object given + bucket = object + object = "" + else: + ## bucket/object + bucket, object = object.split("/", 1) + elif hostname.endswith(".s3.amazonaws.com"): + ## new-style url: http://bucket.s3.amazonaws.com/object + bucket = hostname[:-(len(".s3.amazonaws.com"))] + else: + raise ValueError("Unable to parse URL: %s" % http_url) + return S3Uri("s3://%(bucket)s/%(object)s" % { + 'bucket': bucket, + 'object': object}) + class S3UriS3FS(S3Uri): - type = "s3fs" - _re = re.compile("^s3fs://([^/]*)/?(.*)", re.IGNORECASE) - def __init__(self, string): - match = self._re.match(string) - if not match: - raise ValueError("%s: not a S3fs URI" % string) - groups = match.groups() - self._fsname = groups[0] - self._path = unicodise(groups[1]).split("/") + type = "s3fs" + _re = re.compile("^s3fs://([^/]*)/?(.*)", re.IGNORECASE) + + def __init__(self, string): + match = self._re.match(string) + if not match: + raise ValueError("%s: not a S3fs URI" % string) + groups = match.groups() + self._fsname = groups[0] + self._path = unicodise(groups[1]).split("/") + + def fsname(self): + return self._fsname - def fsname(self): - return self._fsname + def path(self): + return "/".join(self._path) - def path(self): - return "/".join(self._path) + def uri(self): + return "/".join(["s3fs:/", self._fsname, self.path()]) - def uri(self): - return "/".join(["s3fs:/", self._fsname, self.path()]) class S3UriFile(S3Uri): - type = "file" - _re = re.compile("^(\w+://)?(.*)") - def __init__(self, string): - match = self._re.match(string) - groups = match.groups() - if groups[0] not in (None, "file://"): - raise ValueError("%s: not a file:// URI" % string) - self._path = unicodise(groups[1]).split("/") + type = "file" + _re = re.compile("^(\w+://)?(.*)") - def path(self): - return "/".join(self._path) + def __init__(self, string): + match = self._re.match(string) + groups = match.groups() + if groups[0] not in (None, "file://"): + raise ValueError("%s: not a file:// URI" % string) + self._path = unicodise(groups[1]).split("/") - def uri(self): - return "/".join(["file:/", self.path()]) + def path(self): + return "/".join(self._path) - def isdir(self): - return os.path.isdir(self.path()) + def uri(self): + return "/".join(["file:/", self.path()]) + + def isdir(self): + return os.path.isdir(self.path()) + + def dirname(self): + return os.path.dirname(self.path()) - def dirname(self): - return os.path.dirname(self.path()) class S3UriCloudFront(S3Uri): - type = "cf" - _re = re.compile("^cf://([^/]*)/?", re.IGNORECASE) - def __init__(self, string): - match = self._re.match(string) - if not match: - raise ValueError("%s: not a CloudFront URI" % string) - groups = match.groups() - self._dist_id = groups[0] + type = "cf" + _re = re.compile("^cf://([^/]*)/?", re.IGNORECASE) + + def __init__(self, string): + match = self._re.match(string) + if not match: + raise ValueError("%s: not a CloudFront URI" % string) + groups = match.groups() + self._dist_id = groups[0] - def dist_id(self): - return self._dist_id + def dist_id(self): + return self._dist_id - def uri(self): - return "/".join(["cf:/", self.dist_id()]) + def uri(self): + return "/".join(["cf:/", self.dist_id()]) if __name__ == "__main__": - uri = S3Uri("s3://bucket/object") - print "type() =", type(uri) - print "uri =", uri - print "uri.type=", uri.type - print "bucket =", uri.bucket() - print "object =", uri.object() - print - - uri = S3Uri("s3://bucket") - print "type() =", type(uri) - print "uri =", uri - print "uri.type=", uri.type - print "bucket =", uri.bucket() - print - - uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt") - print "type() =", type(uri) - print "uri =", uri - print "uri.type=", uri.type - print "path =", uri.path() - print - - uri = S3Uri("/path/to/local/file.txt") - print "type() =", type(uri) - print "uri =", uri - print "uri.type=", uri.type - print "path =", uri.path() - print - - uri = S3Uri("cf://1234567890ABCD/") - print "type() =", type(uri) - print "uri =", uri - print "uri.type=", uri.type - print "dist_id =", uri.dist_id() - print + uri = S3Uri("s3://bucket/object") + print "type() =", type(uri) + print "uri =", uri + print "uri.type=", uri.type + print "bucket =", uri.bucket() + print "object =", uri.object() + print + + uri = S3Uri("s3://bucket") + print "type() =", type(uri) + print "uri =", uri + print "uri.type=", uri.type + print "bucket =", uri.bucket() + print + + uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt") + print "type() =", type(uri) + print "uri =", uri + print "uri.type=", uri.type + print "path =", uri.path() + print + + uri = S3Uri("/path/to/local/file.txt") + print "type() =", type(uri) + print "uri =", uri + print "uri.type=", uri.type + print "path =", uri.path() + print + + uri = S3Uri("cf://1234567890ABCD/") + print "type() =", type(uri) + print "uri =", uri + print "uri.type=", uri.type + print "dist_id =", uri.dist_id() + print diff --git a/S3/SimpleDB.py b/S3/SimpleDB.py index 18e167b..1346356 100644 --- a/S3/SimpleDB.py +++ b/S3/SimpleDB.py @@ -20,154 +20,155 @@ from Exceptions import * class SimpleDB(object): - # API Version - # See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/ - Version = "2007-11-07" - SignatureVersion = 1 - - def __init__(self, config): - self.config = config - - ## ------------------------------------------------ - ## Methods implementing SimpleDB API - ## ------------------------------------------------ - - def ListDomains(self, MaxNumberOfDomains = 100): - ''' - Lists all domains associated with our Access Key. Returns - domain names up to the limit set by MaxNumberOfDomains. - ''' - parameters = SortedDict() - parameters['MaxNumberOfDomains'] = MaxNumberOfDomains - return self.send_request("ListDomains", DomainName = None, parameters = parameters) - - def CreateDomain(self, DomainName): - return self.send_request("CreateDomain", DomainName = DomainName) - - def DeleteDomain(self, DomainName): - return self.send_request("DeleteDomain", DomainName = DomainName) - - def PutAttributes(self, DomainName, ItemName, Attributes): - parameters = SortedDict() - parameters['ItemName'] = ItemName - seq = 0 - for attrib in Attributes: - if type(Attributes[attrib]) == type(list()): - for value in Attributes[attrib]: - parameters['Attribute.%d.Name' % seq] = attrib - parameters['Attribute.%d.Value' % seq] = unicode(value) - seq += 1 - else: - parameters['Attribute.%d.Name' % seq] = attrib - parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib]) - seq += 1 - ## TODO: - ## - support for Attribute.N.Replace - ## - support for multiple values for one attribute - return self.send_request("PutAttributes", DomainName = DomainName, parameters = parameters) - - def GetAttributes(self, DomainName, ItemName, Attributes = []): - parameters = SortedDict() - parameters['ItemName'] = ItemName - seq = 0 - for attrib in Attributes: - parameters['AttributeName.%d' % seq] = attrib - seq += 1 - return self.send_request("GetAttributes", DomainName = DomainName, parameters = parameters) - - def DeleteAttributes(self, DomainName, ItemName, Attributes = {}): - """ - Remove specified Attributes from ItemName. - Attributes parameter can be either: - - not specified, in which case the whole Item is removed - - list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed - - dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the - specified values are removed from multi-value attributes. - """ - parameters = SortedDict() - parameters['ItemName'] = ItemName - seq = 0 - for attrib in Attributes: - parameters['Attribute.%d.Name' % seq] = attrib - if type(Attributes) == type(dict()): - parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib]) - seq += 1 - return self.send_request("DeleteAttributes", DomainName = DomainName, parameters = parameters) - - def Query(self, DomainName, QueryExpression = None, MaxNumberOfItems = None, NextToken = None): - parameters = SortedDict() - if QueryExpression: - parameters['QueryExpression'] = QueryExpression - if MaxNumberOfItems: - parameters['MaxNumberOfItems'] = MaxNumberOfItems - if NextToken: - parameters['NextToken'] = NextToken - return self.send_request("Query", DomainName = DomainName, parameters = parameters) - ## Handle NextToken? Or maybe not - let the upper level do it - - ## ------------------------------------------------ - ## Low-level methods for handling SimpleDB requests - ## ------------------------------------------------ - - def send_request(self, *args, **kwargs): - request = self.create_request(*args, **kwargs) - #debug("Request: %s" % repr(request)) - conn = self.get_connection() - conn.request("GET", self.format_uri(request['uri_params'])) - http_response = conn.getresponse() - response = {} - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - response["data"] = http_response.read() - conn.close() - - if response["status"] < 200 or response["status"] > 299: - debug("Response: " + str(response)) - raise S3Error(response) - - return response - - def create_request(self, Action, DomainName, parameters = None): - if not parameters: - parameters = SortedDict() - parameters['AWSAccessKeyId'] = self.config.access_key - parameters['Version'] = self.Version - parameters['SignatureVersion'] = self.SignatureVersion - parameters['Action'] = Action - parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - if DomainName: - parameters['DomainName'] = DomainName - parameters['Signature'] = self.sign_request(parameters) - parameters.keys_return_lowercase = False - uri_params = urllib.urlencode(parameters) - request = {} - request['uri_params'] = uri_params - request['parameters'] = parameters - return request - - def sign_request(self, parameters): - h = "" - parameters.keys_sort_lowercase = True - parameters.keys_return_lowercase = False - for key in parameters: - h += "%s%s" % (key, parameters[key]) - #debug("SignRequest: %s" % h) - return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip() - - def get_connection(self): - if self.config.proxy_host != "": - return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port) - else: - if self.config.use_https: - return httplib.HTTPSConnection(self.config.simpledb_host) - else: - return httplib.HTTPConnection(self.config.simpledb_host) - - def format_uri(self, uri_params): - if self.config.proxy_host != "": - uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params) - else: - uri = "/?%s" % uri_params - #debug('format_uri(): ' + uri) - return uri + # API Version + # See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/ + Version = "2007-11-07" + SignatureVersion = 1 + + def __init__(self, config): + self.config = config + + ## ------------------------------------------------ + ## Methods implementing SimpleDB API + ## ------------------------------------------------ + + def ListDomains(self, MaxNumberOfDomains=100): + ''' + Lists all domains associated with our Access Key. Returns + domain names up to the limit set by MaxNumberOfDomains. + ''' + parameters = SortedDict() + parameters['MaxNumberOfDomains'] = MaxNumberOfDomains + return self.send_request("ListDomains", DomainName=None, parameters=parameters) + + def CreateDomain(self, DomainName): + return self.send_request("CreateDomain", DomainName=DomainName) + + def DeleteDomain(self, DomainName): + return self.send_request("DeleteDomain", DomainName=DomainName) + + def PutAttributes(self, DomainName, ItemName, Attributes): + parameters = SortedDict() + parameters['ItemName'] = ItemName + seq = 0 + for attrib in Attributes: + if type(Attributes[attrib]) == type(list()): + for value in Attributes[attrib]: + parameters['Attribute.%d.Name' % seq] = attrib + parameters['Attribute.%d.Value' % seq] = unicode(value) + seq += 1 + else: + parameters['Attribute.%d.Name' % seq] = attrib + parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib]) + seq += 1 + ## TODO: + ## - support for Attribute.N.Replace + ## - support for multiple values for one attribute + return self.send_request("PutAttributes", DomainName=DomainName, parameters=parameters) + + def GetAttributes(self, DomainName, ItemName, Attributes=[]): + parameters = SortedDict() + parameters['ItemName'] = ItemName + seq = 0 + for attrib in Attributes: + parameters['AttributeName.%d' % seq] = attrib + seq += 1 + return self.send_request("GetAttributes", DomainName=DomainName, parameters=parameters) + + def DeleteAttributes(self, DomainName, ItemName, Attributes={}): + """ + Remove specified Attributes from ItemName. + Attributes parameter can be either: + - not specified, in which case the whole Item is removed + - list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed + - dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the + specified values are removed from multi-value attributes. + """ + parameters = SortedDict() + parameters['ItemName'] = ItemName + seq = 0 + for attrib in Attributes: + parameters['Attribute.%d.Name' % seq] = attrib + if type(Attributes) == type(dict()): + parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib]) + seq += 1 + return self.send_request("DeleteAttributes", DomainName=DomainName, parameters=parameters) + + def Query(self, DomainName, QueryExpression=None, MaxNumberOfItems=None, NextToken=None): + parameters = SortedDict() + if QueryExpression: + parameters['QueryExpression'] = QueryExpression + if MaxNumberOfItems: + parameters['MaxNumberOfItems'] = MaxNumberOfItems + if NextToken: + parameters['NextToken'] = NextToken + return self.send_request("Query", DomainName=DomainName, parameters=parameters) + + ## Handle NextToken? Or maybe not - let the upper level do it + + ## ------------------------------------------------ + ## Low-level methods for handling SimpleDB requests + ## ------------------------------------------------ + + def send_request(self, *args, **kwargs): + request = self.create_request(*args, **kwargs) + #debug("Request: %s" % repr(request)) + conn = self.get_connection() + conn.request("GET", self.format_uri(request['uri_params'])) + http_response = conn.getresponse() + response = {} + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + response["data"] = http_response.read() + conn.close() + + if response["status"] < 200 or response["status"] > 299: + debug("Response: " + str(response)) + raise S3Error(response) + + return response + + def create_request(self, Action, DomainName, parameters=None): + if not parameters: + parameters = SortedDict() + parameters['AWSAccessKeyId'] = self.config.access_key + parameters['Version'] = self.Version + parameters['SignatureVersion'] = self.SignatureVersion + parameters['Action'] = Action + parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + if DomainName: + parameters['DomainName'] = DomainName + parameters['Signature'] = self.sign_request(parameters) + parameters.keys_return_lowercase = False + uri_params = urllib.urlencode(parameters) + request = {} + request['uri_params'] = uri_params + request['parameters'] = parameters + return request + + def sign_request(self, parameters): + h = "" + parameters.keys_sort_lowercase = True + parameters.keys_return_lowercase = False + for key in parameters: + h += "%s%s" % (key, parameters[key]) + #debug("SignRequest: %s" % h) + return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip() + + def get_connection(self): + if self.config.proxy_host != "": + return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port) + else: + if self.config.use_https: + return httplib.HTTPSConnection(self.config.simpledb_host) + else: + return httplib.HTTPConnection(self.config.simpledb_host) + + def format_uri(self, uri_params): + if self.config.proxy_host != "": + uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params) + else: + uri = "/?%s" % uri_params + #debug('format_uri(): ' + uri) + return uri diff --git a/S3/SortedDict.py b/S3/SortedDict.py index 778457d..f5ad8e7 100644 --- a/S3/SortedDict.py +++ b/S3/SortedDict.py @@ -6,56 +6,57 @@ from BidirMap import BidirMap class SortedDictIterator(object): - def __init__(self, sorted_dict, keys): - self.sorted_dict = sorted_dict - self.keys = keys + def __init__(self, sorted_dict, keys): + self.sorted_dict = sorted_dict + self.keys = keys + + def next(self): + try: + return self.keys.pop(0) + except IndexError: + raise StopIteration - def next(self): - try: - return self.keys.pop(0) - except IndexError: - raise StopIteration class SortedDict(dict): - def __init__(self, mapping = {}, ignore_case = True, **kwargs): - """ - WARNING: SortedDict() with ignore_case==True will - drop entries differing only in capitalisation! - Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland'] - With ignore_case==False it's all right - """ - dict.__init__(self, mapping, **kwargs) - self.ignore_case = ignore_case - - def keys(self): - keys = dict.keys(self) - if self.ignore_case: - # Translation map - xlat_map = BidirMap() - for key in keys: - xlat_map[key.lower()] = key - # Lowercase keys - lc_keys = xlat_map.keys() - lc_keys.sort() - return [xlat_map[k] for k in lc_keys] - else: - keys.sort() - return keys - - def __iter__(self): - return SortedDictIterator(self, self.keys()) + def __init__(self, mapping={}, ignore_case=True, **kwargs): + """ + WARNING: SortedDict() with ignore_case==True will + drop entries differing only in capitalisation! + Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland'] + With ignore_case==False it's all right + """ + dict.__init__(self, mapping, **kwargs) + self.ignore_case = ignore_case + + def keys(self): + keys = dict.keys(self) + if self.ignore_case: + # Translation map + xlat_map = BidirMap() + for key in keys: + xlat_map[key.lower()] = key + # Lowercase keys + lc_keys = xlat_map.keys() + lc_keys.sort() + return [xlat_map[k] for k in lc_keys] + else: + keys.sort() + return keys + + def __iter__(self): + return SortedDictIterator(self, self.keys()) if __name__ == "__main__": - d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 } - sd = SortedDict(d) - print "Wanted: Action, america, Auckland, AWS, [ignore case]" - print "Got: ", - for key in sd: - print "%s," % key, - print " [used: __iter__()]" - d = SortedDict(d, ignore_case = False) - print "Wanted: AWS, Action, Auckland, america, [case sensitive]" - print "Got: ", - for key in d.keys(): - print "%s," % key, - print " [used: keys()]" + d = {'AWS': 1, 'Action': 2, 'america': 3, 'Auckland': 4, 'America': 5} + sd = SortedDict(d) + print "Wanted: Action, america, Auckland, AWS, [ignore case]" + print "Got: ", + for key in sd: + print "%s," % key, + print " [used: __iter__()]" + d = SortedDict(d, ignore_case=False) + print "Wanted: AWS, Action, Auckland, america, [case sensitive]" + print "Got: ", + for key in d.keys(): + print "%s," % key, + print " [used: keys()]" diff --git a/S3/Utils.py b/S3/Utils.py index badd28e..85b605d 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -9,11 +9,12 @@ import string import random import rfc822 + try: - from hashlib import md5, sha1 + from hashlib import md5, sha1 except ImportError: - from md5 import md5 - import sha as sha1 + from md5 import md5 + import sha as sha1 import hmac import base64 import errno @@ -24,355 +25,395 @@ import Exceptions try: - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as ET except ImportError: - import elementtree.ElementTree as ET + import elementtree.ElementTree as ET from xml.parsers.expat import ExpatError __all__ = [] + def parseNodes(nodes): - ## WARNING: Ignores text nodes from mixed xml/text. - ## For instance some textother text - ## will be ignore "some text" node - retval = [] - for node in nodes: - retval_item = {} - for child in node.getchildren(): - name = child.tag - if child.getchildren(): - retval_item[name] = parseNodes([child]) - else: - retval_item[name] = node.findtext(".//%s" % child.tag) - retval.append(retval_item) - return retval + ## WARNING: Ignores text nodes from mixed xml/text. + ## For instance some textother text + ## will be ignore "some text" node + retval = [] + for node in nodes: + retval_item = {} + for child in node.getchildren(): + name = child.tag + if child.getchildren(): + retval_item[name] = parseNodes([child]) + else: + retval_item[name] = node.findtext(".//%s" % child.tag) + retval.append(retval_item) + return retval + __all__.append("parseNodes") def stripNameSpace(xml): - """ - removeNameSpace(xml) -- remove top-level AWS namespace - """ - r = re.compile('^(]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE) - if r.match(xml): - xmlns = r.match(xml).groups()[2] - xml = r.sub("\\1\\2\\4", xml) - else: - xmlns = None - return xml, xmlns + """ + removeNameSpace(xml) -- remove top-level AWS namespace + """ + r = re.compile('^(]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE) + if r.match(xml): + xmlns = r.match(xml).groups()[2] + xml = r.sub("\\1\\2\\4", xml) + else: + xmlns = None + return xml, xmlns + __all__.append("stripNameSpace") def getTreeFromXml(xml): - xml, xmlns = stripNameSpace(xml) - try: - tree = ET.fromstring(xml) - if xmlns: - tree.attrib['xmlns'] = xmlns - return tree - except ExpatError, e: - error(e) - raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/") + xml, xmlns = stripNameSpace(xml) + try: + tree = ET.fromstring(xml) + if xmlns: + tree.attrib['xmlns'] = xmlns + return tree + except ExpatError, e: + error(e) + raise Exceptions.ParameterError( + "Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/") + __all__.append("getTreeFromXml") - + def getListFromXml(xml, node): - tree = getTreeFromXml(xml) - nodes = tree.findall('.//%s' % (node)) - return parseNodes(nodes) + tree = getTreeFromXml(xml) + nodes = tree.findall('.//%s' % (node)) + return parseNodes(nodes) + __all__.append("getListFromXml") def getDictFromTree(tree): - ret_dict = {} - for child in tree.getchildren(): - if child.getchildren(): - ## Complex-type child. We're not interested - continue - if ret_dict.has_key(child.tag): - if not type(ret_dict[child.tag]) == list: - ret_dict[child.tag] = [ret_dict[child.tag]] - ret_dict[child.tag].append(child.text or "") - else: - ret_dict[child.tag] = child.text or "" - return ret_dict + ret_dict = {} + for child in tree.getchildren(): + if child.getchildren(): + ## Complex-type child. We're not interested + continue + if ret_dict.has_key(child.tag): + if not type(ret_dict[child.tag]) == list: + ret_dict[child.tag] = [ret_dict[child.tag]] + ret_dict[child.tag].append(child.text or "") + else: + ret_dict[child.tag] = child.text or "" + return ret_dict + __all__.append("getDictFromTree") def getTextFromXml(xml, xpath): - tree = getTreeFromXml(xml) - if tree.tag.endswith(xpath): - return tree.text - else: - return tree.findtext(xpath) + tree = getTreeFromXml(xml) + if tree.tag.endswith(xpath): + return tree.text + else: + return tree.findtext(xpath) + __all__.append("getTextFromXml") def getRootTagName(xml): - tree = getTreeFromXml(xml) - return tree.tag + tree = getTreeFromXml(xml) + return tree.tag + __all__.append("getRootTagName") def xmlTextNode(tag_name, text): - el = ET.Element(tag_name) - el.text = unicode(text) - return el + el = ET.Element(tag_name) + el.text = unicode(text) + return el + __all__.append("xmlTextNode") def appendXmlTextNode(tag_name, text, parent): - """ - Creates a new Node and sets - its content to 'text'. Then appends the - created Node to 'parent' element if given. - Returns the newly created Node. - """ - el = xmlTextNode(tag_name, text) - parent.append(el) - return el + """ + Creates a new Node and sets + its content to 'text'. Then appends the + created Node to 'parent' element if given. + Returns the newly created Node. + """ + el = xmlTextNode(tag_name, text) + parent.append(el) + return el + __all__.append("appendXmlTextNode") def dateS3toPython(date): - date = re.compile("(\.\d*)?Z").sub(".000Z", date) - return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") + date = re.compile("(\.\d*)?Z").sub(".000Z", date) + return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") + __all__.append("dateS3toPython") def dateS3toUnix(date): - ## FIXME: This should be timezone-aware. - ## Currently the argument to strptime() is GMT but mktime() - ## treats it as "localtime". Anyway... - return time.mktime(dateS3toPython(date)) + ## FIXME: This should be timezone-aware. + ## Currently the argument to strptime() is GMT but mktime() + ## treats it as "localtime". Anyway... + return time.mktime(dateS3toPython(date)) + __all__.append("dateS3toUnix") def dateRFC822toPython(date): - return rfc822.parsedate(date) + return rfc822.parsedate(date) + __all__.append("dateRFC822toPython") def dateRFC822toUnix(date): - return time.mktime(dateRFC822toPython(date)) + return time.mktime(dateRFC822toPython(date)) + __all__.append("dateRFC822toUnix") -def formatSize(size, human_readable = False, floating_point = False): - size = floating_point and float(size) or int(size) - if human_readable: - coeffs = ['k', 'M', 'G', 'T'] - coeff = "" - while size > 2048: - size /= 1024 - coeff = coeffs.pop(0) - return (size, coeff) - else: - return (size, "") +def formatSize(size, human_readable=False, floating_point=False): + size = floating_point and float(size) or int(size) + if human_readable: + coeffs = ['k', 'M', 'G', 'T'] + coeff = "" + while size > 2048: + size /= 1024 + coeff = coeffs.pop(0) + return (size, coeff) + else: + return (size, "") + __all__.append("formatSize") def formatDateTime(s3timestamp): - return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp)) + return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp)) + __all__.append("formatDateTime") def convertTupleListToDict(list): - retval = {} - for tuple in list: - retval[tuple[0]] = tuple[1] - return retval + retval = {} + for tuple in list: + retval[tuple[0]] = tuple[1] + return retval + __all__.append("convertTupleListToDict") -_rnd_chars = string.ascii_letters+string.digits +_rnd_chars = string.ascii_letters + string.digits _rnd_chars_len = len(_rnd_chars) + def rndstr(len): - retval = "" - while len > 0: - retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)] - len -= 1 - return retval + retval = "" + while len > 0: + retval += _rnd_chars[random.randint(0, _rnd_chars_len - 1)] + len -= 1 + return retval + __all__.append("rndstr") def mktmpsomething(prefix, randchars, createfunc): - old_umask = os.umask(0077) - tries = 5 - while tries > 0: - dirname = prefix + rndstr(randchars) - try: - createfunc(dirname) - break - except OSError, e: - if e.errno != errno.EEXIST: - os.umask(old_umask) - raise - tries -= 1 - - os.umask(old_umask) - return dirname + old_umask = os.umask(0077) + tries = 5 + while tries > 0: + dirname = prefix + rndstr(randchars) + try: + createfunc(dirname) + break + except OSError, e: + if e.errno != errno.EEXIST: + os.umask(old_umask) + raise + tries -= 1 + + os.umask(old_umask) + return dirname + __all__.append("mktmpsomething") -def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10): - return mktmpsomething(prefix, randchars, os.mkdir) +def mktmpdir(prefix="/tmp/tmpdir-", randchars=10): + return mktmpsomething(prefix, randchars, os.mkdir) + __all__.append("mktmpdir") -def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20): - createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT | os.O_EXCL)) - return mktmpsomething(prefix, randchars, createfunc) +def mktmpfile(prefix="/tmp/tmpfile-", randchars=20): + createfunc = lambda filename: os.close(os.open(filename, os.O_CREAT | os.O_EXCL)) + return mktmpsomething(prefix, randchars, createfunc) + __all__.append("mktmpfile") def hash_file_md5(filename): - h = md5() - f = open(filename, "rb") - while True: - # Hash 32kB chunks - data = f.read(32*1024) - if not data: - break - h.update(data) - f.close() - return h.hexdigest() + h = md5() + f = open(filename, "rb") + while True: + # Hash 32kB chunks + data = f.read(32 * 1024) + if not data: + break + h.update(data) + f.close() + return h.hexdigest() + __all__.append("hash_file_md5") def mkdir_with_parents(dir_name): - """ - mkdir_with_parents(dst_dir) - - Create directory 'dir_name' with all parent directories - - Returns True on success, False otherwise. - """ - pathmembers = dir_name.split(os.sep) - tmp_stack = [] - while pathmembers and not os.path.isdir(os.sep.join(pathmembers)): - tmp_stack.append(pathmembers.pop()) - while tmp_stack: - pathmembers.append(tmp_stack.pop()) - cur_dir = os.sep.join(pathmembers) - try: - debug("mkdir(%s)" % cur_dir) - os.mkdir(cur_dir) - except (OSError, IOError), e: - warning("%s: can not make directory: %s" % (cur_dir, e.strerror)) - return False - except Exception, e: - warning("%s: %s" % (cur_dir, e)) - return False - return True + """ + mkdir_with_parents(dst_dir) + + Create directory 'dir_name' with all parent directories + + Returns True on success, False otherwise. + """ + pathmembers = dir_name.split(os.sep) + tmp_stack = [] + while pathmembers and not os.path.isdir(os.sep.join(pathmembers)): + tmp_stack.append(pathmembers.pop()) + while tmp_stack: + pathmembers.append(tmp_stack.pop()) + cur_dir = os.sep.join(pathmembers) + try: + debug("mkdir(%s)" % cur_dir) + os.mkdir(cur_dir) + except (OSError, IOError), e: + warning("%s: can not make directory: %s" % (cur_dir, e.strerror)) + return False + except Exception, e: + warning("%s: %s" % (cur_dir, e)) + return False + return True + __all__.append("mkdir_with_parents") -def unicodise(string, encoding = None, errors = "replace"): - """ - Convert 'string' to Unicode or raise an exception. - """ - - if not encoding: - encoding = Config.Config().encoding - - if type(string) == unicode: - return string - debug("Unicodising %r using %s" % (string, encoding)) - try: - return string.decode(encoding, errors) - except UnicodeDecodeError: - raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) +def unicodise(string, encoding=None, errors="replace"): + """ + Convert 'string' to Unicode or raise an exception. + """ + + if not encoding: + encoding = Config.Config().encoding + + if type(string) == unicode: + return string + debug("Unicodising %r using %s" % (string, encoding)) + try: + return string.decode(encoding, errors) + except UnicodeDecodeError: + raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) + __all__.append("unicodise") -def deunicodise(string, encoding = None, errors = "replace"): - """ - Convert unicode 'string' to , by default replacing - all invalid characters with '?' or raise an exception. - """ - - if not encoding: - encoding = Config.Config().encoding - - if type(string) != unicode: - return str(string) - debug("DeUnicodising %r using %s" % (string, encoding)) - try: - return string.encode(encoding, errors) - except UnicodeEncodeError: - raise UnicodeEncodeError("Conversion from unicode failed: %r" % string) +def deunicodise(string, encoding=None, errors="replace"): + """ + Convert unicode 'string' to , by default replacing + all invalid characters with '?' or raise an exception. + """ + + if not encoding: + encoding = Config.Config().encoding + + if type(string) != unicode: + return str(string) + debug("DeUnicodising %r using %s" % (string, encoding)) + try: + return string.encode(encoding, errors) + except UnicodeEncodeError: + raise UnicodeEncodeError("Conversion from unicode failed: %r" % string) + __all__.append("deunicodise") -def unicodise_safe(string, encoding = None): - """ - Convert 'string' to Unicode according to current encoding - and replace all invalid characters with '?' - """ +def unicodise_safe(string, encoding=None): + """ + Convert 'string' to Unicode according to current encoding + and replace all invalid characters with '?' + """ + + return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?') - return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?') __all__.append("unicodise_safe") def replace_nonprintables(string): - """ - replace_nonprintables(string) - - Replaces all non-printable characters 'ch' in 'string' - where ord(ch) <= 26 with ^@, ^A, ... ^Z - """ - new_string = "" - modified = 0 - for c in string: - o = ord(c) - if (o <= 31): - new_string += "^" + chr(ord('@') + o) - modified += 1 - elif (o == 127): - new_string += "^?" - modified += 1 - else: - new_string += c - if modified and Config.Config().urlencoding_mode != "fixbucket": - warning("%d non-printable characters replaced in: %s" % (modified, new_string)) - return new_string + """ + replace_nonprintables(string) + + Replaces all non-printable characters 'ch' in 'string' + where ord(ch) <= 26 with ^@, ^A, ... ^Z + """ + new_string = "" + modified = 0 + for c in string: + o = ord(c) + if (o <= 31): + new_string += "^" + chr(ord('@') + o) + modified += 1 + elif (o == 127): + new_string += "^?" + modified += 1 + else: + new_string += c + if modified and Config.Config().urlencoding_mode != "fixbucket": + warning("%d non-printable characters replaced in: %s" % (modified, new_string)) + return new_string + __all__.append("replace_nonprintables") def sign_string(string_to_sign): - #debug("string_to_sign: %s" % string_to_sign) - signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip() - #debug("signature: %s" % signature) - return signature + #debug("string_to_sign: %s" % string_to_sign) + signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip() + #debug("signature: %s" % signature) + return signature + __all__.append("sign_string") -def check_bucket_name(bucket, dns_strict = True): - if dns_strict: - invalid = re.search("([^a-z0-9\.-])", bucket) - if invalid: - raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0])) - else: - invalid = re.search("([^A-Za-z0-9\._-])", bucket) - if invalid: - raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0])) - - if len(bucket) < 3: - raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket) - if len(bucket) > 255: - raise Exceptions.ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket) - if dns_strict: - if len(bucket) > 63: - raise Exceptions.ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket) - if re.search("-\.", bucket): - raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket) - if re.search("\.\.", bucket): - raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket) - if not re.search("^[0-9a-z]", bucket): - raise Exceptions.ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket) - if not re.search("[0-9a-z]$", bucket): - raise Exceptions.ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket) - return True +def check_bucket_name(bucket, dns_strict=True): + if dns_strict: + invalid = re.search("([^a-z0-9\.-])", bucket) + if invalid: + raise Exceptions.ParameterError( + "Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % ( + bucket, invalid.groups()[0])) + else: + invalid = re.search("([^A-Za-z0-9\._-])", bucket) + if invalid: + raise Exceptions.ParameterError( + "Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % ( + bucket, invalid.groups()[0])) + + if len(bucket) < 3: + raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket) + if len(bucket) > 255: + raise Exceptions.ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket) + if dns_strict: + if len(bucket) > 63: + raise Exceptions.ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket) + if re.search("-\.", bucket): + raise Exceptions.ParameterError( + "Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket) + if re.search("\.\.", bucket): + raise Exceptions.ParameterError( + "Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket) + if not re.search("^[0-9a-z]", bucket): + raise Exceptions.ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket) + if not re.search("[0-9a-z]$", bucket): + raise Exceptions.ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket) + return True + __all__.append("check_bucket_name") def check_bucket_name_dns_conformity(bucket): - try: - return check_bucket_name(bucket, dns_strict = True) - except Exceptions.ParameterError: - return False + try: + return check_bucket_name(bucket, dns_strict=True) + except Exceptions.ParameterError: + return False + __all__.append("check_bucket_name_dns_conformity") def getBucketFromHostname(hostname): - """ - bucket, success = getBucketFromHostname(hostname) + """ + bucket, success = getBucketFromHostname(hostname) - Only works for hostnames derived from bucket names - using Config.host_bucket pattern. + Only works for hostnames derived from bucket names + using Config.host_bucket pattern. - Returns bucket name and a boolean success flag. - """ + Returns bucket name and a boolean success flag. + """ + + # Create RE pattern from Config.host_bucket + pattern = Config.Config().host_bucket % {'bucket': '(?P.*)'} + m = re.match(pattern, hostname) + if not m: + return (hostname, False) + return m.groups()[0], True - # Create RE pattern from Config.host_bucket - pattern = Config.Config().host_bucket % { 'bucket' : '(?P.*)' } - m = re.match(pattern, hostname) - if not m: - return (hostname, False) - return m.groups()[0], True __all__.append("getBucketFromHostname") def getHostnameFromBucket(bucket): - return Config.Config().host_bucket % { 'bucket' : bucket } + return Config.Config().host_bucket % {'bucket': bucket} + __all__.append("getHostnameFromBucket") From 11b2ee99f7fe66d998e2cfc391e845a295eacf09 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Wed, 15 Feb 2012 19:03:42 -0800 Subject: [PATCH 03/16] First full pass at content-based upload/download --- S3/Config.py | 2 + S3/Utils.py | 38 ++- s3cmd | 445 +++++++++++++++++++++++++++++++--- testsuite/etc/brokenlink.png | 0 testsuite/etc/linked.png | 0 testsuite/etc/linked1.png | 0 testsuite/etc/more/linked-dir | 0 7 files changed, 443 insertions(+), 42 deletions(-) delete mode 120000 testsuite/etc/brokenlink.png delete mode 120000 testsuite/etc/linked.png delete mode 120000 testsuite/etc/linked1.png delete mode 120000 testsuite/etc/more/linked-dir diff --git a/S3/Config.py b/S3/Config.py index c25f05c..9f83ebe 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -33,6 +33,8 @@ class Config(object): get_continue = False skip_existing = False recursive = False + flatten = False + deflate = False acl_public = None acl_grants = [] acl_revokes = [] diff --git a/S3/Utils.py b/S3/Utils.py index 85b605d..e207951 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -243,6 +243,21 @@ def hash_file_md5(filename): __all__.append("hash_file_md5") +def hash_file_sha1(filename): + h = sha1() + f = open(filename, "rb") + while True: + # Hash 32kB chunks + data = f.read(32 * 1024) + if not data: + break + h.update(data) + f.close() + return h.hexdigest() + +__all__.append("hash_file_sha1") + + def mkdir_with_parents(dir_name): """ mkdir_with_parents(dst_dir) @@ -289,6 +304,25 @@ def unicodise(string, encoding=None, errors="replace"): __all__.append("unicodise") +def fooise(string, encoding=None, errors="replace"): + """ + Convert 'string' to Unicode or raise an exception. + """ + + if not encoding: + encoding = Config.Config().encoding + + if type(string) == unicode: + return string+"foo" + debug("Unicodising %r using %s" % (string, encoding)) + try: + return string.decode(encoding, errors)+"foo" + except UnicodeDecodeError: + raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) + +__all__.append("fooise") + + def deunicodise(string, encoding=None, errors="replace"): """ Convert unicode 'string' to , by default replacing @@ -357,13 +391,13 @@ def check_bucket_name(bucket, dns_strict=True): if invalid: raise Exceptions.ParameterError( "Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % ( - bucket, invalid.groups()[0])) + bucket, invalid.groups()[0])) else: invalid = re.search("([^A-Za-z0-9\._-])", bucket) if invalid: raise Exceptions.ParameterError( "Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % ( - bucket, invalid.groups()[0])) + bucket, invalid.groups()[0])) if len(bucket) < 3: raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket) diff --git a/s3cmd b/s3cmd index 0f8f1c0..d2a405d 100755 --- a/s3cmd +++ b/s3cmd @@ -22,6 +22,7 @@ import codecs import locale import subprocess import htmlentitydefs +import hashlib import Queue import threading @@ -151,9 +152,9 @@ def subcmd_buckets_list_all(s3): response = s3.list_all_buckets() for bucket in response["list"]: output(u"%s s3://%s" % ( - formatDateTime(bucket["CreationDate"]), - bucket["Name"], - )) + formatDateTime(bucket["CreationDate"]), + bucket["Name"], + )) def subcmd_bucket_list(s3, uri, select_dir=False): @@ -251,7 +252,7 @@ def cmd_bucket_delete(args): def fetch_local_list(args, recursive=None): local_uris = [] local_list = SortedDict(ignore_case=False) - single_file = False + is_single_file = False if type(args) not in (list, tuple): args = [args] @@ -268,7 +269,7 @@ def fetch_local_list(args, recursive=None): local_uris.append(uri) for uri in local_uris: - list_for_uri, single_file = _get_filelist_local(uri) + list_for_uri, is_single_file = _get_filelist_local(uri) local_list.update(list_for_uri) ## Single file is True if and only if the user @@ -277,9 +278,9 @@ def fetch_local_list(args, recursive=None): ## and that dir contained only one FILE. That's not ## a case of single_file==True. if len(local_list) > 1: - single_file = False + is_single_file = False - return local_list, single_file + return local_list, is_single_file def fetch_remote_list(args, require_attribs=False, recursive=None): @@ -768,6 +769,12 @@ def subcmd_cp_mv(args, process_fce, action_str, message): info(u"Public URL is: %s" % dst_uri.public_url()) +def do_cp_mv_work(item, seq, total): + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['dest_name']) + extra_headers = copy(cfg.extra_headers) + + def cp_mv_worker(): while True: try: @@ -775,6 +782,7 @@ def cp_mv_worker(): except Queue.Empty: return try: + # do_cp_mv_work(item, seq, total) response = process_fce(src_uri, dst_uri, extra_headers) output(message % {"src": src_uri, "dst": dst_uri, "seq_label": seq_label}) if Config().acl_public: @@ -831,7 +839,7 @@ def cmd_info(args): def _get_filelist_local(local_uri): - info(u"Compiling list of local files...") + info(u"Compiling list of local files for '%s', '%s', '%s'" % (local_uri, local_uri.basename(), local_uri.path())) if local_uri.isdir(): local_base = deunicodise(local_uri.basename()) local_path = deunicodise(local_uri.path()) @@ -843,9 +851,12 @@ def _get_filelist_local(local_uri): filelist = [( local_path, [], [deunicodise(local_uri.basename())] )] single_file = True loc_list = SortedDict(ignore_case=False) + # info(u"Compiling list of local files for %s..." % (filelist)) for root, dirs, files in filelist: rel_root = root.replace(local_path, local_base, 1) for f in files: + info(u"File %s (%s) -> '%s'" % (root, rel_root, f)) + fileNamePrefix, fileExtension = os.path.splitext(f) full_name = os.path.join(root, f) if not os.path.isfile(full_name): continue @@ -862,10 +873,14 @@ def _get_filelist_local(local_uri): relative_file = relative_file[2:] sr = os.stat_result(os.lstat(full_name)) loc_list[relative_file] = { + 'key': relative_file, 'full_name_unicode': unicodise(full_name), 'full_name': full_name, 'size': sr.st_size, 'mtime': sr.st_mtime, + 'file_name': f, + 'file_name_prefix': fileNamePrefix, + 'file_name_extension': fileExtension, ## TODO: Possibly more to save here... } return loc_list, single_file @@ -1178,7 +1193,7 @@ def cmd_sync_remote2local(args): seq += 1 do_remote2local_work(remote_list[file], seq, len(file_list)) - #Omitted due to threading + #Omitted due to threading def sync_remote2local_worker(): @@ -1245,7 +1260,7 @@ def do_remote2local_work(item, seq, total): mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time()) atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time()) os.utime(dst_file, (atime, mtime)) - ## FIXME: uid/gid / uname/gname handling comes here! TODO + ## FIXME: uid/gid / uname/gname handling comes here! TODO except OSError, e: try: dst_stream.close() except: pass @@ -1564,7 +1579,7 @@ def cmd_accesslog(args): output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled()) if accesslog.isLoggingEnabled(): output(u" Target prefix: %s" % accesslog.targetPrefix().uri()) - #output(u" Public Access: %s" % accesslog.isAclPublic()) + #output(u" Public Access: %s" % accesslog.isAclPublic()) def cmd_sign(args): @@ -1870,39 +1885,379 @@ def process_patterns(patterns_list, patterns_from, is_glob, option_txt=""): return patterns_compiled, patterns_textual +#============================================================================================================================== +#=== cmd_foo +#============================================================================================================================== + +def cmd_upload(args): + if (len(args) < 2): + raise ParameterError("Too few parameters! Expected: %s" % commands['foo']['param']) + if len(args) == 0: + raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + local_list, single_file_local = fetch_local_list(args) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to upload, %d remote files already present" % (local_count, remote_count)) + + work_info = { + 'remote_list': remote_list + } + + + + + # if local_count > 0: + # if not destination_base.endswith("/"): + # if not single_file_local: + # raise ParameterError( + # "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + # local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + # else: + # MLF:TODO:Want to flatten this + # for key in local_list: + # local_list[key]['remote_uri'] = unicodise(destination_base + key) + + if local_count > 0: + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + item = local_list[key] + remote_path = item['file_name'] if cfg.flatten else key + item['remote_uri'] = unicodise(destination_base + remote_path) + item['destination_base'] = destination_base + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in local_list: + uri1, uri2 = calc_upload_remote_uris(local_list[key]) + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri2)) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.parallel and len(local_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in local_list: + seq += 1 + q.put([local_list[key], seq, len(local_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=upload_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in local_list: + seq += 1 + do_upload_work(local_list[key], seq, len(local_list), work_info) + + +def upload_worker(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_upload_work(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def calc_upload_remote_uris(item): + sha1 = hash_file_sha1(item['full_name_unicode']) + # destination_base = item['destination_base'] + uri1 = u"sha1_%s.blob" % sha1 + uri2 = u"sha1_%s.blob__%s" % (sha1, item['file_name_extension']) + + # uri1 = unicodise(destination_base + u"sha1_%s.blob" % sha1) + # uri2 = unicodise(destination_base + u"sha1_%s.blob__%s" % (sha1, item['file_name_extension'])) + # item = local_list[key] + # item['key'] = key + # remote_path = item['file_name'] if cfg.flatten else item['key'] + # item['remote_uri'] = unicodise(destination_base + remote_path) + # item['destination_base'] = destination_base + + return uri1, uri2 + + +def do_upload_put(item, seq, total, work_info, uri): + remote_list = work_info['remote_list'] + # output() + + # remote_count = len(remote_list) + + # info(u"Summary: %d remote files already present, %s" % ( remote_count, remote_list.keys[0])) + # for key in remote_list: + # output('key: %s' % key) + # for key, value in remote_list.items(): + # output('key: %s' % key) + + destination_base = item['destination_base'] + item['remote_uri'] = unicodise(destination_base + uri) + if remote_list.has_key(uri): + remote_file = remote_list[uri] + output(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + else: + output(u"upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) + if True: + do_put_work(item, seq, total) + # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) + # time.sleep(5) + + + +def do_upload_deflate(item, seq, total, work_info, uri): + size = item['size'] + filename = item['full_name_unicode'] + + if size <= CONST_sharef_size: + output(u"Skipped %s with size %d" % (filename, size)) + return False + + + #Could require a --force too + with open(filename, 'w') as f: + f.write(uri) + + return True + + + +def do_upload_work(item, seq, total, work_info): + cfg = Config() + s3 = S3(cfg) + + uri1, uri2 = calc_upload_remote_uris(item) + do_upload_put(item, seq, total, work_info, uri1) + do_upload_put(item, seq, total, work_info, uri2) + if cfg.deflate: + do_upload_deflate(item, seq, total, work_info, uri1) + + + +CONST_sharef_size = 60 #A bit larger than... 40+4+5+2 +CONST_sharef_prefix='sha1' +CONST_sharef_suffix='.blob' + + +#============================================================================================================================== +#=== cmd inflate +#============================================================================================================================== + +def cmd_inflate(args): + if (len(args) < 2): + raise ParameterError("Too few parameters! Expected: %s" % commands['inflate']['param']) + if len(args) == 0: + raise ParameterError("Nothing to inflate. Expecting a local file or directory and a S3 URI destination.") + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + local_list, single_file_local = fetch_local_list(args) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) + + work_info = { + 'remote_list': remote_list + } + + if local_count > 0: + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + item = local_list[key] + remote_path = item['file_name'] if cfg.flatten else key + item['remote_uri'] = unicodise(destination_base + remote_path) + item['destination_base'] = destination_base + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in local_list: + uri1, uri2 = calc_upload_remote_uris(local_list[key]) + output(u"inflate: %s <- %s" % (local_list[key]['full_name_unicode'], uri1)) + output(u"inflate: %s <- %s" % (local_list[key]['full_name_unicode'], uri2)) + + warning(u"Exitting now because of --dry-run") + return + + if cfg.parallel and len(local_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in local_list: + seq += 1 + q.put([local_list[key], seq, len(local_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=inflate_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in local_list: + seq += 1 + do_inflate_work(local_list[key], seq, len(local_list), work_info) + + +def inflate_worker(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_inflate_work(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def calc_inflate_remote_uri(item): + size = item['size'] + filename = item['full_name_unicode'] + + if size > CONST_sharef_size: + output(u"Skipped %s with size %d" % (filename, size)) + return "", True + + + content = open(filename).read().strip(' \t\n\r') + m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) + + if m is None: + output(u"Skipped %s with content %s" % (filename, content)) + return "", False + + output(u"Blob-reference %s with content %s" % (filename, content)) + + return content, True + + +def do_inflate_download(item, seq, total, work_info, uri): + remote_list = work_info['remote_list'] + # output() + + # remote_count = len(remote_list) + + # info(u"Summary: %d remote files already present, %s" % ( remote_count, remote_list.keys[0])) + # for key in remote_list: + # output('key: %s' % key) + # for key, value in remote_list.items(): + # output('key: %s' % key) + + destination_base = item['destination_base'] + + item['remote_uri'] = unicodise(destination_base + uri) + item['local_filename'] = item['full_name_unicode'] + item['object_uri_str'] = item['remote_uri'] + if remote_list.has_key(uri): + remote_file = remote_list[uri] + output(u"Inflating from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + if True: + do_get_work(item, seq, total) + else: + output(u"Unable to inflate (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) + # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) + # time.sleep(5) + + +def do_inflate_work(item, seq, total, work_info): + uri, should_inflate = calc_inflate_remote_uri(item) + if should_inflate: + local_inflate = False + if not local_inflate: + do_inflate_download(item, seq, total, work_info, uri) + + #MLF:TODO:Should also have a local inflate location? + + +#============================================================================================================================== +#============================================================================================================================== +#============================================================================================================================== + + + def get_commands_list(): return [ {"cmd": "mb", "label": "Make bucket", "param": "s3://BUCKET", "func": cmd_bucket_create, "argc": 1}, {"cmd": "rb", "label": "Remove bucket", "param": "s3://BUCKET", "func": cmd_bucket_delete, "argc": 1}, - {"cmd": "ls", "label": "List objects or buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_ls, - "argc": 0}, - {"cmd": "la", "label": "List all object in all buckets", "param": "", "func": cmd_buckets_list_all_all, - "argc": 0}, - {"cmd": "put", "label": "Put file into bucket", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", - "func": cmd_object_put, "argc": 2}, - {"cmd": "get", "label": "Get file from bucket", "param": "s3://BUCKET/OBJECT LOCAL_FILE", - "func": cmd_object_get, "argc": 1}, - {"cmd": "del", "label": "Delete file from bucket", "param": "s3://BUCKET/OBJECT", "func": cmd_object_del, - "argc": 1}, + {"cmd": "ls", "label": "List objects or buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_ls, "argc": 0}, + {"cmd": "la", "label": "List all object in all buckets", "param": "", "func": cmd_buckets_list_all_all, "argc": 0}, + {"cmd": "put", "label": "Put file into bucket", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_object_put, "argc": 2}, + {"cmd": "get", "label": "Get file from bucket", "param": "s3://BUCKET/OBJECT LOCAL_FILE", "func": cmd_object_get, "argc": 1}, + {"cmd": "del", "label": "Delete file from bucket", "param": "s3://BUCKET/OBJECT", "func": cmd_object_del, "argc": 1}, #{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1}, - {"cmd": "sync", "label": "Synchronize a directory tree to S3", - "param": "LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func": cmd_sync, "argc": 2}, - {"cmd": "du", "label": "Disk usage by buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_du, - "argc": 0}, - {"cmd": "info", "label": "Get various information about Buckets or Files", "param": "s3://BUCKET[/OBJECT]", - "func": cmd_info, "argc": 1}, - {"cmd": "cp", "label": "Copy object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_cp - , "argc": 2}, - {"cmd": "mv", "label": "Move object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_mv - , "argc": 2}, - {"cmd": "setacl", "label": "Modify Access control list for Bucket or Files", "param": "s3://BUCKET[/OBJECT]" - , "func": cmd_setacl, "argc": 1}, - {"cmd": "accesslog", "label": "Enable/disable bucket access logging", "param": "s3://BUCKET", - "func": cmd_accesslog, "argc": 1}, - {"cmd": "sign", "label": "Sign arbitrary string using the secret key", "param": "STRING-TO-SIGN", - "func": cmd_sign, "argc": 1}, - {"cmd": "fixbucket", "label": "Fix invalid file names in a bucket", "param": "s3://BUCKET[/PREFIX]", - "func": cmd_fixbucket, "argc": 1}, + {"cmd": "sync", "label": "Synchronize a directory tree to S3", "param": "LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func": cmd_sync, "argc": 2}, + {"cmd": "du", "label": "Disk usage by buckets", "param": "[s3://BUCKET[/PREFIX]]", "func": cmd_du, "argc": 0}, + {"cmd": "info", "label": "Get various information about Buckets or Files", "param": "s3://BUCKET[/OBJECT]", "func": cmd_info, "argc": 1}, + {"cmd": "cp", "label": "Copy object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_cp, "argc": 2}, + {"cmd": "mv", "label": "Move object", "param": "s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func": cmd_mv, "argc": 2}, + {"cmd": "setacl", "label": "Modify Access control list for Bucket or Files", "param": "s3://BUCKET[/OBJECT]", "func": cmd_setacl, "argc": 1}, + {"cmd": "accesslog", "label": "Enable/disable bucket access logging", "param": "s3://BUCKET", "func": cmd_accesslog, "argc": 1}, + {"cmd": "sign", "label": "Sign arbitrary string using the secret key", "param": "STRING-TO-SIGN", "func": cmd_sign, "argc": 1}, + {"cmd": "fixbucket", "label": "Fix invalid file names in a bucket", "param": "s3://BUCKET[/PREFIX]", "func": cmd_fixbucket, "argc": 1}, ## CloudFront commands {"cmd": "cflist", "label": "List CloudFront distribution points", "param": "", "func": CfCmd.info, @@ -1915,6 +2270,12 @@ def get_commands_list(): "func": CfCmd.delete, "argc": 1}, {"cmd": "cfmodify", "label": "Change CloudFront distribution point parameters", "param": "cf://DIST_ID", "func": CfCmd.modify, "argc": 1}, + + ## ContentAddressing commands + {"cmd": "upload", "label": "Push content", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_upload, "argc": 2}, + {"cmd": "inflate", "label": "Mark's Amazing Command2", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_inflate, "argc": 2}, + + ] @@ -2010,6 +2371,10 @@ def main(): help="Skip over files that exist at the destination (only for [get] and [sync] commands).") optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") + optparser.add_option("--flatten", dest="flatten", action="store_true", + help="Flatten a recursive operation (makes target name space flat even if source was recursive).") + optparser.add_option("--deflate", dest="deflate", action="store_true", + help="Stub files during an upload.") optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option("--acl-private", dest="acl_public", action="store_false", @@ -2280,7 +2645,7 @@ def main(): sys.exit(1) if len(args) < commands[command]["argc"]: - error(u"Not enough paramters for command '%s'" % command) + error(u"Not enough parameters for command '%s'" % command) sys.exit(1) try: diff --git a/testsuite/etc/brokenlink.png b/testsuite/etc/brokenlink.png deleted file mode 120000 index e69de29..0000000 diff --git a/testsuite/etc/linked.png b/testsuite/etc/linked.png deleted file mode 120000 index e69de29..0000000 diff --git a/testsuite/etc/linked1.png b/testsuite/etc/linked1.png deleted file mode 120000 index e69de29..0000000 diff --git a/testsuite/etc/more/linked-dir b/testsuite/etc/more/linked-dir deleted file mode 120000 index e69de29..0000000 From ee80927747dc2b6f178952e7340f4d54594037d5 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Wed, 15 Feb 2012 20:38:40 -0800 Subject: [PATCH 04/16] Fix to error messaging, and a bit less noise --- s3cmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3cmd b/s3cmd index d2a405d..ddd408f 100755 --- a/s3cmd +++ b/s3cmd @@ -855,7 +855,7 @@ def _get_filelist_local(local_uri): for root, dirs, files in filelist: rel_root = root.replace(local_path, local_base, 1) for f in files: - info(u"File %s (%s) -> '%s'" % (root, rel_root, f)) +# info(u"File %s (%s) -> '%s'" % (root, rel_root, f)) fileNamePrefix, fileExtension = os.path.splitext(f) full_name = os.path.join(root, f) if not os.path.isfile(full_name): @@ -2180,7 +2180,7 @@ def calc_inflate_remote_uri(item): if size > CONST_sharef_size: output(u"Skipped %s with size %d" % (filename, size)) - return "", True + return "", False content = open(filename).read().strip(' \t\n\r') From 32eff080094f6972aa52e89120407fdd1e0cd870 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Wed, 15 Feb 2012 21:01:44 -0800 Subject: [PATCH 05/16] Don't upload deflated files a second time --- s3cmd | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/s3cmd b/s3cmd index ddd408f..f41cfb7 100755 --- a/s3cmd +++ b/s3cmd @@ -1999,6 +1999,15 @@ def upload_worker(): def calc_upload_remote_uris(item): + size = item['size'] + filename = item['full_name_unicode'] + + if size <= CONST_sharef_size: + uri, is_sharef = calc_inflate_remote_uri(item) + if is_sharef: + output(u"Skipped sharef file %s with size %d and sharef %s" % (filename, size, uri)) + return "", "", False + sha1 = hash_file_sha1(item['full_name_unicode']) # destination_base = item['destination_base'] uri1 = u"sha1_%s.blob" % sha1 @@ -2012,7 +2021,7 @@ def calc_upload_remote_uris(item): # item['remote_uri'] = unicodise(destination_base + remote_path) # item['destination_base'] = destination_base - return uri1, uri2 + return uri1, uri2, True def do_upload_put(item, seq, total, work_info, uri): @@ -2062,11 +2071,12 @@ def do_upload_work(item, seq, total, work_info): cfg = Config() s3 = S3(cfg) - uri1, uri2 = calc_upload_remote_uris(item) - do_upload_put(item, seq, total, work_info, uri1) - do_upload_put(item, seq, total, work_info, uri2) - if cfg.deflate: - do_upload_deflate(item, seq, total, work_info, uri1) + uri1, uri2, should_upload = calc_upload_remote_uris(item) + if should_upload: + do_upload_put(item, seq, total, work_info, uri1) + do_upload_put(item, seq, total, work_info, uri2) + if cfg.deflate: + do_upload_deflate(item, seq, total, work_info, uri1) From 3c9c69605f7d95dad8e02426d8c18af497525805 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Wed, 15 Feb 2012 21:44:00 -0800 Subject: [PATCH 06/16] Manifest implementation --- s3cmd | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 8 deletions(-) diff --git a/s3cmd b/s3cmd index f41cfb7..984073d 100755 --- a/s3cmd +++ b/s3cmd @@ -2005,7 +2005,7 @@ def calc_upload_remote_uris(item): if size <= CONST_sharef_size: uri, is_sharef = calc_inflate_remote_uri(item) if is_sharef: - output(u"Skipped sharef file %s with size %d and sharef %s" % (filename, size, uri)) + info(u"Skipped sharef file %s with size %d and sharef %s" % (filename, size, uri)) return "", "", False sha1 = hash_file_sha1(item['full_name_unicode']) @@ -2040,7 +2040,7 @@ def do_upload_put(item, seq, total, work_info, uri): item['remote_uri'] = unicodise(destination_base + uri) if remote_list.has_key(uri): remote_file = remote_list[uri] - output(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + info(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) else: output(u"upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) if True: @@ -2055,7 +2055,7 @@ def do_upload_deflate(item, seq, total, work_info, uri): filename = item['full_name_unicode'] if size <= CONST_sharef_size: - output(u"Skipped %s with size %d" % (filename, size)) + info(u"Skipped %s with size %d" % (filename, size)) return False @@ -2189,7 +2189,7 @@ def calc_inflate_remote_uri(item): filename = item['full_name_unicode'] if size > CONST_sharef_size: - output(u"Skipped %s with size %d" % (filename, size)) + info(u"Skipped %s with size %d" % (filename, size)) return "", False @@ -2197,10 +2197,10 @@ def calc_inflate_remote_uri(item): m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) if m is None: - output(u"Skipped %s with content %s" % (filename, content)) + info(u"Skipped %s with content %s" % (filename, content)) return "", False - output(u"Blob-reference %s with content %s" % (filename, content)) +# output(u"Blob-reference %s with content %s" % (filename, content)) return content, True @@ -2243,6 +2243,120 @@ def do_inflate_work(item, seq, total, work_info): #MLF:TODO:Should also have a local inflate location? +#============================================================================================================================== +#=== cmd manifest +#============================================================================================================================== + +def cmd_manifest(args): + if (len(args) < 2): + raise ParameterError("Too few parameters! Expected: %s" % commands['inflate']['param']) + if len(args) == 0: + raise ParameterError("Nothing to manifest. Expecting a local file or directory and a S3 URI for additional information.") + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + local_list, single_file_local = fetch_local_list(args) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) + + work_info = { + 'remote_list': remote_list + } + + + if cfg.parallel and len(local_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in local_list: + seq += 1 + q.put([local_list[key], seq, len(local_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=manifest_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in local_list: + seq += 1 + do_manifest_work(local_list[key], seq, len(local_list), work_info) + + +def manifest_worker(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_manifest_work(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def calc_manifest_info(item, work_info): + size = item['size'] + filename = item['full_name_unicode'] + + #MLF: Can handle everything locally + if size > CONST_sharef_size: + sha = hash_file_sha1(item['full_name_unicode']) + uri1 = u"sha1_%s.blob" % sha + uri2 = u"sha1_%s.blob__%s" % (sha, item['file_name_extension']) + return uri1, uri2, size, True + + content = open(filename).read().strip(' \t\n\r') + m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) + + if m is None: + info(u"Skipped %s with content %s" % (filename, content)) + return "", "", size, False + + uri = content + uri1 = uri + uri2 = u"%s__%s" % (uri1, item['file_name_extension']) + remote_list = work_info['remote_list'] + if remote_list.has_key(uri): + remote_file = remote_list[uri] + size = remote_file['size'] + return uri1, uri2, size, True + else: + info(u"Skipped %s with no remote file at %s" % (filename, uri)) + return uri1, uri2, size, False + + +def do_manifest_work(item, seq, total, work_info): + uri1, uri2, size, should_output = calc_manifest_info(item, work_info) + filename = item['full_name_unicode'] + + if should_output: + output(u"%s\t%s\t%s\t%s" % (filename, uri1, uri2, size)) + #============================================================================================================================== #============================================================================================================================== #============================================================================================================================== @@ -2282,8 +2396,9 @@ def get_commands_list(): "func": CfCmd.modify, "argc": 1}, ## ContentAddressing commands - {"cmd": "upload", "label": "Push content", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_upload, "argc": 2}, - {"cmd": "inflate", "label": "Mark's Amazing Command2", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_inflate, "argc": 2}, + {"cmd": "upload", "label": "Upload file contents (and optionally --deflate the files)", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_upload, "argc": 2}, + {"cmd": "inflate", "label": "Inflate any deflated files from the source", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_inflate, "argc": 2}, + {"cmd": "manifest", "label": "Create a manifest of contents", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_manifest, "argc": 2}, ] From d94232eb0f4fae96c9d07759c2057c6dc0b9ab7b Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Thu, 16 Feb 2012 13:28:23 -0800 Subject: [PATCH 07/16] Support for blobcache --- .gitignore | 2 ++ s3cmd | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c38fa4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +*.iml diff --git a/s3cmd b/s3cmd index 984073d..ec64a5a 100755 --- a/s3cmd +++ b/s3cmd @@ -23,6 +23,7 @@ import locale import subprocess import htmlentitydefs import hashlib +import shutil import Queue import threading @@ -1947,9 +1948,13 @@ def cmd_upload(args): for key in exclude_list: output(u"exclude: %s" % unicodise(key)) for key in local_list: - uri1, uri2 = calc_upload_remote_uris(local_list[key]) - output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) - output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri2)) + uri1, uri2, should_upload = calc_upload_remote_uris(local_list[key]) + if should_upload: + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri2)) + else: + output(u"skipped: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) + warning(u"Exitting now because of --dry-run") return @@ -2006,7 +2011,7 @@ def calc_upload_remote_uris(item): uri, is_sharef = calc_inflate_remote_uri(item) if is_sharef: info(u"Skipped sharef file %s with size %d and sharef %s" % (filename, size, uri)) - return "", "", False + return uri, "", False sha1 = hash_file_sha1(item['full_name_unicode']) # destination_base = item['destination_base'] @@ -2073,6 +2078,11 @@ def do_upload_work(item, seq, total, work_info): uri1, uri2, should_upload = calc_upload_remote_uris(item) if should_upload: + if os.path.exists('.blobcache'): + local_filename = os.path.join('.blobcache',uri1) + if not os.path.exists(local_filename): + shutil.copyfile(item['full_name_unicode'],local_filename) + do_upload_put(item, seq, total, work_info, uri1) do_upload_put(item, seq, total, work_info, uri2) if cfg.deflate: @@ -2236,9 +2246,16 @@ def do_inflate_download(item, seq, total, work_info, uri): def do_inflate_work(item, seq, total, work_info): uri, should_inflate = calc_inflate_remote_uri(item) if should_inflate: - local_inflate = False - if not local_inflate: + local_filename = os.path.join('.blobcache',uri) + #local_filename.split("/")) + if os.path.exists(local_filename): + output(u"Inflating from local (%d) (%s): %s <- %s " % (seq, uri, item['full_name_unicode'], local_filename)) + shutil.copyfile(local_filename, item['full_name_unicode']) + else: do_inflate_download(item, seq, total, work_info, uri) + if os.path.exists('.blobcache'): + shutil.copyfile(item['full_name_unicode'], local_filename) + #MLF:TODO:Should also have a local inflate location? From 5dd7c20f9267dad381476c0b709eb5aa5334618b Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sun, 11 Mar 2012 13:08:39 -0700 Subject: [PATCH 08/16] Conversion to support 'link' and also a couple other fixes --- S3/Config.py | 1 + s3cmd | 164 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 9 deletions(-) diff --git a/S3/Config.py b/S3/Config.py index 9f83ebe..14c70a0 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -82,6 +82,7 @@ class Config(object): select_dir = False max_retries = 5 retry_delay = 3 + can_link = True ## Creating a singleton def __new__(self, configfile=None): diff --git a/s3cmd b/s3cmd index ec64a5a..218c140 100755 --- a/s3cmd +++ b/s3cmd @@ -2047,7 +2047,7 @@ def do_upload_put(item, seq, total, work_info, uri): remote_file = remote_list[uri] info(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) else: - output(u"upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) + info(u"upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) if True: do_put_work(item, seq, total) # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) @@ -2064,6 +2064,9 @@ def do_upload_deflate(item, seq, total, work_info, uri): return False + #MLF:Need to detach a possible simlink + os.remove(filename) + #Could require a --force too with open(filename, 'w') as f: f.write(uri) @@ -2081,7 +2084,15 @@ def do_upload_work(item, seq, total, work_info): if os.path.exists('.blobcache'): local_filename = os.path.join('.blobcache',uri1) if not os.path.exists(local_filename): - shutil.copyfile(item['full_name_unicode'],local_filename) + if cfg.can_link: + try: + os.remove(local_filename) + os.link(item['full_name_unicode'],local_filename) + except OSError, e: + cfg.can_link = False + shutil.copyfile(item['full_name_unicode'],local_filename) + else: + shutil.copyfile(item['full_name_unicode'],local_filename) do_upload_put(item, seq, total, work_info, uri1) do_upload_put(item, seq, total, work_info, uri2) @@ -2234,11 +2245,13 @@ def do_inflate_download(item, seq, total, work_info, uri): item['object_uri_str'] = item['remote_uri'] if remote_list.has_key(uri): remote_file = remote_list[uri] - output(u"Inflating from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + info(u"Inflating from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) if True: do_get_work(item, seq, total) + return True else: - output(u"Unable to inflate (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) + error(u"Unable to inflate (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) + return False # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) # time.sleep(5) @@ -2249,12 +2262,29 @@ def do_inflate_work(item, seq, total, work_info): local_filename = os.path.join('.blobcache',uri) #local_filename.split("/")) if os.path.exists(local_filename): - output(u"Inflating from local (%d) (%s): %s <- %s " % (seq, uri, item['full_name_unicode'], local_filename)) - shutil.copyfile(local_filename, item['full_name_unicode']) + info(u"Inflating from local (%d) (%s): %s <- %s (%s)" % (seq, uri, item['full_name_unicode'], local_filename, ("Link" if cfg.can_link else "Copy")) ) + if cfg.can_link: + try: + os.remove(item['full_name_unicode']) + os.link(local_filename, item['full_name_unicode']) + except OSError, e: + cfg.can_link = False + shutil.copyfile(local_filename, item['full_name_unicode']) + else: + shutil.copyfile(local_filename, item['full_name_unicode']) else: - do_inflate_download(item, seq, total, work_info, uri) - if os.path.exists('.blobcache'): - shutil.copyfile(item['full_name_unicode'], local_filename) + success = do_inflate_download(item, seq, total, work_info, uri) + if success: + if os.path.exists('.blobcache'): + if cfg.can_link: + try: + os.remove(local_filename) + os.link(item['full_name_unicode'], local_filename) + except OSError, e: + cfg.can_link = False + shutil.copyfile(item['full_name_unicode'], local_filename) + else: + shutil.copyfile(item['full_name_unicode'], local_filename) #MLF:TODO:Should also have a local inflate location? @@ -2374,6 +2404,121 @@ def do_manifest_work(item, seq, total, work_info): if should_output: output(u"%s\t%s\t%s\t%s" % (filename, uri1, uri2, size)) + + +#============================================================================================================================== +#=== cmd cleanup +#============================================================================================================================== + +def cmd_cleanup(args): + if (len(args) < 1): + raise ParameterError("Too few parameters! Expected: %s" % commands['cleanup']['param']) + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + local_list, single_file_local = fetch_local_list(args) + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) + + work_info = { + 'remote_list': remote_list + } + + + if cfg.parallel and len(remote_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + global q + q = Queue.Queue() + + seq = 0 + for key in remote_list: + seq += 1 + q.put([remote_list[key], seq, len(remote_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=cleanup_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in remote_list: + seq += 1 + do_cleanup_work(remote_list[key], seq, len(remote_list), work_info) + + +def cleanup_worker(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_cleanup_work(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def calc_cleanup_info(item, work_info): + size = item['size'] + filename = item['base_uri'] #item['full_name_unicode'] + + #MLF: Can handle everything locally + if size > CONST_sharef_size: + sha = hash_file_sha1(item['full_name_unicode']) + uri1 = u"sha1_%s.blob" % sha + uri2 = u"sha1_%s.blob__%s" % (sha, item['file_name_extension']) + return uri1, uri2, size, True + + content = open(filename).read().strip(' \t\n\r') + m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) + + if m is None: + info(u"Skipped %s with content %s" % (filename, content)) + return "", "", size, False + + uri = content + uri1 = uri + uri2 = u"%s__%s" % (uri1, item['file_name_extension']) + remote_list = work_info['remote_list'] + if remote_list.has_key(uri): + remote_file = remote_list[uri] + size = remote_file['size'] + return uri1, uri2, size, True + else: + info(u"Skipped %s with no remote file at %s" % (filename, uri)) + return uri1, uri2, size, False + + +def do_cleanup_work(item, seq, total, work_info): + size = item['size'] + should_output = size <= CONST_sharef_size +# uri1, uri2, size, should_output = calc_manifest_info(item, work_info) + + if should_output: + filename = item['base_uri'] + output(u"%s\t%s\t%s\t%s" % (filename, item['object_uri_str'], item['object_key'], size)) + #============================================================================================================================== #============================================================================================================================== #============================================================================================================================== @@ -2416,6 +2561,7 @@ def get_commands_list(): {"cmd": "upload", "label": "Upload file contents (and optionally --deflate the files)", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_upload, "argc": 2}, {"cmd": "inflate", "label": "Inflate any deflated files from the source", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_inflate, "argc": 2}, {"cmd": "manifest", "label": "Create a manifest of contents", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_manifest, "argc": 2}, + {"cmd": "cleanup", "label": "Cleanup (list or delete) any fingerprints", "param": "s3://BUCKET[/PREFIX]", "func": cmd_cleanup, "argc": 1}, ] From d876815a43c13ede6ca89b505e267a48aee6673e Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sat, 17 Mar 2012 10:04:30 -0700 Subject: [PATCH 09/16] Add copydeflate capability. Removed the testsuite. --- .gitignore | 2 + s3cmd | 263 ++++++++++++++++++++++++++++ testsuite/binary/random-crap | Bin 20480 -> 0 bytes testsuite/binary/random-crap.md5 | 1 - testsuite/blahBlah/Blah.txt | 1 - testsuite/crappy-file-name.tar.gz | Bin 281 -> 0 bytes testsuite/encodings/GBK.tar.gz | Bin 170 -> 0 bytes testsuite/encodings/UTF-8.tar.gz | Bin 231 -> 0 bytes testsuite/etc/AtomicClockRadio.ttf | Bin 34532 -> 0 bytes testsuite/etc/TypeRa.ttf | Bin 56708 -> 0 bytes testsuite/etc/logo.png | Bin 22059 -> 0 bytes testsuite/etc/more/give-me-more.txt | 0 testsuite/exclude.encodings | 1 - 13 files changed, 265 insertions(+), 3 deletions(-) delete mode 100644 testsuite/binary/random-crap delete mode 100644 testsuite/binary/random-crap.md5 delete mode 100644 testsuite/blahBlah/Blah.txt delete mode 100644 testsuite/crappy-file-name.tar.gz delete mode 100644 testsuite/encodings/GBK.tar.gz delete mode 100644 testsuite/encodings/UTF-8.tar.gz delete mode 100644 testsuite/etc/AtomicClockRadio.ttf delete mode 100644 testsuite/etc/TypeRa.ttf delete mode 100644 testsuite/etc/logo.png delete mode 100644 testsuite/etc/more/give-me-more.txt delete mode 100644 testsuite/exclude.encodings diff --git a/.gitignore b/.gitignore index c38fa4e..25b4871 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .idea *.iml +*.pyc +.DS_Store diff --git a/s3cmd b/s3cmd index 218c140..32ba859 100755 --- a/s3cmd +++ b/s3cmd @@ -2073,6 +2073,30 @@ def do_upload_deflate(item, seq, total, work_info, uri): return True +def do_copy_deflate_into(item, seq, total, work_info, uri, destfilename): + dir = os.path.dirname(destfilename) + if not os.path.exists(dir): + os.makedirs(dir) + + size = item['size'] + filename = item['full_name_unicode'] + + if size <= CONST_sharef_size: + info(u"Skipped %s with size %d" % (filename, size)) + return False + + + #MLF:Need to detach a possible simlink + if os.path.exists(destfilename): + os.remove(destfilename) + + #Could require a --force too + with open(destfilename, 'w') as f: + f.write(uri) + + return True + + def do_upload_work(item, seq, total, work_info): @@ -2290,6 +2314,244 @@ def do_inflate_work(item, seq, total, work_info): #MLF:TODO:Should also have a local inflate location? +#============================================================================================================================== +#=== cmd_foo +#============================================================================================================================== + +def cmd_copydeflate(args): + if (len(args) < 3): + raise ParameterError("Too few parameters! Expected: %s" % commands['foo']['param']) + if len(args) == 0: + raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") + + ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) + destination_base_uri = S3Uri(args.pop()) + if destination_base_uri.type != 's3': + raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) + destination_base = str(destination_base_uri) + + local_destination = args.pop() + + local_list, single_file_local = fetch_local_list(args) + + local_list, exclude_list = _filelist_filter_exclude_include(local_list) + + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + local_count = len(local_list) + remote_count = len(remote_list) + + info(u"Summary: %d local files to copy, %d annex files present" % (local_count, remote_count)) + + work_info = { + 'remote_list': remote_list, + 'local_destination': local_destination + } + + + + + # if local_count > 0: + # if not destination_base.endswith("/"): + # if not single_file_local: + # raise ParameterError( + # "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + # local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + # else: + # MLF:TODO:Want to flatten this + # for key in local_list: + # local_list[key]['remote_uri'] = unicodise(destination_base + key) + + if local_count > 0: + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + item = local_list[key] + remote_path = item['file_name'] if cfg.flatten else key + item['remote_uri'] = unicodise(destination_base + remote_path) + item['destination_base'] = destination_base + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + for key in local_list: + uri1, uri2, should_upload = calc_upload_remote_uris(local_list[key]) + if should_upload: + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) + output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], uri2)) + else: + output(u"skipped: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) + + + warning(u"Exitting now because of --dry-run") + return + + global q + if cfg.parallel and len(local_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + q = Queue.Queue() + + seq = 0 + for key in local_list: + seq += 1 + q.put([local_list[key], seq, len(local_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=copydeflate_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in local_list: + seq += 1 + do_copydeflate_work(local_list[key], seq, len(local_list), work_info) + + + #MLF:Now do the pure copy + if cfg.parallel and len(exclude_list) > 1: + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + q = Queue.Queue() + + seq = 0 + for key in exclude_list: + seq += 1 + q.put([exclude_list[key], seq, len(exclude_list), work_info]) + + for i in range(cfg.workers): + t = threading.Thread(target=copydeflate_worker2) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + q.join() + else: + seq = 0 + for key in exclude_list: + seq += 1 + do_copydeflate_work2(exclude_list[key], seq, len(exclude_list), work_info) + + + + + + +def copydeflate_worker(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_copydeflate_work(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + +def copydeflate_worker2(): + while True: + try: + (item, seq, total, work_info) = q.get_nowait() + except Queue.Empty: + return + try: + do_copydeflate_work2(item, seq, total, work_info) + except Exception, e: + report_exception(e) + exit + q.task_done() + + + +def do_linkorcopy_from_to(source_filename, dest_filename): + dir = os.path.dirname(dest_filename) + if not os.path.exists(dir): + os.makedirs(dir) + if cfg.can_link: + try: + os.remove(dest_filename) + os.link(source_filename,dest_filename) + except OSError, e: + cfg.can_link = False + shutil.copyfile(source_filename,dest_filename) + else: + shutil.copyfile(source_filename,dest_filename) + + +def do_copy_from_to(source_filename, dest_filename): + dir = os.path.dirname(dest_filename) + if not os.path.exists(dir): + os.makedirs(dir) + + shutil.copyfile(source_filename,dest_filename) + + + +def do_copydeflate_work(item, seq, total, work_info): + cfg = Config() + s3 = S3(cfg) + + uri1, uri2, should_upload = calc_upload_remote_uris(item) + if should_upload: + if os.path.exists('.blobcache'): + local_filename = os.path.join('.blobcache',uri1) + if not os.path.exists(local_filename): + do_copy_from_to(item['full_name_unicode'],local_filename) + + do_upload_put(item, seq, total, work_info, uri1) + do_upload_put(item, seq, total, work_info, uri2) + #Now do a deflate copy + + source_filename = item['full_name_unicode'] + dest_filename = os.path.join(work_info['local_destination'], item['key']) + info(u"Deflate copy (%s): %s to %s key: %s => %s" % (seq, source_filename, work_info['local_destination'], item['key'], dest_filename)) +# info(u"Deflate copy: '%s' to '%s' ", (item['full_name_unicode'], work_info['local_destination'])) + do_copy_deflate_into(item, seq, total, work_info, uri1, dest_filename) + + else: + #Need to do a straight copy. + source_filename = item['full_name_unicode'] + dest_filename = os.path.join(work_info['local_destination'], item['key']) + info(u"Straight copy (%s): %s to %s key: %s => %s" % (seq, source_filename, work_info['local_destination'], item['key'], dest_filename)) + do_copy_from_to(source_filename, dest_filename) + +# if cfg.deflate: +# do_upload_deflate(item, seq, total, work_info, uri1) + + + +def do_copydeflate_work2(item, seq, total, work_info): + cfg = Config() + s3 = S3(cfg) + + source_filename = item['full_name_unicode'] + dest_filename = os.path.join(work_info['local_destination'], item['key']) + info(u"Straight copy (%s): %s to %s key: %s => %s" % (seq, source_filename, work_info['local_destination'], item['key'], dest_filename)) + do_copy_from_to(source_filename, dest_filename) + + + + #============================================================================================================================== #=== cmd manifest #============================================================================================================================== @@ -2562,6 +2824,7 @@ def get_commands_list(): {"cmd": "inflate", "label": "Inflate any deflated files from the source", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_inflate, "argc": 2}, {"cmd": "manifest", "label": "Create a manifest of contents", "param": "FILE [FILE...] s3://BUCKET[/PREFIX]", "func": cmd_manifest, "argc": 2}, {"cmd": "cleanup", "label": "Cleanup (list or delete) any fingerprints", "param": "s3://BUCKET[/PREFIX]", "func": cmd_cleanup, "argc": 1}, + {"cmd": "copydeflate", "label": "Copy files from one location to a destination directory, deflating them in the process", "param": "FILE [FILE...] DIR s3://BUCKET[/PREFIX]", "func": cmd_copydeflate, "argc": 3}, ] diff --git a/testsuite/binary/random-crap b/testsuite/binary/random-crap deleted file mode 100644 index d24db3a3529d537bcc5edebc575bff770f1a529c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmV(lK=i-kTUtDSqttWt7;q`vA#S&LjjbxD5VB1vHju6N@02{|LWjYn${BU;T zwmiLj6u65cHUV=ouF?$@+s)6^nh%`8YtKX~T9nXQ&rxNGpoc4c*<>G$>gVOkgDvav z)Ft)KR{C&W6em~LP;F$V_O58}6ln!1T%g=Vbm6=u9kcR8M| z&8jT&TY3!8LCv9!nL12kax9Q(jpcHCug(2JqtXOA2H^U8?+(r`2dimNFD1u`oUe@K zRQxI+jdQgIuhu+|5By|JB>XTv-rR~Qv?_N+g*WD?1EfBOFGC)rH``=Bqpw%{8?Fbp zF?qUU_#zhXpzyFi^k$Foy(JV z*)FW1+?&dn9emzWzVz%av%}#4kWXSH#~+R<0J`B zrI)dyN-h&>K>8mJ;iNREl8(Lz^O6>($}T;D8y8Yq=#Pbm5;vB`#5dRH`*Zz7#BTmn z^nA|7u{H~>kH6Fo&?v-2C&l4!`6^q2GL02b?YuP}-0dsAaqOG_)BoVp83{7P>ZLcMQ21*up%}2b+bhrz*SF0*~i^Hv$P~BYiYvT(JUD@ z*8ZeM+;%6*M~#BFhbR$|G<8Bn@39iT(?T}Xf{|{(jvgS!#qEM5+idQ`mlm)?f{FI` zyBr8?3G!fJe3OqyNZ|RqOX46dQxI;z;!oim`cH;M&EAkH_^;8?UvA4Nd=XIHFQ)>F zU9A!TR+L8mnN;PRjyEA};kPi?2%3$-r#0>csBi;nL=XV@lFc3ENun`ikr7kWFdLW7 zocgml*GhJ-BwIBKTo!*$n;wn_wEM#kIhrtnulPs3gQ?^n|t^9zaXNC|rxWd9Z+gH|ijX(wmMmJ_c9 zmcVSN!A|&gV2Z6cK3&voNXGy=?{Kvi%^N)t0bgI!R~vTNOl4ut>APnj^V+u*QRRNes!s>$ z$X(&YCxO)33ZeO+elHX^2(gscWgc4HQd5xKfHH~=_hn0(g?h~X9GEL1kUHd`)x1Un zZTs$&(&Y(+{8l0PEEMcw2_%F*TY`TCb zJmt#>xfv2%v^@<>L3Ug%4n{MehW`c$?iIN8=TS~7G2<(4y(R94Km`_(W6ecoYRqlX zP+3&o-GeleFE9&V%;px&8;wq;%mqzD>yJCxfcw}WXlPA#u?+$`=Cft>6I-W^Xlg&J zQ+cc_6AL|bkmTleNtd?~M=JGxo2o*4Ne8ZV{0o;zfK&pN2E)qTjHV!gR7B8Yg}?-+ zi--bO5&kHs%r5ROxy%Ymv&<31u!ZeCVEq>T0Fru!1FLE>3B*5T z{~^2WYFaL#WAa_IUQBo3$YFchK$b{y*E`_CHrIP^=F*|SzQn0HR6SR4dw5a_Z^MVD zKFm?V9u{kLf=6?+;`)lnxp{%`OQ%q&`XmB`sKrd*9a?HSc01gV8(zroDTjj>06mnh zRL5a!I%1)r%e}|QOJ7r#2M0-vm4w@5lQ;$_bUmqXuBPrKVUlawk|8maP^MqG?aINX zaL($-GU-qbu+jlDT=>Y&xi$5k8HPVw*=;-9Z;)jkJe0J`_f?frZw`MN!n7eNKw*ituFUR?Pe;fn5r|r$iDL{lzNo#i!EpwUNxMiywJ)tMD7}s zG0%O(_h2LIFQtvVSc-ENNhJ(xP0QNFxUgb{f{=mlKQ-?SffZaSCO$WIG*O||jlpo* zP4Vs9vMajKQgy4y8HRV!hHaC{rGf_*xKZy%?c_xhguK9a#4Y525~4h=R5-w_7^;o|4VTf+NxkmKCtkBztT_lj0h-1_QTx_xq^d>@ zGPllH*P9G#AmoxLxbR29yf8sIYx@M@r%1^6%5-L|G-Nbltx|l7<$49LJ%phn1%3fF zc(HnNJaudoq%6lE=lGD~S5VUjZ54X6f>EQ;aW2~T>_xZEn2q*6z0v3=jHX>A1Ug^4 zS^cgB{urTJfWA(4W(7^um6xV#tFZS`jLN}#B_bNNK#t|! z(NL8&zaxzx=mr)w-hVv`yDWJ857MN$S`RUlVH|K->)%;5q{0pL2^nBML=wSJpHSE- zZa0>tjnBlMziQHNu8$Yvl!e%CUxVzj(3z0{b5gUJ-00&-W;)hib5vMH+n8taqUoO?hycw1+Prn<%=)|nP7UN$Q8Qdfq0j-lKgLY|r*qpWmV%(6wq+eY z`wvPM(;b?3^KU(QbkeyLkc`z}ZNW(>VPahfaV`){fzj!!9-gQLO9KwWrH04W#-F1N zpOb=W7-tiF7i}&x{ZoHzIS!y#AuwEeN~2~9ttVN*`?X{ha`B-9ysu>tuaLU&HJ?82k-(B> zt}26hotc|})p(uRdq3t`n?1Z9VAcR1@tLiMz(eI3G1vyl-DX5hR$zVZx)~#6$S1=b z#(J1F^lH8g7Hhav7m{f`duwG|AAxF}1H>wh#4XH}y%GYV69$^TB1S|EyUx{paY7ae z6n%jM;k_mlF4a>RO2Z%@d6i)vE^~15l>>4FHoyLO-esbc&;3Q^>XT)vDF^Dxhf5I6 zTIiWD>{OEpk2& z+pN&JLVBd&u;R8(`!wH zew)LW1ds;him{_uoa<^7Yh9S>wGfY-WCmgeh6j0TqxXbKSBAHc5iLSdT&vR+GMGbhYBCd2r+ONzRh|Jwt_HM^?d4lG%!nZg?vz zD>FRp!c6~r67URy29pZHaJ};7LtM<;BX%g56l*LK$i_8oCwPN;>cxp!M;xq=BFCF` z&~InBHrzXL-?C9=o5SSHgNv?lG;vE+%}P#w(*JawZ2C{jVfZbay=pPPEEBJ#3cf8p z;Owusq{J)%aX9CmoVOaj@z7M}2@^_gB%uw#>vM~#ifuJPvKZ~<%06`l5Lt*u38WR~ zz!OO8nm4IW7p(<8$3EIe#POgRNts0_RHZfX1^nty>y78bQ9)4mTTF%si!Bo>2xWrtEfM^2F<}cCH4+TUcM2 zwfy7(v5XPj6y$K!ephE^w?v0DUE zeyXh~*2QIPyB<5bdE!8K`iHCIlU#u}j`eCQ5-siyAVNpJzznVd)~`YJJ%$YKUE1g~ zrkxZgG2C7PVf%qak7S(xPWVr@rnX{rc@W{+b)Pc3rEJ?K8=1Gi(12jIG-A?Sibj^KstXCT>c22m}<-;*kuNdz-y>lN|YyyF-iL3w2_cMU##iPLvl4U z;xX7cA65+_zpfewkiG1HTa8f!9)!zGXjV@-a;L{EA?}(<9dQOXtrV@guY^`7_1-(x z;HeUa>iF`v`!y_<{>eI>SqCQW!j4a42&~#8y@KZKH4ftly-olJt+}-2;)X`=*id`! zTZ!GH>XN@+Edzm&hX*b9L$%w$BIsg4s2vVmjHx0}EIH^(?=7d5;2-C*U+?fTt&gdB zs6?HjwfG~0e&fIXMs=uTQJWiXAztnp#EwV-Aszb{iemlCL7P+S&#>2ahsAF1hX1G?;9*SA4nAXQyw z4kf+A+OK**5sWcCJGyKT4hZ@t#Gld9bx4w!#V)L<#vNI79>($~J@x_lR+t6Ql@1$U zDZWo!4&qX(Glu)2A_t9r41T72Tmc&@NFYDtRQqIp7MI(_x__?H*U|%~jQ#9AEyEDW zGQMjQtK}%KfN%Tuw0(yuzTzX1P3DXRgx*FE3bTRql25y3gh&y2oUqF7e*PP`{u@5k zrpcfQdvgI;_FQc}5)XN(BZc)C~@1zK)YAK;1L*$RdBR?VGOY75EWi9i7vB?{o zd_*=aAiNp#`ZbeLUb(-OAW}Y#-^B5Yuxm_4wrQ!0DAPYNBGru?x4b7UxD@zj!CN#% zdlc?%a_B{FCcXuW6f4IZ{F8R(fbDeoQ#4@)j8&50P|%r8*D=`X|n=pbgmU? zH+kt$$4fUofanP~k2xu2XC1q+;= zF9-*|Q|%NIx#CCuONx>2UKT;E01e)(cHpH94F*5Je;YeR5g(BzC=OX1&BdQ7=FE6J zDBExs7@dzh23MO}6~j~O#$liy!5-NJ0km;~lJmipq5rBc^$ty5rR z0@-Ho{v01}1S%|_mK6bB_QA|jw()*M**a9afM3^I&P7Lp34&{SB11|V*MvyU886!% z1XKaIY7S<}?idszUEUb%m;OXZLP$9Kj4&|E2n_%w;o?jP`yEndpsb5pNBC}1uZp)+ ziSQ43V~Fvg^?3*^G5aqHq}M=;g(0|`p%;Uk`0YLMQwI;f8rikvH=NMWr01swT`!t4 z`Mu%&!O>{9_zTqJj=JU|O=)Eb#FUcN$lvSYU~5MyJ{!BxubBkqaa^CI6$h^r2#t~1 zoqk5L7gRHNR5{lcMADc{jQTaweQ4&RyJGn4 zak#{u4ke2GYI19rL)I+44)R5D{#=th)K-w!g8eSncojuZcKLmH2?0N+E(B>W^Jjv2 zr`hnZDT&m)0J0&;uUNig5&{QGmvu={nWG^gIEZlNhZ?7)&{@b$gefJ=Y$*TEr8K=4 z4>XpxXxg9zcYJ^C0qiGXp_mR<@ZLpO6yQIb?3J}z1HYfHhz{gEou)PS1 zSK1NW=q%yR;*HImLeT0%9somnoTor+2K1cmtD$U|gQFpY+1_)m#^Imsbr3 z6NY!&&g3kK6nhOq{?y}+Fsu%EHk{|!gt_)kTazru_0OYEt~PR@Jv52uvBnblSmfDU zkM+`atlt9`>i`(_fIMmFNxJIf64|%-*s%@Z#uSjlC&lXP@Cf1YY&Zfy#ym^UZllQb z;0Hu*8I6~f;&e~vB2QWD&1NtLX_ymX=Vd9f&21x7x4K8ipx8c(7E*&K(rV2SBVEvq z2sWDxO3hxEr|0VtMVO*9^CyEf8aFwP>~9V^4==C#pDrc;hEP zZ<&PCx)u)ZwANyxXOk%4?!j^!hCxw-$Ql)@$8ojsLzBADk2_MHFvQhyx z>G_#)i#t!dv8{d{1(E$N$2vV6ZYWC{+$7u=6IY3vZ zo}=Xeo1a**Jzb&`086m9)_?9_QPpRXGXAS+Eq50%5{vnbuXJ zrkrVRJarE0IHhnJK+fJl+^pF2=uy&{D^!&pDf(gmB-YVNF^8KXe~3qjDC_X@Zd2ZJ zXrIzBgwe;N$?}FseGWg*a(KNV@-c)kT9TA|{lx{6BxR0h*8IDUp7`i#{{UZEC^Lqc z2=q`i@p>WQO1C8oM-K~!OiP8gByT)}Fbbw^N?v;JPhGI5QMg<0&GWOKr&2FC>qkUB zsrW*9QRB}}6pcJB8*SAS1@z5pSiNlZC?!u9AP<7&Amlzmh+&F^oSEhdR%y8;g3qO5 zIe!{G=zFBt4MK9l%hX*gi6AwOgT0?QGP93808ExICWiiSXJPH20INQhjX7PxbRNiM zb47rXT68%zo66%}ae)k0J?U9M1!~S)7xyQXTAG@wS~eXU+|tTcqSn+Q|`h7)$mO z-^*MbDivg1AQeyA5K7|HC0X}25`oP*rlkYFhNWOti>i<7<)_FgBEDLY<2}hcB;%N_ z0G3wie+3471;WW+0L6Qws|;j5VX1iIia5Uwk&hvX(u=+wmiOV(%s#4DVx&5sz9Q@Z z+f-9;`gRRiHQ8k;fS}XirQx)RV+$)-+t=3E^i%>)gQc5fO(0_#F#1G|gJgaUKabvn zZm!>=D_e&I)xXw$#QMjH^YjzM>JL<57Jo6obZd*Nqx>k_vfx31OJ!~3AfC-WrN{Rs zm7}iM$tjg6AbA6&b-Hz~cUkjw{pK~=s-C7gP_y=V!4OC`g=05_)_Ev<4pYYx9uPq) z47NFB0}2W%?n39 zG}d^5S2VJwc*j_~nhP{Mz=)OB*dPX{(4`z1qeX#@a2Dy%oqD`VbQ6==xLx1mWf5i0 zV(5}XjeWC&%cCrqi1s1Qh$XvGmAlN$oo*cs5EUQ6b`vTPfCkg+P9XAAYb7jlja_?m zQb39}CCkHZoBRUHXTQq`lioD8fdx-Ow8M(u-Ep{4<4R;DZA|HSaGVrshzqrmJ&vLZ z?mOLarIN5Y?xMfFdby7Cb*%-c!&hQ`Ti6kBLAnJXj9Hd-%m()X@-2RNajh(y!P3i&(Byb-emimG7!#!>BOk;7Yw0SZxBI3`p<+ zKwpmV=TarVR_Q3>(RE;Oo77&vZi|5U4~UcM7#L}=%KNfsX5*7t&%W_}xFF*R^BPg5 zYeIC1)+kj1Aykf!*~&zogZhT~PJYG2RP-zzyh#xF>W+_(tcOaG_^Y~qlHqYZm!&IZ z|C-EM;LhC0!KKi48qKTR8ND5l`Lr*`bsrLkP%LpIqhPl$j}cGYq4S)4SAyy&a0o|k z5uSl|z>{a`w~jX!2x`QKKbxAr^Me(L50rCUyid`!0BlZN8M;s0+_+XY*sV z`?`^Dt%9PU4EbudT7Tvn6WnE1rE?-v@J}KXQY985U+mt4C`L2nkXcVe#vIM8vm*A= zz&`CMDC%<~(0Rf*V2y<>u+ra7G1AxMrIxnzW`SKVHPzj=IXrVi806BRgIZbL9U5!k zjH^Gx$R#vDomi}Xcm%Xj)+>cTLz?-?##72w9Gq&OepJhg-ui?KWMTNbs#XVS-q=wdf;zM!!}8|B7ze#Pwq zd!cqj?av%9CZK;6O45RHOom;?x<^8;W`sAQZu#2By8dIZ)KYn7WWYQppfp6SM%+kIP`$ zyj)QLRR<%fKOW?7Mvzh>ysKEuG~?&TAwVysIKzd|6|oPs-H!x9 zi9ka!H#kL}*pz6R+ml*Dic4m=MsOmS=O$pCKL8S|fcHu?N z=8<6|*eX2WYI_;5W_?(C8AbSJ^geJG_h?M0QUWCyFn(1~MG^qMK7(IPDMu0(eEc=5 zwS}&U1D-rFSy3G#CLG#hGAe_^D%x-Th=#a=S4)L=@cDO^NkHHxb!7*5A+DiWIh~x{ z;zWinC|J&P+!${N9*c}3fnSV!S7P*19)ht_{@R`mmxmzqTn^xUCpPx6BQ@fl<@-fd zKDX(9+@|0Q+W^D16}AfpVPAgku^{sK-{K`Zy&u`sBKvZU8Hz_=caXDoiVBQNn@+Nh zj%*(-V1_Mqq;5Nl!Haxbr6GrfZ4;q!VoiCBJM4$OlZSNf%bPMyx^C1HV9hLn(#R}7 z>aX=C_&)>AsMXCCm;oPftM|(-_xVb08mCroU zP9|bR6sKNSCZC(_tSC{lfX~oNg7W~1+-7H$58DSn#2|5dsWXr(L~qn$H9fTgw)iqcOZ-N)8TK{{wfOp-H zO4Hs^3&iEIXaB}Sox9h;zQGz!#>7%#%w54Sb1x_Ntt^#=emh6xqj=5zE$#aA5V{rHKBJ%h<;%ATK*ERUoP=Lq>Y%@TMZ@?h49f@~Whp^Df+`Xx zSQn0;UA$;f<1%+SoPYC#yO(oHpmAz!5|OT9JA5ci{9_%U2uNeCRdz^0Qd=mHc|ZGA z?2{VB6r5DiHHn=RmCyY~89sMkaCj|{rMkX04wTrABJT8<)TfSj2VfvXz+H3| z)mi5ZvlBLFD4zNC+BR2ryf&`Hz`^r+=d)Is@R3~i31Z`oO`(~$pEQ`9uU zNE{-?O(L8$4j3oy$9Nr!h`+f3oQtJtLVfA9^JYghZUO~E^`LKYNjHZ7D^l~iBg1ft zI3Dn2sWWEM>zAa`sW2_#j>O?kI{wJ@1I#Gnj(bVwFbpOqg6U^J|A9-bVtf9D3NBT_ z)aj12oE#ofa%(W#7z2qc$An21W8_E7H(GxlX{0)W zVPIBwV#U<>6|;8*+<~LP)?i~&+gi`Kr4L6j}$d-kIors*?y~%fe~RV<(rHtq?N(f z^%1V!tFbr2FaU7uaYPsM#47E4%oWEExY!An{uTL1bwwf5>aR48g~1nu4bfIKbJl3D zQ{Rt}2z5j|gWM$94ptFv?>*%CGUIHK(%z(0aQ%h7 zQ6qiFHl|Y#DG@gK7`CTKV zi1l3=ocX7UDu?@{RalxZ2-0DPf)L-*#;8}X#UV2obk$yLs)TaC?O$QP+s>1+0UW+$ zvmjXfk`4s14V{te=?Rtbt{D~fgF!?*D+O;)64u8f`?0$vVJDA7L{*h4NC1}kUIFHi zRYilYf3LxQC9XO3q6;&Jd^jiE^7=NjfBk?yy9w8tE4}ddv%RT<%X0}TmiSH*6L(p> z8ecO+C?zQ&zph~Fx9{&?uS*TrKMOzwciIW8LqFQ!c6VqyObqU0Nf3=Vnfh7L<$drf zg548(c|{%#FsclKBZq#p2_}Ko^==}7y!Kbp4plq&B`}AlVQ1GZNkHotf&Qd6#u#8@S8%Bh?!E)6 zef)4xy(B)Q>PEgyCz9JUr6v^O`o_RbiVUk1BsKlG*!9x&OQ(bD*x*3}FWFfUx#tPu zTaesL_lix@quY4~dMeyN&+rpNe1ql$fTm6ZmE>ArD3F5#Ty67MR|q$wF9br(HZ!J* zwJvvMWQLS5N^rv}w9d%6{^d1(@w$iL#6>vTF=h`;%Z7jYLg|)*mnZE_Zegsr~l75SKnQ&V&Y2bIA*giegZ2vO@V+<3QY;K=lpWgfNkIx|70h;fu9?e$XV3I``I`Y_H}4%=UCA0QG+S!)`e46?F$jaV2FWxujr53>YLcL{e2e zg^2NBHIS+~Pe7`D{&4?`FaBDwj~gfGtf|ae>IX5~Lxl<3_98wn$p0hx?&I(NzaxN^ z15^rC28~+W1I^mLV@|*Kk1#d$_rqE_#G@G@eZwK8=B}fhZW0efA~O-Ra^_O(j{zqD z3pIMKkDDEj0FLfuq!N0r9}2s-UB$F~_X?Yy1R9W6-FVFb55MeOU3^ftDyzknfu7!- z)c%5_*Pxw-G2l^nO3)gUrka@pCR9Pq?Abd3 z1(IzpIWN9!q;Z0irTJ~NN$xX!6m^DJZ(GPY23oI=+XK+T6k&pMdIt-C(_m8-lmrG+ zF#HP*`sCy3$Mt_8ZtR~`h!l&lpe5oN!RQEkEDja`V_EYPBkoEbo=#E$yzePb&+q3gpMRU z7d;8Iclk(RnF*7lrr(NwaXJt{X%lI$XX|Qj8eTqoaFcpLAxNEj^*Jj#;01|(ijd`q zI;{=iYizhfoJ4$J&CHp@OQM=*f2Q#LCK{!QcjxMoz}E1yM-fhCfIX@ron)#*9pSER zK>dV-si(xzZRMoE9p=b%aDBjq^QEE}Y>^e_FZ{i?i}D2TyJ5v6X$!WseaT!-Oj z_Vt;@t0;zCFMC40x|S&e=Fu{-yM=Bitm~+HNT?bcBN%4rUt_6+>NjKRzm#b2^cv(J zEs`;7l#kWjub}UUhvS1Nm-NF=t$7t0x;i3I@~j@ z+m<`WV;M`n!|VeG>X;}@x#k_?$B*-=RzGN&zXjZ>+Sy=Z;N!`LbwOm+fOl#`&8$Ka zr9keSL1z#2mpp6{xLUhGnw18S9ALs4nQ$ds(X&AXUR=9k+&)!1yNk@~)6*$*xCQ5M z$9jQw6qXN~F_D1VgiR%Nxa3CS|Gg39UmqPS-3I+ob40kmlZQx;vfT1p4?y! zF^KX}m8S9XGigvXXuWSvGdtX#)0V8*7}cX;B9+$D(FP4OzuHK zmw3=Qy=a}@j$0`$gFB;78_`?j>1C~s-Tl(sh}XuxV-9fwVk18d)!G(HbRm$-=Q}aI zE9JA12827o6<*TrV#d@-F*7bhM;vm!6j+udVJ4=PJ;;QOchKh|lM{eOJDU2{lVmA) zvwLdFPgjl5fx;H~L=V8}pmUktBs0++6WeiM>T!o8WV8Y>bXWHEaZq{#C2?x$+mo4v z^(%^}M6=xKBG7u!`S%qZh8APDuXPPY^}yqIHd!-gZDDw;_SrPs(%}^ z8C7o?_hk=#9hW1}0n1XnQwuo$rN1s$AP#LMrG(TCZ>7ygLTI0MEG!nDU%3s|A?Z`d zYp#rpwknaiA&XV0M~u=p;ixyz7W&8}tAJv6ASMyqql4=@%TjG9`8iplYPDt!-OChX zc@HCKz(!}gD&*S&?V3<+v4wh<7Gx?`WtbMyqIm=yW;oka1jyJDJqVe*eleVzgMy>^ zs{=;pODi5S%6a8vMOVE2iG?r1^-#$Xq4|3~ONkIRuMqo<%&oP_5k0JUNQ9)L zUdU!C@Bn)ZxfWX}8@L|0W-}xr8|tI2N`&2UAnZ&Au+Ey=?RigJXS@Ii9)e*6ltBPV zBZ*a$eQNMkk!k~w8ILz+6SpK#ab-(BU}-*5-fF7KlbGH{epha3z2aO=Z9UlOwV51~ zGbqOsWx%g<;1C@iSBFo!0^XZSh6N5p+Q8iqa&$3lOD}rdY{EU!C3{VJ-t~AU1@i6r z@S_sVHofuWYTqacLsb+PFY{^9lt0GppH(?Roec?CF5_BA?iZbyP5 z(n)dx(eOplCsW_49npI7J~$fTg9zXE>T5uW5Wr{`+E>n0(QNQy$xErIDszt?7je?H zy_jlRvb&apHzIT1``MQsa%p=rRk|8u{pjR6yM)&+qVPzFeEVyd*DD7=bK9r|9V-N# z^0|1a_+KEsaU7E2W{oMK&!4qy)4>qmIjMX+^2s}2)h~Mfy{kiahagt#DESwaJ#y+;2PcB zbEOA~#XI%7JhtIi(Z5fP=+ARrS{KH08(Sd*z-v#wEXC!d4*o`2S!>lNoXZ#t0v*eU z?W#87z-@vd9U0{LVev}Zxh`<{YC3_t)QHerd9C1VdUz@(Fx%csd(hh_ONewd*A+)o zm_|xU%aiRBO;V;t+S?M`?*r?q ze@fXpFiNgOf3VzIj^i_;9+xBxNOABu3bwL)iiYFHUwey@C`rLNEGoHu_aw?%TJLCE z>?B;mCyo?+w-Xik7jG`wnOn^Ya_h3rHY_Iy;tnIFBz|9S2H0oO`<#_Nx4J!i%fUhr zE#zj?pDDAW5|FkHBY3n#5HRYj=x9$roVnwUrkjv_Gk&a4-SKE5+UWDX71Zu0@VPI% z+haKfoEiOak$+2Yn?A$W#~CsaP3bY3BMdhmu`k&wenYcGdkh;RvHop ztk$!6pX0+HUR#dzD*&P%1l^f62|TRHuN81%)2wy(ri*h>4=D`JhHS`bt+14R5Z%Tjj`!owt7cvzWq)Hapwn#H_D zvSaEvZE%@PXXYClE6zxDZ0V`A1IdS6h*g0)0DBKXb09zf%JU|up7~PKTZSBi3~Has z&yxSzbg2xAJT*{_Afk~-u`+?EFGrGdY|7Z?qKA6W&!7uT_J304shy%a3imQZKw#z{?Wpd3Rp3W7@Xo-kL$lkE9*;Q!m3BlxtG4 z0|?8Qyx^n5Bh89~bPY6Ypia9<^R75>z7xl6&U5)m8NQ1!e-dMsc zi~+rC-pVeNr8N-=tbJQkvz!0}vXd0@x-#VeMI21yGn>a3xN>yl=h6W-KT0(MFWH>4 z`N>F>T8%C_Si@tQ0#p;uwZJUk%|2Y?jmiEQ!yXJ1` zUNjNiV!s)jR5}FxWb~YI6L4kH?GC_d!X`WHx_$3aFujgPDDIfd-`fgMp&$rCRK<_B zr!}6Re4mZ)>KM6&WA^vdHuL>z0ibrrB(4I17c!N0`A5dfF*$<}1Vh`*_#7kCZ?8ZR zA#H5JB!3Ra4(9QSHXDCxCJ60YcwHs}UTVQ?Vc-=SX`;FrsJNj&cp9rfVpBs}<`XU9 zQ4uJ6!baFaGKPoOfW%wv{!@xKc0kpeOA;@`f<$hzxtyCd7+BRYaW64vWs(F5azaR=JZ5sC{Pa&89IO^d-3L zMqR0bVUIfp;O5t0VtsN*DkJL@qjmVursZY~01 z2>bR%an%kjWRZBD1)3y>Q2KUgU!r~7+QaC&YDnV*leKN$`hVZo_&xrol|#mjED$b& zJJRS56rYwt%3IOgVgxr%-B}#NIRsvj?s{di>VdV}BdFsZuUosBwx#l*T@Ag8_{|U8 zb>m%`up9CZ<|QNT2?pu<)L>;+k`Z(3eIz@ZG3Hbv-Gr3rxKMTTHaP08Z=+BErp3~} zYJpAZoYMwJ`Rn*f!Eog1WVnAlE~A;@HWh`tj91m+G!;(^t%VNfSKhH<)l|og?Tftu zT|y0a-wUT6aE9+GUD<6_=|FyQV-pSJ@r3Is!z~dG&pyb;``BOB2TA;15ZBj!UL)e>evm{y zq-I8(-gGtnb5f@QH#o6(XB~gLuW(~(qb!z24dOZ*KevpI~F-_ueE4 z!Z?<;FU-vL3N7-E1Io}RdQ@8(yS3X>H+EJN+oPY#ULsuOhwrE9N+ z)+uT_z)gtTY+u5dDe69Af!K*O697kGYLW-_5XE;bHlUhA4TVbB`uzd$L!yXFe|+vPqBA{i!AiFfTtJ*)x_qF%Gy2;Q z-~i~)vaQJGe3dErC2lvAF4cECuAR+4G$DPiS*btS^>C%Tv0kY8Atn(sG50(16LMA~ zn{|*k&|4v@QmAaM@H1A5E1WP!P`NF-U;$NTVw>!$|BiS%0nas$*;(#4=La0G>MlbP zGn~hdyRCvnHs(Z$hvM1Z*biO{nW~Awm7iO)1$YAABl1$dF9emibNRnZK zLbGmfUhQQdOjk=bE-*Q#wfE|b;k=_8oF znbWl-0)207hC#*;on6a-H(CF4!oAqv=L*S9{~k8iAh~Ap6l+p#-rU(c!!v%eOc43) z7ZVh4@071{8bO%eGW?o6b-F^!GHYDYN$Rsnfj*58W>LN&SnSyiR_3)m=#zyxFzpfa zJNE|5Qx2Z6Tjlb3a5dc9M3CL!{HGgc9gElHyqugB&rERG0JW?P>+C$#Q9tf(xvw>$ zH_}>^sMPho``a$p4_k`IRhCp8V4Y_xP)$d|P*ZZUgr#NUUEH zla`J`#Kq+)e5q?0;HyZ(h2Z~(-#So};KUSV^I+EI-}o^j@1 zrf5?^zbnh7T{V}F$#o6{X*8pL%e78JOuS3XA`iIjUXDlkRs8pR!mLxQhM;%>S~*TgO&1^R6*qTmtp+f6kNn^oyhtj45YO0%n*&0zgFyaS z%4xm0j@XxqIp}O>t0Ik3zquQr!Pw71gI+sNTD3-g;G*r4ph1N3bz(RYuW_Cn`Wpcz zB4jEu@Ja(4jwsimy8_y?48Z>3C$Rzb4+`*!hC)fzyNVYA94GCa=$S_7?y2w|AwE`B zhoft@%NiCgD7!(&-(2n&1)g^_IC_wmGZmy7Ni8g~!AfO>^0+l8u>#Q{>$hdP#JRGC zmy4HdMVM&5QJEh;M--wF2i^^&D#x6C31vvTIMA;Hy6C3}*(o1YEC*w8%ODz*2xs`m z?(zo0UD!Zgdl9xRFC2F(STl303$*EE?=yO*p}~jySLrNK9M|z~Jm(GvI1SR;0H%aU{Vyo7tzmDqpOOH%CAp;sd;>7BSKN?C#-2 zx|Ja?4HNx1*$>2T?(-r8=R4uP(V#XJ${cth56gyW)if&S?{RKdmh-Wa?AsYaVlx(6dc`Sy)Z5EGVFgq3b!lL z&}RSk^l&`zGMv!-8_%{I7{FD^Z{WX}3`%i~huEY-J@J!@CTV#`rNwbT$DPr9+Rc%K zl(eHq&#Zy?@SXA$alkKc6Wk*h+Bq7GeC3*6mShTH%~=hmB0B%F;LS;c*U_t1Oor}$6s7nY?)`Fb}2LwRusT4jX@MW zwPl7x;Wm7MzIHvo5&qQWRC{EkH9yuT9Y2FyeSKoJc5k=k9$}p)IpzK;JPZ%&>Jl`e zXXIVpb=()evIgt1nfH^%Rj3X(F z_7y2MZNzHz)8o%=oKNJJDHg5Q>U&fM_`zqFH*U?HKl5jvj|fPt)cNE{fB`49^m-OW z(OOpMXX02IXQXw43NBc>|`42v?VBBHL6Efy26TDA0XCc`Q$RJ_cF~yyA z4eT7`Woutq<=^?x-{u#lOpt4|>EaDN<8E~HaPZ~?|4z~xZsI_^;rb3(Duas185AEC zO%9ItwC~wD9Diaiu6bT&Xtl}Nb;!q^+KnQecR{}1Q-P`}xikMUWzf6VG+| zcVZ#xzC0TppI>|i(Wda*yWL*I8Zz0oL9f?{Afxg8WhZk)tm};?CUlaPU6twTpph_@ z_^qA$>lW)L#FEOZy1h>p!TOfM~8q_J42Pd$-G0N3Wv{%f(kk}3WE=9h(W<@lvsKH8+Z*1Ejd)8mxq25)lZOu~-efpsDpH%%V;+o6mOMzj1&6pkeff)*!wPK$l| z6|2&GYR(_oQa0~h|A>x+7K6Y+tdeM@Yxkb?KD%*rkLef?WP)hUrV(1g-C+6ScL3s9 zrcX6!@yrVrc78w;!gsNy_rn;%MJipZDvJ~C=Uw~mUZBZM^9#fgA%9Y0^Im4~cb}YBU(e`Q7yY;c7^X1#;j#d&EuJU5Y#OV`4Dg z`rX}XLrv4EB*?S)QXeyq$sFPFyACSzb@Ah6pOKgWO?sC$^{{r%Zs=soZWX0;#(#Oc z8h>4B1^>S|d$l%JGchYRHF~k1$glW~ARspIl70??ee(hrQI@Ro}!uOr#2#%CA^7U+)l(*=@CsA7}cz%0X+NKzsotT{-sWiu47 zqq9X4`f#6LAjFKGSH4f`8ZX@&kA5t%1kHi7GPL06AZ>E*e}zn6o3R6B6vK$3m`2GG$4 zyi&DSPd+Who}2Qvo5@RgRc7(#k*k|Zagec8bfkB=SyFUye;r?whP^%wAN80afzIb` z%YPG5(AcZ*%qN)?%{taI@!YyCBp!d5S~e=#Nzh&Pq!S6T`yAgnQUvh$hV`&Uhw=Iv zVA)Q;+r``KH&D{=EcMeJ-UDZO6s-p=JwKfF@hTO|)W)V{(w~VB&Ki!g;{`QcZ6y8Z z$pjLf>Vo}{)qM%<+;4+TxIif3oOIK@n}_fmN+Q!YpNTZlZXE$~D6xd^LAkML#nT4E zeqvX`ud5G7`DB8xY+G|d>z{QM7u<}dY5Q2!u;kcFttY5}2QSf!J3-|+OKO@Y67_OR zE;pYQV+CAdhw9D$m-B%&r`nlrPql5#3JP$2lA62fr7ZSQpbE(NsH9;=ZOt4(Zr7KT zDWTya`@x+t#b};SIt+wH7cVp~;rF@jBo+v!-Hmh|#{A+@fW3g|T3(N7849fkI8Mows)2)VM2@Sn z@cxA-&M!RAUD-Mdx2RN(w5~{XMY8s_L43DF3z{^ zS2GZ_6rJa~nlh_GeQ%CE4Tc`R_-=dC*;cuDWEYUeDTVd`M9AcS9*z07pj<{8!~DPk%FmS|%>njJ_0!Mss;+uLw7b!&k+s zv7Z?@HZA?%hnQ#hq2GfvnVSH01BMA?Ju728M_X*e7xHcAF_b;9iK`0^)1SazN+cD38Z%*FJaA|EzGnFIW5wO^NA#!5)-%6Pt-8 zQ^@6hA`|9)fA3<(+VrgIVL+3kBd=@qvl;9(?0F`-M*cXNX_E?~lp}3H+l`>}R%S2U zRfmriIQogu;!Y=+bP}nq6yDN3BXw@~hvjf!3iz8bV_ore!Iq|CNIkbCWZx>)&v7{4 z+{B4{sM*i^K>3cpFfjpwZeU53JH+mZv&{lYYBQ z{ckG^DqXbo2zs+CajBOwmG*OU1VNE@`<~{?TnWl1Z<7fX(t>{Q>HsQyWEZBOI${^B z>JwvipPS(Tv$s$r>C#Z?91ZNp>FlRE?#*>v%m{e{PAGpmI2a#-OP96vR9BRnmZ z0*5ctoV4o8Jq~A8h?&Fu$%I~SR4{ZNyE=j~i-K6P& zQSk#3$nZ1nyNcSd770&Eq8e1S^rrh+^u`>Uq|JClBu`a9E5ZYaPd{_X!wP3<-ZVvq z8kh@xAd3?l@c7FR=>_n9Yy+m3zlWmmbT9%#=UZ_BgOa(YWmGCR_{hoATqLR|$$GDT HWEm{3CTNCP diff --git a/testsuite/binary/random-crap.md5 b/testsuite/binary/random-crap.md5 deleted file mode 100644 index f870fb1..0000000 --- a/testsuite/binary/random-crap.md5 +++ /dev/null @@ -1 +0,0 @@ -cb76ecee9a834eadd96b226493acac28 random-crap diff --git a/testsuite/blahBlah/Blah.txt b/testsuite/blahBlah/Blah.txt deleted file mode 100644 index ecfa2db..0000000 --- a/testsuite/blahBlah/Blah.txt +++ /dev/null @@ -1 +0,0 @@ -Blah diff --git a/testsuite/crappy-file-name.tar.gz b/testsuite/crappy-file-name.tar.gz deleted file mode 100644 index f11f6d78348aa26381d259c257a5b4dbcc56526b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 281 zcmV+!0p|W6iwFRE4>e2x1MSnq+F35fr9b4UmzO-d#8 z3nNmbWqeF|Oi2Eg^dsR=lN$4H~b-~`43DKP1q6Xh8W!s_7f9_AKI3vQJW;8GD)ArU0E-FFsteL5F$J+qsCR+aJUHKwmO&CCx!tzX`~>fPdi{rC9g|Lo>q SKn4%g{X6EUSu)L; zyIp>Ej`)+`-9F17*O^G1s;Qe)c%|MmWuHZdSIEK0=n&zkx2NK!|6M*!Fe>}8)AAn9 zR->?1KabGdyZ4uzjMkqPDsiko63eBMV>_}*aKT>Q zdp*DI+1{DmJrI{lRR;EYx~FHR->+Z4e(&|ikRhUJG(cH;Zqt_j<@Z0okIcC+kg1| zH$1pKHy65nzAM}0tBSd4*5Ebqk-8;PJA0B+qDSrN_ zJb#oEntn5TkmH|_=Zzjdd1~-b*B=s{DiN9I4jmmCrk;P=Nc8x-IsWy-!>5kXPczqY z{;zZX;*sIQdw*kk?-P7}fy>u*?C7H>zjOBeuM<7VN#@pLC-xp2d%1U*%X1ee4DqkW zEaJcR_#=8CIy1D;bVuhIYB#H+^DL+Tc66TO^PSOo6J^X-qVqfz&9|cS0>AYQ5`^WD zr`vNwT_zKqXQ*HXqw_4Mzdt(9@%eqxc@r7)Y;>Nd4)fLM+&(``CfCBxmh#(kTYh_P z%WuzZ`R%zazdg6*x97I}_FSL8cw|ZO?%{(+cNfI}FJkfXTz>`l7 z?|E#`-o8Bti`NzJ*i_uOX>0M04O_MjZMc2g)^)}08@ArHY1`J~nr$2o^=>I{UN>~# zhAmq*Y}z=F7SjkVp(5Q)!*r02(r!LqPY38xP9LScbb{ld{qts2`vTlj2UNH8rMozfq!ek5h7up5 zUd~^n&75zD?&Ex0_}>P;vXSPkIeGN(fsx``=D&l*q2WCTj`E!+xy*;TL?e9HS~|qB z5iQG*mhS)^9ooD9u|vZr_;Gt_KNt8AUpR4TkGizGmeX>3qZYin+`iPV%tR$a%N}1> z{9N{$bqj_^dlroDy5qX?E8{O_=4BUfy_{rH-eDdw+sqHm-$H+n~n|6gdzt*2|hP{{M&&*yTS^Yi&U|7DKO z5Vt(@MSgRZ3iLp!XquZ#Etx_tXDHv4%VwH#1uEnVCd&;lYiRxB{VRKD^{W0=9T(-l z>0fn`BNtcE^#f+Z=(GID<_)96pDPVLIM})JS!y1CZt$TSI_Gs>b3^BX)#i!v3%~e9 zcEN@3t`} zL;3GG-)EODU9v3-DNkeOOXep5eFsX*%)mhDddf9XM^krOQ}di$PqAs?;@q;XOg_J; zJwLy7>FmC_nVI>l=?30{xWKHCU+j1P0DCtL#9<#0*hBaL1K>d%u$i9-ZCN4gyqujA z(#}gXD`!f*P`;k8mA}a;8#p2mP&eh7e@6pKeNFCUo<7Pl=DqY3eUyHeem}i#uZij^ zEF|ha48#10hxrf((RhBCKLHHojPlt^NHp=ZaLVa^KFVhWqI3S#UsC=#FIUb9JC)DM zX~0wht04J82(56|=Q#hj`C|E{=md#>%$zpFY309Xw)+mBNP@5+(NwLG-8E~(wJ%@; z+7SXHTnBatk&b-04RIp=AKXCC3)(geXXXoYdEkc^HqL?97$C=#_f!nV;WzCb;K9)u7uh==)r3qs^a zJiHI-F@Eua=oIsJf4NhzQr1z1(FUhjoz}M1DXgT3os#%@!9lM$26DnnTa&qmAE&$| zJfulzL+1X7!?bldq0ar}1(xc**VG^y?w;Hn>2C`2`wB~MDlBi!w|C`d&dtwWZnWfWDEVqBx&6gxKoOWgup+bM0^{xCXsBUqzs@(&RW@^eU%oUgA77yfB z*lOc$xE3C+CL6;emw*TO9dw5Dr^SEf1V$1ZXN1!oe<#AxL?589nb$M(Vw%>T(h{?H zacK#O`jK1MoNMdO<%s!W(cHr9Rz_lGzH9#M_I$>G6GR1JR)}nn#Q*!)p28 zBU=eZOb zHgg6Ga&Ef$`Ry6A(DAu6Tr0(0`8@>-6Sx9>HMk-zv^vchVfHhE_8DoPXM|H!Q3=Hn z(ews1vbe2KR~rG=Y0?#x;5tI2gEH_ru7}S>yFfl`E~y^pwUn5dplx4!t(exL%5Aw* z;L=_#rroJspo(5CwZ&NokcVD{Q47>;B{tv>(4|92$@J^%#!HGEu_ANFZ|@FLMeL#OEawRFrLO2KmpIg z^O0VIPu`RkQ$8y+s!sc+&~FVyWdw!7b3-LEPLoxZxQCzgB0oo!+45FKM+`!(cYzA} ztDF`%4db;LaC#8iIN$& zDV>Qe&vE_-F^^v6M+lBiyohy44I%bk-wuIM7=n91JL13qA<~f#&p|rky-_|5f`XzPWoeKJ%;w~?yieO{ym?+a=Yd+F18PA%(t!@H z<36N=!;l~MBYzA%HPAf2sT(KOzlDg!nOd{1M?Tz#bmYTzgqb5!=e|9L3PFs=thj$u8kJhJszy8Z5g00jmg6NA z?i=|oI4G#IAt`RXt`32El!HR#0|H1#h;-ybdWaMdM!rju@v$Ztc98)qRpD2+9ai?0 zo~zz+y#k1VXB;Ad9ugu3WN{zD8c14c=1tC+NDKU|zW1!rXG-I%6lE12SM;Ae^h4vc z-u$S-Ua=!O?_)nysZ?4cekgW7u{~>DQG>Zv>@j0gMm!FEiwaBU-^RYE+3bmG&)>3q z?#xQ>Lu7&_4TQw4+%_u3EwWNAPXy2mQAX|Hy7>{e%y6VgeUjSGYUip>Yc+c$uV}4u zs8>;%yEI%mh+5QJmxR-u>9PtXsaH~zyx|gC2dTJ~@yVmNA9q)C2?u>AlJ(2LO0?JZJOcC>y{EYdtxd&m0eQ0Y25PRIT+%dXyW z(I-AKc3tH2OXtsKPIW$v)#}4w2aIs4^JP?eeH$6718relBi;JTbhJ|cs2^MzQU5rn zSq1+I4~kFW47btOD%vP&u!Fs#)*Z+`1?$PK@+rU%TBinc2|fi-gPKAJsu2EE_|OlC z)N?6JsnZWO-1mhX7WQQ*UpA81jY;5`eW%`lyr_?Lq^+~$oSMz!>!#B5m_BRQ6u$6wmdjlpD=gvB^?3}MvloG+?`U}Jyo!%Q z#DFZ229n5!bkqz&q$3~h3&YsBoVHGN;aK)I$3?pMuFz_1S|c&owMH~kp!y2T@bWlC zSDq++rkYF&v`18cmcuE-JO@GAN9~c`xL+|g2LW{VrnCs)5aNV5a72i7JU6@_^@RL* zZ&iLB2TSxY#taI}=AP{*YJz!&uVQv-0d*DVbkj^3d_p7eKmdpUIfOtKVVGVI*-&CD zjB3Z4H$*jyd1h;fhzj8?7zuC}Uk0m+_cgi26Ln39ORHKuZ~^ClUf|JaI8oeO{&bFv zi(~VKv`uSuOa#~(tf*|La`)o5`Pu5iYXoxT!RZt1J00$zI}FosMtz`udLxMfzP{A^ z@cvkzsH__k1GXcKYQejdy}cJP^u+o<;chuR{Cei}6K4pD$GL7_H?6^XxMKJN$F%Tr z@Ktl$&r+!L>sS=);|NpR_iJ5swA;$Lbq+aj95@gZg98yqKBR+l5h6V;tjqC%#E5Zh zqJJXRdzm*xQ!dwI*eYqP910)=T<1SW{5g=zV0 z|L;Yo>Q+gUYSbs9-I4E;(1znUh#m^Mq^q$y{ZwkJYpYf94a(dbXrXB|BE4P zl?;Yp)fbp(6KN9@@+&k%j534%YS(;wL+z*vi8P8pF_20N8*3@5_SwmpYCfJ9uY>Y} zj$CMK38xmOH4EpmPnwJ=jB-d9CJCu_1rk7X)Gh$29sB% zDRA{J1+U+Ftm6Xb%Jfq}GcBx#Dz1m9A|6B6OWqJz-vq0zh(^#&L>2-oFdkCCjb=G8}C)#0K#&1Xf7L{orssAMXq8S`}z^E|Ts0l?$t()E=NwOIICUZnB9m zVgO_f!`Ev5^|5Ri$%!I_{Apng1Y;^|%$Ms8t5n6mqPG;_3B|W=)Xy_X5W;?ThF-Cc?+sUrOa4xvcU;fJC#qC z6}m}7YEX${<>x6_Ju-b<$VpVt6o#N3w1yaj`4NxV6@fwJXTJm%MzT+&LQJBMM$_aO zQH|N+b?HQ1*9$=(^3`jZHT0QF^@BQ>Mui$PF~_H@eXEb$C8L&x64Jp%y-^El5GBr_QmTyzM@4&loZTxiO|D}hT`04v|mw^z(jl-5F5Mq37#q=mo%tqdXZ123c_ zj<7-8;?_rE$Ei3fgnnsd<7`&lTSAaaP3g>#%i=b)F;ERO(n915??ZYFHJu1zRbe%H z{k#I-zo}fZo)2BR;N=upO?ow)hayV+9!^|QYt2@d+TKD+Ldro|7&b&{nhh$gr50tE z*=E4|C^Kjb(|dzn7WvcK2=XIe7~0t}x*SE_$+7B9pH|;09k6y}-b?2DHDw*Qa?7pt zcP_7ZpDgE@zm~`GfzHjwY@FwRT}WK5DgMt?4hg9Y}1Jfjwf(YF2_~C)6-hJT4Q@j zL<5{d8(vo7|2@1Uz=tJWSWU1L|R@ftfjx6CeITg0m}bG(GH zt2M{lnP9!ztX2K9WcD2XVKHFRLbawyc+{(k%nlh%Qhb9yZIRzz+AV&vGs+mFixtG{ zOFgEeqtq$uhIzR#z}xQHb2CH$6dn;BEcmZ*Q zY3Wwa)6qpNYLbWst5#TV3)>c83k->w^Yh zfH=ambgS;>D;cPEyV9{Ta6@QDY2V8wFTWDoM<}kxxRTc^8Lo$+tMx-^h1Q-U>-hM0 z#N4Q0;dr6Jo!M`5uE?)gd$%`70opD23vrYfB?a~fksop74@29o%k5hJ845$le~=!!ViuHdWj5^{4@LY55wlBa_6u zpaW5#D&p;`_j-0m!cGb(HR=|nN8O?>5l0xN_Xd3f;&^^c_Gs&~#)tNDv1x-^>9&_w z#YiuCyF`ayl>43Z5mpxyM3C{zj#UHjD-QI6vrSyLPi?w>qZVqeEdUqL1ss4I@JI`R zLm0+z2`#g@anXn%uI{*k8Iy6bz_i!4A7U%78>KzO0&!pk?C`w$v`@&M@M<3h5mJ<4 z9+wh*s8+q(^;i?GeNYT8Ks*hBkQ)%cd}LHj)IK6BYL06!)l4FzEy+MhUx)7F%3u*=9)u`2=;@8>4k6N!5BDLobb0H@D+k&+1Qw#w z<%zRoMUEDDsVXc&Zz;9A6@9WIzJq#r|LVoO|N9`7cxUX*^dRlxeV)mnz>PL=Ff~Hl z4+2BN0gQR_Kcx7vt*Oq7pVk|T@lL|qjNWw@jl2pYsE!oh`&XYVveNmRq=(RY7ewsH z>lBN~#OoZSf#fh=4?3$I27?}rE%I{G4n3YDFj>&as(#%CIg!d#t_B655OGk7N=0Rc z6qVbh2BI2F8Zob+LmjQ?*<3|wC0Xsy+V|;Pdn~5Yg8xvou&9WGG{nRFcuq(<(j(J7 zaD#>0&1iG>iK+$LAzfW-s&U2QMwzeYy0FTvX0-cvNg|*t26rb^!ypG_q=m?b8n@)? z=wRcl7Ya%N7eeq{`ck@ce;V z+))ijP3}m71FZ|S4zj~I(h>S`T4fHE$hs|wGKY%c+GuoIOSuXSta$TdP*3?DHY^q5 zkgte?Dx@Qh>%EbzK{}WM*Fj|%#)kPl+19b zWyQp02?~7Z2aBmiddYjkv`)lNh(r8^It3YeMb)Bjr$E~I^Y{Qt)mZcY@A+G?vqY?7 zl7=B&<8X@#ozVs5C23JJuBqu&T?IV*RcDo4ngjxegM&lP1xMG*2hK)5+=u5Pj(njn zqMpR@c&fVNr`kUwT&)xPbiz`F&g^}DFGa3>=jL@d!j=`2KyU%(`+7c5?91eI zLQn}5Ks`{!b%a0z`6en)@SGZGdK*JWMj8?u-lT%o=#T_6GM?5KP)|azH?`(W6!ed& z7=lKGz!F#iYlKKgK46RMz#PvDvGai04$xZQ=Y?Q3LaU4uKli#CH^Al0b74R=Ln}6N z7WG>WVM1C^bDpv6dJ4YtJ)i6c_80U5y+r>=->28;$Mjz`ZVG0a;oZf6NJu#l0TKv- z9O7Y!=Y{EE{7U5uYZP&m8*$JV(v5hm7xpuQ{Ssj7R`YS&A?vmaL88iiy9QI$C(lC4 z8`i)L5^aC&$FUOoj)1jP^ttX_1eNq85{ily&iAfZdkjW&Q^}AZ{siX$gGbCeu%+v- z7MgK>Rg>{LBKAQ{Ok1nLv*zul*Yul}X0^G~Y%<%;PP6}ipN~VnLVTip*A#dNz>i_r zXg;(v+>dtH8)&!n;n!w@>+jT?;|p=u}m zJw`tGTdwh+*}>DB*Yo#V`5UykS9wcN{(ftLKaf)RD;}`mw3Z(bwSJjs%7=*B9_HU0 zyqN6cJSFJ_Z;gMBH+z4c7lv>+eI|c`mcR2lgTFO9lfMr;>#Iap%_f@7-;aIU<3w`? zh_24@?>~v=^7XfWmT2Cu5Y2y)KRvsf=otQlSJ3@^Oo^_*F8Yg_b#G- z{;2H0E~4xIoalx*L@RjILLEZW|3$O zKj(IS|Jn@yxLkK^C0fU2TlZT;>;IJK&N9(m|3-8-FATef@4I&?(S7{vjn5Hn;`eS| zL3DqKKVjQOwB>f9t+x?vyM;e7%VpjEDWV4%i--7`5AP#-$Gt>5T8ZA-$G_K!cJjTu zRuK(z`tFB`M!4L2___Ny?|#01fS>osuMr*mO6BheKRf&#f7SdK`mbUBxwoX2sB=NEb<4h)k*pO10Pt~%$H1bU30mOUyHj$@o=I9HaxP}Ia<4K47fVGGYJn!@ud zrpjsnwlO++9r|>>e+JFuonU+DDw<7iqdC0BVlIC-wMg@5K5K*v=^9$ZUr}Am-~C-m z*U~cnZf_rdQ+0r@r{#15t)Ls}CjP?eAl*#2(5M0Xx|hGxE8qLwO!xCgcI9io+xQE}+j#`@A$pkJK|AQ3^e)ykck!2bpP^r; zBlHRS1OD!^5X4I(Oq0p8cG=u?^zh#O!v+3W^|8^LS4QDr6yDsd>DLbo;N*IoESCpm z5(g@5M7aoJq+#w_P9_L@{{5`GJBIzwj zU}-j?W9@`;S7+WGB#i{|y+NEMDIfBk>is;Het;g~FGC;VuSFl>@66`RT)x)9|Acqm zLw`x%r616LXTMqO7$4`9B7K8X{ymn$iS96b-;4Au{`WWXkMF4Dj{jTja+tsRJ)&Ro z-p5FFL{v(6vZaNE&14)ccnND2txZ&zjH4AV;l?~APtB8YOu6xT=_V{;_$-HcyVU3+KU(I}od(@BG(0qfI8`WCb^u?iZMDAO& zc8GQEB5NQE%CE5cxxla$>aGtAg~dGyy>r6%|xe zuw%ilV6UjRxAnGx*b&>Kw}=YK?C*Of;Nx@O&-1+R=l$>fe102-*_qR>b6wZD+PTg$ zgkhK-%tZ{xln)tR7CBhAsf=MrPrPlLJhx-so2zTSMuis`#`5;`js^3QNz`PHu0(mt z^f}9>ebBsZEyH9y#4z`!%$V9Sg>QZAHIyB~6`z3@1%tHf@cksN;EcHomtOnBi3aq4 zIm4ty=3FqjhtIjL1%7 zQpjPvuS~wfFJMIOz2qO?KQg%) zb#nO2@DAL>$CDGeH*l}P)dv@CI|$d5AQxpyk+{Heg{D2<@Sl6BH7h`feaGgnh&Q%ywxR;VY(Kg)8 zC{IH@I+vT0F5YLnz~>wHa_f>Wkj|t;a+80MXOow34<=8WaF6TS$K9R$f?tdJ%aa%K zGVbGW<>R7rpz}@`FYMNfWdnUp+({!`eyP=*Wj zzT;;nf3{$CSg<0|=YL)b`oA7^uo7vRu(P|21W`4J`vqzw*oRK4Gji=|bVaGdGpuFZnHa`)Bk1OY$QVzh}U^B>B2Y zs}V`VTuWzhbZ)Hw=}yNsVSNw%?_fCY-Q+=T3*blx9Ds+GTPWOs)4b!^#1ZiH*FL$Y zK=bR8pK*^Rf2VB(%;(&^{%lYCqy3z{({k{phoAkQeV-fS&-=f$G24FE-N&DGD17JO z{BQ2(VENbkpYuCA&vP>9zq?Z$Sv~{2+q&mqKA*+QIU3S;6pnLumdXu0v%uBAj&<&t z2i`4*yX*ZceCM8NU5e{}yZ@Cgl#jkZJ&JeSxxYN08|(k=UBnz|-E(xK{h4z-d;fFp zm>cDLG50LJQGSjmETg(%5g^wPSkl1b?!ttzMrdS>Dyi3YK|UJxk%g8yDtf8H4%L z?@PNc4t>xv3%$?AMdyw_%{zm-pd+29K>Nbo?6U~Zs7v30HOz5&yyKmCN9&wxkG{;Y z@t(G&_@%g`cd;+(UWE6kW5Ph&p-=7vuKp-L%hUgQr#LY2ezrdCi=ZF&z3%rWKY!QV zZ^|>>xM6n!FIVH;-!LZD_FpekN1(5Nz0)y7^ht4Q;*R!Dbq&4K_jjM2JH7w^?K(RL zIuG_A-IpmBfH(0k*SWc#yVJTUMYv|8-pFoRoAmAr9N^+9uAI1-?lMY?vmcfr=C^c) zqu)ptE;Q|ai93r@dj0i-LXNixVoGXS539}YkYq(ocV@UU-C5Z=xgM|2AJBrKyl{R& zVNr2OX<2ziBpTD>iJp~J)xCPx^yyn$SKrXs)ZDM7|A2vm1`iq9I&Aoek!_>eM~@jh zZv2GvCU!8BCQq3uzrUJ@?*s|A7Y{ zeCXjv9)0ZbCk`HZ@~NkvdG@*IUwHAQmtQ&j>T9q6?Tx>`$-MRUJMX^tkM}?L@S~59 z9R1`N!}E_}%Ku<4!oq1|Ruh@z5-%wu6{IhzBdugO8BaRM60(NuAW8OR_Aq;tTg`3Z zwsJRfZ}3)r2tSTLkH5mR!SjpH>GSvkzC2&KFY2rE_43vD8hmSf`+T?i_WLb<)t})H z`1Aaw{t5o8{dWY|fDmv5lz=mk74QZM0!4v-fsVjbjnyR0A7Vmm$QqJD>7mR}PAC{E z4#h&fLUTeFh1Q2Qg|>#S4BZ^sA9^74MCefHh0trEzlS~yeVx}UuWw#m-o(7gdDHS{ z|HS@eIm5yqV;CQE6H$nV_((a4kXlkthLI6u0+~dXVqT}&moTpnFt7EP*Ja#Zp73_Q zm7l<0?zzbGPoL^@`F!TQBEHH$=e76W<~7!T`JeNWF)w$(b9P=615?a-`TmD_4gGUo zmxpc&-Sy|ZUd6mVz`UydoY&O6SwE4V_%kG#OhPgzyKq5sCI7(n8?Ik*{fsL;2_0oP z7^d^4uAjR8qwB}6AG!{9J<;`e*JE9qzS#9eX*Z*Py5ZBSKl|m=^`CwB*|(oA`3&pj zv-O`!pM^im`YiLAp{Qe6sVCTRypzVLrKyw)<55Wd0}T;i>#suVZD$ zd`Ih!)*kJ7R6iOyT6VPfXy~ZtsOzY7r0dAHM~)x)^2ldLJ~{ICkynqrc;u-gPab)M zVUFB$u<|pQ7W*c)2vx#|*Il}x0b0c#X^9%D�G9*_A{?AFERHplbFfO z<;)c3Rc0#lGIN-DgL#d4o%tJcoSDYF#r&Ollew3f&iu$+!Mw}7&Ah|RU`{aKF`JoL z%uHrBGnbjeoX_0ET)@m{<}nvC3z$XBLS`|ug!zV9$}DG=F)Nr=%u41_=4NIMvl>$2 zV&+@sd**S%5{~f1LIff*XP8csLQ+W@>A`d{2C)(wu`@~L6!RPNJM$j%Df1a~A9FwR z2XmVFocV%zg3Khd$ZRr)`I7mH%!RZ$#(c_VvSGJJ3j`;x?xs}<$Rx{h#-ppexd`C$9ooD}EvYZqa3x+sGyjOfh z{4ymirDw{ll=Ugsr`((JMoMSuq||+>pQL%xCZ@g8qqN86J>Kqd(wb$hv(B=vw?1R* zXzd*^ zR%ZR4JuLg$oDXuB=Dy&`^DOo3^Zex9;XUb# z`*!$Qe>{*EcuMQ5-4x`5S-~m6)xj@At3%)C4b3|e4u&rZznh<*zpH>Pm{#y=p;S1y zaBJb)MZThOMb{Snqd2E{WARHRY)RjeEhVp)W|WR8y`!wk{M%A?y1cD?t^P?9+w?%Q zr}-rP>sQ?GP>b~E-?Wy0|9hc-M*j)@cl_JmHv^gmY#PW6Z2vERI|uF@6dAN{(7wT* zfBV}tgdMVY$m>IEhTbyt>(u1bY&;ma|lbnLwPA73vO`6FN zGLlRqbIAg-nQS9hkZZ|}>PGEdntPbdmVcZdx(98eT)5= z{enHop5|CCjmzKyT$HQh`f+Vs2RD~n%x&Oya{IX_xTm=nxmUS2xVO0vxzBhPpUVdj zk@mY}(I1vw&Ztv#IYrHxh$ftZsOh36hBaq2;S!u-XH;}bmW-%RO}L`ECI<9yJerQ` zE)hkzJ0oc4GTS8Z7}cXr4e!MCs55{PRHV&N9@Z1#gpU8C=sM~WHQE@xxY9KzAVE0_ zP&I6h?h-UQg6@ja7U^hV4uw~-sEb}E$YFq*S46^ShVIfGM8g|22x zXAfOHeF=)=`Ljt-!K z9uxghG^770^;|KaRW~UEL?xW~105;0F|JPgrM+VkCLI!Ci_=9h>WoMIdKgrq#ohm0 za=OdnPv~XDAFxElFc3=F!o29Xpf#Nf21SSI=9|Ahp$1x*$fvV1AppTNC+I{UU3#GV z9c2^~%YiZw62O2-0F;Iq0^n}^nSg5+%uox2;}(-uFtP@|)tnk&p=2;SG5hSUR!!2u z<^ZpZL_%m%O`HOnCn&Dy9QC-#m=rZ8fU!PA$LmBj5lnEq0ZmYxb=S$$%mOSeQ29TH z#0oIW|D1>k3V7SRpIsz$X@t|eM?&MM9;0J)4}2C5Obd(%qUoRyKm%EHFUSv!MD;v? z28Kc>7N>)RdAy^fjnI|Thky?BOCm(OM*Fbn>?XCj?yvyRiUNeyos7)5-FG2;O<_%u2yxFBWY!)ol~qf~W}I8h4Q> z+T5z!s<=g~;8rS%1>EVUSTMh33O3QEdcCby>%7H+V6*1+w`OFBqNv~!tg0Zm1*;X` zY$&V|gmp_6P7@Sby7T%9UuZKR5ZIqp<3~5Sz$W-At=2nj>P|s1_K~rNTa(QFPaD$? zdC9%2$R_lJ?rgS(l~d3q?GPWUFra^fFw0v|Ca%O|=e6w1qvqXri{QO$jty1Rd8oqn z5Cor9wFwFy0vaYJ3c{JKHn%W}jHkE|Yy&bf1nZn_UD3(Eva%#T;sKcGBZO$o^=3*98VF1I|dZ~Cb&U5N+XIW;~Oj9?N(LQsa^mR3EFU2>p+R%|zOd9iXyOUt;Yr7^ovDe@n&H$s;cCl*6Y2-Ua`gtR&{TFRu#POs%|$KM@ezt9d4V~ zJ0SEfdKDuTJ;5;S1$ZwU#4~IC?RKj*Wpp0KEHcLz#*C$fsyJYZ+h*HtGZwv&N7iMc zagBFMS}Fjuy1f+nSOG5!;(Zi5R?s+Q75ZKA=TCssJWUJftG1pp3Oa>6=W6L(Svb#_v}$$f3}$1 zJa?=>HXZag8Hg|1Uv#n<0b$0+fKuwGAX+&eU zRV>1a7+y}{!-Y;%<2mENdW?=LkQd@A_JS2Xz;D7sblL92TJ&Nypt2x547#TDFeicu z6azuvDcTIzDq$U!8s^FXUPNI4ssLDZo(*)TRU%d(NL62} z8r6`h7Eu_jtZc=UfB?aU{^?{yYeWcz)WxG99o%apf~qJ*Kro$)*PG{U^a{nQm{|<2 z8D$ekVQJY+XhdJg+u-wgAz2y(TgWRmU^umyViRkGPR535+Z;T`u!;kOo;K0zwqmWI z%!Yb3HS|@_K4Jsd(I)oC4Dzs|!90LdRNNs|>?sN%n^=PeN(3e7#;pKuU`bWo#pv7| z2LfJE%w?tG4QN%9CkQ>Vy(+L@L&cor0DeW3;1ddUQwD4G&yQ7@9|7hDSalBD?TGp& zzeIk=@BA%{!eC!Yjkug#06I6I!*B?2VLhg1I0dSd^TM%d)kEyrc6*M!AmX<*9CGJH zBJR2j{+5a1uHP%us|K+ntJG-czRHfJj`Z|~Q5$Lh*pI)NjNo5j>}LNNP_|~@Vd%}c zD@vc~PiSybR5{fxXW75aDiLHkMN8567rg0zi?+os3i%3CGIHN9>Xo`H=$Fwq zm;3{}-!|+mwJ?Ji?5vO|V5Jrsls9a#2(*7J;o@8gSeIHr%y6ZPP8J#%c40WcVN!Xz zO2>`E$fEaHA`Z(ShRZZ}V8Br8IvR)CCUZ(_YP=2y{yC)H!?LB7gXTD-sk-B?Ea!IX zF0#a(GmsZ1iG!}!)ANnS@)WE{WESMs6cl^la5@8gnV z_W9$DgAT_yX^yv4%k~=k9o{kr8R0d~_evi7U?L83c%@QDcB?z&(6Z-9j=LmuR#Wq# zG^cb>V6>Og&(b9)zi02}pJ39MdSGEBgUBSI26qt&U`DWDovwH_alzgK4$6q=d{Ltj*fa1KVEMrKgW1Dnz_fuSn+T|)467&7lhV_gkykL2OK)wwri zWH{^&M{aR(Q8n&=6dKPI7vsyKD$V7{t#Y_)raA1nsBrn>A$RU5=w0k^*th>`uUZx# zvZ`}PRyfV?a9CL>o2`g3hAY%^S1~?&4SJaFsNMA$*YR{*m-1IX7Fr5Qog?+qt*N0VsQtxaKouuav zA3M&-8#udm-($t)&bAd9`Ht(@_?=ryOSkl%)7x7b&pV&L+2tzj9I%o1E~kt__GflZ zKXm7i(Z=_|QKZ&5I&j#u*-Jip*>PD*Y3;3ESH@R^Jj;njd6^NDr?^k~!+^P*>3Noh zBIu;Rlyp%VfqAJP7=TG{3HBxl%CIhH256(l!kWOPTjGf_5+E&SP7v?JTD^8iMkGFZ z%!&B$tAoZb##8TapIuV2J-#})svoi4z`r=UnLk{6$Lw9bm5aV6#Tm_)D-Gj=)pc=w zdwtdD6SL1O8QlEx-sTtI*Zl(qbS18g)aRd|IaLnis=UqLi}lQ?|8+G_o}Q9Jg+orE zi@!H7^Y`A<`|K*tkJ2{DXUUuVamGg1sEe23;X2c`pe&IEp`ONbq`1=9!XMXvH9pae ziTCJQ7dcAO8It@q`2-n?b}GhVWC?DW9`6J1Gr?-HGNNNc!#JtLpU(>_+K9FN6#lsJ zopFni_i0?4_v|V~8AE=}&Q^4~PAJ?NWFqzubI=#$K{S!!g0P@IPr4X3y@x#NE5x4k zX1lCaF+m*X2)=R+(IB!uChN~^evH$_p|Z#J-WnD69E0?{X;fss)l2D?~Y+#3D>laGgp zD!QP{;azGX$Oy{#srv}xONm$rvN;@Z5KM=B-nmB8T8341PFLim>~qoSZN~T0WqH~D zpUE?AM-y!W*2NmP@&}Ap1F@s&y(cs(e~-zYgoKZ^r%ZZvXix5cO5P%8WxZkI^WkI@ zSI0le_$famEG}O3L0%&~g0VtDIOQ}TqsLWq{lt>}$e6hCLlbc?`J=k71m_xG=oT*V95)|)riiK<ziuh@mb}^vtn67pR(~?Ehk;?5_fZ1*O#uc@nvPkn8n);X1caz#%15gUrF0{ z*as)cdy~DnKH%>H%0s}l1}loL>4-BF<#UNdLl7}K`8=2uP%0c2RP+{B8`j(S`?37= z^n6m$i==cNdH&)8?6+FC^lo1N{oMTCC~&6phlk}CIGy><{B-AX=eQH;#+SyMt*r}7 zCw>0fwANmik(6oq>5PSO7+Wow{P#>2lgC7uYMNO}*BW-?ozUezHA0ma7=VgUH3&52 zQY}su0!eovlwqp00D(0`V|8)S42i_q(F0SAFFwhgGGYV?ltiXJ=F=p3s`1Vl>}$PZ zYt1ec+B<(8l>KsL{}f%x<}1mUz2pXK$i+TTG;CZ-apQ%ZUDjZEZJU`j4gib6_x{2PER0O}6G7gZPjf?H z^M9P&!2iPE$9O51lPCh9kc<%*4Q9f3rfL?Zu&P^pvXsCaV0XYuk~A{(7@0Jn=hbz6 zdye1r@s-AstH(ca=r-ewao~xGdgyBP>QDLmZqEz#Ul+RC@`G{g#sSKR)WM`uRlzGf z+O1u;bmXcULX{1Rl%F7(!PO)08VU}UgcAlS#0vdJV}bFH#zihmf28;Fw)%dp!{*j< z_5SwW+p@0TX5@L#Ygv8EUfKtf^pRQIX!M0Xq1CZk1L(s1f@KQB2-H~9v24vpWa;V! zQ=^T&r!FhEa-$zk-*z-^41IW8_-L=2)!uy<0VlsR>&WxmG)87JfRl)mGmU?&dr%8D zBc+*67W8^BJwLy6!%$unVUA$=uFQ~uTK9dy%$;W*@99C`feP*!~DS>U_ZkA z&>xi~qF>BI7fx!=;Q=Bn+Ct6*0!GW?K!9AV9-i!J>3S#_Jj#AlQeT{LL9y|@)7hHd zS}>HVfaJT(W#n$|0>*026kQ?NMHuu-6!yPqW9l(vghWP6*g9C96ee=@{m(BN z+;O?J<>Srj@VNb!!5#dY#>d9t`rFgPgP$JOTs!mpMd{(0k*dKY*x5Mf9}6c@V~f%@ z#nJ<7q@EcpVuR0RJF?T#kKBCyqnKI@#QV(q_tL(j9tEzEw!-lFC=_LrrYnU z*xON2Ui%AKSZ~adr3*h;MK!jZUQ$teay;4MBexrsYj5c*&!6NoX$?L!xGw$yrVyt9 z7znCtI$=LuBS%m}}Hd!RlrMIIiIq(v}K?vMiPy|T2i zGs#vOC*e)}ZhXpp4jSYDsyNe&>5mMjiIfw0IUR&B_ikWK03cT)ta1=s2;Vz_f0r|; zC-NYOIT=t}sL%>&gc2|=W>DLM&I^8ZJYjmRzyIZ{C$4o#KYwF)ZUg9u~(MUi=beZlCmO8 z$jq{5*}TU06*eg(D?I;1o{_x0&u?{4Uh4Pz>p)^@J?SF({)z?bDk2N~(z?p3bqo9k zXDpIr=^wu9`*%%A?VakKnVN-=w@tRCrdlK+HOrG?>ya)?tgSe!XEuB;WDjxc`IFT9 zViJ^cP7NDS8Z3o}>O1-NaF%5f~iQ5@8XvphjyVqDM&m$dLz^EP4I9 zsUrgoBO3yNK%(s0k@wFJ1ftQs+kb$aH!`r`h9``tmTGrKNAf2F)F6bZ=+0{L1&rlb z{f;GLA>pS*1I88-fxXyZ?Dn&@4a9OBwjj$0$u3?5EnQ3y6a;i2A_c)82824Nnk2Ef zWWNfRoyE!y_y0+j2}3pqHhkW7U7%qX?AOg0XSz&chZJS$u4rQe|AZ_rH$IL2WH5no z*{H6tu_b|k);6;9HA3Xg)5v07rg0Y7qKt+6o_`R01RQ0^rWS@34GhKU_GI^NBDOcp zg$o?Hq;C0Uh7e@RF6#M+O zy``0ZFk<}JxBMOXxm$A5GxKX}JB&jW6&Fj!Y%D?&y~6k`ZQ#5XrrrluTFZmt!Bz(^bJY;~YqArA?5)i?rz*C`JDQGmbVp{jW zD+M2ZzK@ea0PI^m~4F8)G=?=sF)m+wQ{HS_C2xX!&}JjYcLpMe1N2CilO%?lr>F57dp2-sOwEs?ouDAlUvCjqf!_CZ2~$Bq{Y4>{=r?y z@M1y~Xlz4*CxK!Jy0Dt?pBMn$`Pw^lw{mfN~EjLV6&1+?dD=u7IG z8Ob(Hllfx-c{;o|x$%(Fu+*3$N9DF?Lt7xwfJO}pQI-;2#v@%WCR_&h7W`0+(wm~t z$@;0%i^HK25wVeB?!r2u5hVf)ADzzsgnG{$D~&pXTY zu3vRSkcsQwnf>b;e+8aj@zAd)J z_<@YM?v~;}BXDGEdn1M~jhd#k$xElpZL!@;V}Uj~8kLu7In>^61TwXyyHSU{)ey7h z2f(wpRv4D7#x@x^ObiR;Tgmh|c1mSA_GAp0UkWDDw1OPEm;#NYo%mNq4@-X>M zGKk#U2bh@2xvm5c9AgHE*P46nRBwe`z-*YVeX4v#KlK&isL3&N?Z>HWNo7Qss^R%f z?PP7!JBp@gvId4-LBw_}xa=%2qoQbCPa4OoOZ#3tQ;R7x$Jz_I-2>yb!Co2I`g=f@ z(+27KjWcDX9n!&=3ocfarkU+cT6@!%=*~X2)}?S6eF_zQ=9!SYw0~d>0`t@-Qj+T} zm+@~wqHkb!Fnf^Q2)aSJW{B9o2l^ev6$a>9*n!}sP(~27MC>c>g7`OMSFqmHsH%zZ zo3MY5PSHbe4o(Tk?i&lX{3xQ(>5vT!x_6`d>7D?Cb)gvJ*^EDzE}cWqYuM=XxM%SRew=Inu;yv*Lvuk#uj~7^k{8 z5NnL_3+1X3WvLuXy;wUIYozN4F_gx%<4c!OGE)|kV{MI%`F(18(yx?@va)3P*o+xc zESBTa6ge7`XP0rm+a=zc?r@|l*<^;t?RKQOwcU1!^C8%3L)wOm^&uZqAl5W&{-$8> zg$P-e)Xe&)_H4yIjdv2y#`Sc<709!ShW#c3LfivM3(n2lN+NHn#R^J zgg=`xV`;3-xTBH$G~rC=al8*SVhuFNfy^OMdH>Qj_Bpg{JM)Z|pXNx(PFKVh<>SDGo4p=Ll^z%TifL&a z$JsbRNaZ+5;8yvHIa`iPtyfPM>Do#~AGdqpa4etZp($CfD=V7y_F(Wb1)3nJ=$f|j z>kY;DUo&QITM4`dvDllHl|i1%%P$Z-8D%?C>-e;k+T2Wknxf`d_-k{-tQ@-~Gfh>< zHMyBNYDSOV7g;6FquTAN?B=Rd`IHnM9}Ayik$GM&y>i}Ep0_S2Xj?Fzy$b7igi$-l z_+B9!$el_ct>A^$-J<{S6xMnG@Yl&1FfW~eWP{rx@M zI#W8XuS^?oXHVmBYF%pJ!nB-h0i)Yf(*ReFKNa6IEi#vulO?99Dd}=nsy{EyE@r9F z3M`Y8e3qZc-vv($B8(uv3kh3TZAk^mU3c7ib$5 z6%{czm|O_@@#K>_Vq0D3mogDWl(N=cejXQ`G;>aL$?)5a&8z)Ri{$tJeQb7E&dZT% zJj+4$Xfl)uaPVEC{F@}06&CkOZ;rK0@gN0)9a_7nz5^$G7w5C3SS zP4-K2og@vH^ontfm-%ls_L3dDR?963|FD}wFa@dqeVXshzl|sYwY)OGK7KDvEd}%`KrA#3hHd2dhmWITAdwdU7%_{@(gJX?Uc5=aeZUWMl2LDd%O>U9)h(C5v8J zv~byuol`bMb`GR=wk$`Ci>BOeyt#MsMHzls8WE|#eM<19I@q(wBfp(g7_R0W8$bMR z@`j(c=U9v!qozJ`X$g?ZCnw@`^3TE@hze@h-y4gqBJcy^2-L=smslMNWuonB3U&fdT+k;o$K;GQ6+k>r3MuKFRn|^7(vy-!3ojy0owF^Ku^zg3y;V z6oUW#^W>o&<&wnzI5cb2+j!$!nek7_k1z1H=9fQLZoE-(MQgcKjxHu? zkjha1*82XpDc&OGVs)q|IR0oN)c^cD_ddMdZ4D`*{)yN4?>v)xD&uZ5JadRsVXOOx z5=tmEuD^2TIqu1eRkeSA^-p^87eJ2CJtj&o*q7#iq6;A{3nFq*wiGNgWz>a;i5P`B z=F$Xbx48xHoGN^(e6e!#DzX-X>uD3Mp8AVp2oKQ0hl;j+GzV| z(Z$s2dAl*HErpz&Mf?i8ClG)={bcet-fiJ=bP&1;fetMkq7DP~^qd^rs0=F1A$9VA z%&>;YC?3-x*%L06rKTnxVMNlvZH6;q3ihZCD?iA+F`TLR!R_gj5J+A4&9`pBjs+)$Uw2!v%qm{Uv;hhzPHEqNKH?Oo+DWjp0 zFB`qb_zgBiN$;l7x5uU&2M5 zbVv}GX)qN5KT&BuESz{sD3D2{^MH71)?Zi|eD5aBSY~_~JT>+*KukYCpAw6AWi*q! zj?2w*vvF^5v@$vf83|DMGx%Qy7;<^n{l+rj_xWZxLb`t0WnVY3=ab3sj^esLd)8TyeJ>7ZF#c4{v)%&2kwX5_?;L|ZFb`j{$rFf~@8E_5&&3$h z2PqDV7ZDp+Gl(bo>F${y)>aYr2e2Qc(@ka?Zv=L6cy*3FCuhVYd+zYUt@fLZcW2d% zCd$^;TjzDyW2L)R{B-HAt#>?M$PNE=M4@pY$6h#M>(<1Y>+UGL!`@W5daDt%E6X-1 z>Gu~jnKo<}w-o!mIp&TpwYQ^jM0pXR%Y!g5C(I#kgeQW?4J-+S_VE6b>F_yMZomF~ zN9CuT#U6*fmmQz>86?qB+Z6CzwfB`){_utwe_w2zdccUM&tI%wc!0Pa)#SpuVO=?A zJT1g(fA$B+-WTx8Js)%PV4-502eKEU7s3_eQq?7NpFGHbPD3aIh7D{1#F3%(3Suu4yw1^}5r|C%4F!T)QeiPDhZTRsCBp~~$2sU+1Qb+P zB!R?sWbU!owg4Fp$KKon)z$XBxz*LiH;zVP-__x{C6HEDwxPOu@$?%5)gH$+{EO{f zd)qhAq_M@hj$od&zGjNuegW~elQmb8NxA)tji$xzCDq0u&*IU=(`VcYe&mopw3uHD z9OVK^YJXtA6)A%Va|duB9kv~HQ?c}dG>K19p`K!4?0%aX3`r3!#JeQ6xNh*e#e;^9 zn?9@V@;aNbV_aLu4aXj>h`YRVka-4g#m=|`od(p<-fUP!Jrz zkU#*(U!;L$v3H8?3$lTcMV!mS~IScL`&z~Xn0@Q za^2&eEe^+HUmH)AKE`!SY#$N#jYW0GtO>W)BCoQvcd4iROLU5aI)`J>Y{$@@r59Kw z?>O)9h2GiUW_t!2PkA@0!3`Zf)4S>`CrXaeaa+#O;}q}V-)DM5?_lQ^j7iT-=~%8D zHJYi3D#BbrT4OjNA@x&+YYM?a4H&vdoInZHnBn-+7d(#Ig_B~lN@~*_p4-6FCB`M5 z0b>E=D?66;atwUvVF#W(4n7M-n@Upo_Xilu#x5HWO)dG^C`Fnl=F`jLC@JY(a>+G> zFEHLQ9v)lbK_jp9_P_z3&m4eGdugWJ`4O?}C$Lva<)3LXBB;apT@feZcnNA^gDQUP z#bJLA%CdX!kp553pLk!t@Y}t8$H?lweSFfq3(EUuW_G<}eAajScG&BVoABQSKK_X{ zwbi{^MjH2x&NV*hbT~#ASGR^ndpr&je*ja!$5ZS90$s@uj4^zj35$fRESeQify~JU z>==&2$?DlSFTI}Gf|J)b;w<(Z%me1R<<9^$U3Zjepd_%Vi<3i;8gwZDae`I>~gA9!g4&D z!f`CirAizbO6wawyVIiJxDAK0K9;o9RO5cuf)AI*vT6Jwizr%3cnd2UKd_=jva+e3 zn_9Pu7K?bJFPPg{+}bhwh(v~ipwbl&7=}MyT~Utj7fVaiuvOM^@v=90HFaVtcR+My z@S@7*R3I9aB3dnUa4juW6z!c)+oe>Cr3W&9L!z20rYb{t2W*s!s{l|jT#t_=>3iC; zZT`cy)}cF$>*pI?LpzKj0Q+`p`NhwbBbUV2>ML(0zy4Fo96g_CbVL?54Aw$Q@;`9A z-h%V^%dmo}0-~XO=oso~fHnd6P);g*Oy7#y;!$VBG|FM#L_lQf^-(^+I){XVM2n__ zr*yC(?E@W#Ev6V`p;6Hnl?F7&8OlpFgfwNoQ?LbSJ$P(}&gBy-YzSL&?-fom@Ng5i~f>d&IH?_afyR%3F|4-lm)L zx|#Gy)7>+wiL64R;0=}Y(fA$YyyjhnJ*f#Zsjw)={vRU>n{u}GG*0cF)Yi~=VIh+K zC)w?fju80=_Br}f*-=vL>1`bLEOAutqmm>&?0B$*WRvV;cv#}_^g0U{e0A0S*Z`di%O-ZOu5ZjDk z2Jllh?K>UDTD$S+2>=pHO9gDBaP#2mD{#lPSnB1$-7=0^>EM4Ds0ow zjr+Jg=r=?y9TmPMHI2butRIF$)Z9u07lD-K)*4Q8!rFp~j$j*(>I%+663AUP?4@PJ z1;)o8I<6=#vbP+{UAM5Gb@Zs>QPb?HWZo6NHGN%E9sQo<){JtDI<;+*Jt0mjEt`*sgsO^Bg9@21*3fXKf1jK;8JP0yi&)kuQ=81!~Ad+GA=mvfb#{I@lwzj+;l z202Psd{eW$&tdQQ-zPH0c#5tXvHG^Q8!Y;qm>n+32B2l`p#vvDvf-xZ|q7QY5Vy$uH0@fKy{nFFuNef*qTkY=h!<6j34Zz>|1wLjy=1uv0%7S++i(~xNh>H^*RkhJY&k}VH16xL7xu*QmSaX5l*R#j zM@}Kdqb=!5-pF6epF*rdLM$U6GN~t11G%&Tzjbsob31c4coXA8N>bz1RD9R~`|oXD zdMt^0VtTCG+=T;1O<(hWn7i}{1NAbUBCHsg(E&s>&G<7FWtb0Eu1m*I)P;v~q5`JWny2i@Fl@sgQhiQmzKJt5k&>|*CiYPVA3kYiIN zmJFe$D>-g=3W~LdVc}ELQi-Hynvjz_(ms;pkQ+w! zgD9ckIqa*~Ru+*PjK?dR-^=Y4&+K|L%_3XScJlZjfu!CP@Hi2sgDiQRS;>g7KFmSJlqwUBT2)B zkgDdQPe9z#vA$7UAj@OGnw&VO>ZmtJ6N7cuJzJ`wQ6glwRUg@!4 z%-!FwdYYrShw34<#joe)+KU6m^|@=tbzYR6osE+{Y)i{WINyhJeK`CCJH)uxSkAr6 z^@3ihMb7Ck_>eUF84499sp3K&0k!<-HW-cI&;tN;IZ|zd@E-UQQpeOe5fvEYpm7LT zJ$`mO0{&YBd*&2Lz-pLGuEDWO?_C4uNF&2uGP$($Qb(W&wzM=}lE&Zl_?>svI7Z}` zji2Aov2^v|6&~(2Ovo!#*I(VzbyXKD61mUXYGnhjB^%=7Zziu~M%z$m{WZ zHn|jsJt1-(-Ze8ePavV1!Hzjz(%|skN;cMb=NN|^gU3HD1soo?C*OD^Ki%SX7%m*N zqA}zbl0R^*{6_FhDR!G-3)3}A*Bsz?Mx1zp6haml%>zV&TAaYWhZRiaDPs0W2y+s| zK#~#SL7p0=v4S$0;)dE`KL zPIl1Td0Q;o^{21i@z=v2iUN9hu)JNS*cFG)T`F0j^mL(`aR!LUfu#VjsqUpZ~jb}gL4NeVmNiC_M^cM1eZcoy+Gj8Fi3_|g`&Wz z6vUg-{dtI$!8p=rraDX&8Ws!35d+8BY5c$m_q%_J-_pO{a0~m(BgS##H)NcYlRcn`(6ukw$`2CW1pR1S zY6N62B=4VV>72l%dMm7} zXCrrl&edgl%sA%UgG51lpf*#(~4(jHtAZW zcSWqpctFN})J!=zQyCT9Iy2v|OG{HlLY*oRyg z1mbDXAgqi24D+|FkSc*xeF!LO;vt!WQe6klX$`DnC_8zl#jt!qhMnEaIY z3D0023T%ot@HK<*IzYC;Xb`9iLxZI>0}R^GJU;`S05(O9PzDqrHSsV-9RA_34};(; zWo-}%273a(Z2*r)gkgdRP##YI!@~$S>)L2mBE{|cz{Q6yR&;G<$vQb0C22@n1s7pE zzFi|l=iA%$U~rojJJ5d%a&K2??JJ(s0@xO$6@to*PV!O{O}Y8kl5|UxF0?FdXbXiFN=vZ? zWG$4PvdiLzvM4HoAc7mpsG#CLisCkkyEB81I_|sUGCDfq27Ud1&TDaSzBAwN?@wv^ z^0s^Kx#!+{?z!iDPDvu()5C7AM-4`(A1SIdge^YtaT)U#)lco$so#p%RA1C_SeaC` zSn2slo!qze&cl;xmU6B2?6C{_ww7J}`3WGzEvCMNvGg0kERMzaW7B4FqK2#;evkvr z86zeSl@H7(hnF;#LUTN3c32;93L_BhPGR|n`4Mil^J=rtuFI@r50?fqFP!ObWv=C= zt)<03A!cQ!wbWX|I%n3l`u+amXzk6Ve*ZZc;@FHyS2?R&XBwJjv@N%Gsn;pBBdmQH z>MnLWyM(z)F7(%yz}070X-VmiR~5ImvZEW1U8vEm6Zn4NA<$i*`A7{6kTJx$YJNsA zz|n*dgDy&wfgjOM&i%m}2(0yX`4+#Wif9+5LWo=c%Q4yyd>NzJnc| z;Tu;!!@cC;=z{7c^KQs@Uz*>xL3pUm{8WYc0rk-b%wI<|E9ip95vJv+jaHVXS6vsArgnWS!c%&cA?o;1YcsS|A@ zJKkD(m)&)j#WE>8$8uxe?(We^dvJu+vH^1NqmU&02}mjAbr!?u(}2|g_WsaWH4VPJ z>{pV(*+5M656B60C2%(Aq+JEVb3?z4>0`8S3A+HzO2DXsrjOMZ{Luu?;yLO$TyR4p zTWj`w?)DyDevSG`$F$*#S^4KHIyyX_?l&*wB#+snzFaVB`-KN++mK_g^76ATuH&`{ShZ(fEo%#jkZd>`YOcXxk`(kZ5qgNmUHxlxJsR z%P<=ouBaxKov0=YL6fE0!B0|dYFb^*c2;MG8&X2ovHjj1!#tS>x(Xh+&!JDUPilXJ zsYf2!_9Pn>>0Avn^6^!RnPF2;=L?4{J zPWI?Tm@V(@@XUVUa`wL2<55pp%pKb=-1Mr)92IUT@PwC#Jq63v14qs3li}rP9sROp zQx|kR;wjiBcsdHqRd;T$!c$LB+gwue$eq|09H_@f{C)bJnm$*qMS_=7v@zm0pkdQ^ zkJ&;beU5%L^E3t_^z59N6`O}JOH-$Uks1v{It~WRtQfoI>8c^?)lUjr+ZrnCJ?7Y1 zAE{VAR}L3vn$Oy@=_mIwMC>L%_@Fz#Vdi9J-`8HTt$^j}cdFl8jx}w~|L`B}?t05M zpE|7JTs0!3dEB1L`n}thE@c_!%FEh4`H@!7Qgw*g-445T`!T8qga7Tt+y{IsRz#4j zK@SH6K*602BP6XHuyHqFJI6jiDUeJCl@A0!z3N@xTUps&QJ_9I?yn2aYA|~)d9Hm* z!?5e!^@K%?-%OUlx4QQ&NQq4p$=oVLxQu4UYnnbol1QoNy~Jg!I4(Liq!}LolFY(;=HyJnZ?iJogL8k$ouHtU6*zm^TlP_V2V=x+a3Ho|iMRrV7Vhcepo*@_m=3N>P`TG2i2mV#B6 zAKte)v>RpL|5F)%&^+YPsv+uK>JASIA9ZKs#vv~z`ofIu{=CX#UIh;_Z9e;fPsh4W zhNTMTD+W+bAUHr?DBSyB6p$i8CM8oz%mD=2z3siuCxqfhVg*0fh>wlsNvp%&*cHl%!uw$5KDBD+9Gcq>$9M-us z^BmP-sad<_ylzY3>L0(_&xA3rF+n|5F06DmUucgtb~u~QN$NI_Q@>a58B$<2kGdlq zT~PH`HqY|aYwA}(9a@1#!hQ;TjEkW|>&cUiI*_JkVu22S0#eWr_@*h5P_#(P#3T@B zS9iWry_?O0&DuNW!@{hzzPVl1yVqc=%)5KoB(GQatkd#pjrac4WreH9sDH3@byj=f zND#RNTWOJ}3&%{Qa>f2%_!8mIgXL1O<-F!G{J)g@X~nVjuh?8Fx9I~acR^?MuGLiT zu3;TsFaM3V`naV_Jj-m#W4hADpO-Y>GC_UKx_R?_ zz~5ANRl~L8oChM1ON|!wcf#-C2O(qaoS>FJLmS6rk1e%w^6>%e(Z99;?S=t20&NQr zQy;nfVxWa0I2dlz`*|Y2y6ei>Fo(0t*~vsk-_+00`LC#7`SZu7dnm$5@s z9&sj{%M)fc>)zV0%HxYlebdZqCz#pZX>K!I7YF+M8~i-sHSpIxfT=Q|WMMvcLZ>#+ zOo1gH0dSNJ=+i)YfQt|NICILI)}gl#W5Sgd->?J!u)aTc=uivmc=OO(`_w(weZp%~ ztZxprzM$Uv^U9oI&%ar+=k_VXu*-1!I`y`NZ%$qEAR8ES3>fbwX}%@t)SG_Q@i1MT zK>#*U<8bPNGZ&O_c0+HD7U(E7H|y_bLV3 zLI0|Ic^OPqgC(QltG_X)-=1-O?v>+DY_WKU?EA?4>hvK)=InoIhkB=Zr?Btw%=+`s zFOs863)RQJToN4e*vp06_fJ~!@Mm*gI&1b5kuOR2z<%sdw*Ulyl zid+u<_v*8W@xtaTFFTS`+)y>~_V3r6Bl)U!zg_V1*_E@#iPzqKxq4fVaE+&Zp}lfq zb?&TltsZyf6!oQ|Yu5WF9(lI#(rYK0zaOXm(D>}c8IPDh8I*(X`A+0TQ{Y8Ms2~4g zK@mqh`3(=CDv{bpsuDYJ@={A?&Gmn?yfF*=bMp?ovP=D)Wf!s@c(J=Tdb0Xmm!*s! z`|Z-~>Zgtu?d;iXY0h}{yPj7%=NyZC%l;7Ydc8dnZ)EjAoAe1q!Z(=v3N)mI0a4UL zEyI2jB7(3cFNY5(7Ea_qN6V8K<{?->$qWo{9QX9&mTF8Q{gKXV8Z1qCuL^gj176qo z>U(!=Jv0)h_;ic#O~P`#+H&9MzO9kO!_{nbb=WebKeFosuXo6*RrkU$j+cD5YZus} zjrD!d?eEoS(vh(lgD#9j|2IWkd#QY9t9owu$8g84gk@BB*z+Uo<2QI*%~gjlyZAuD zJgS@j#$5G+rBuCYRNuAX+6SuG@T#z->c{Z*w>+N8Wy|jLaOoBVBMs2;1OKCN0`dwI zfi?;bN~;#DdIo=JmlOO+)tYFKMQIF@&HnOkvg=0I&g{k6>ZMN(iLBmN=Us)Z*L$&l zI=UK68olZ8o+mAe+Kh@&EQIa?E1O=zwoIxhY{|)FPlk#chjggQ?8De?@W7mH_BR!V~zSTeC3-c$y=4tN!u#erf;J@FICiNyHU_}v)g&GlvOb283Om;;81s`;S@n3sPi zV!1>ep{^-3-@!c}nVP%MoH07snn+IEl$2Sp*;lmZzvIs)yR9Qokgi{1#MD175!Gaqay%~cL|0n8Ox8O6%WFcs3S zrnYw&V())DXJlY&L#eW6qca%2J!;!B(w{YPZgh>3o@Sf)=-$<~wo8jfHD;D=wJCqx zy0zn5)ms*9F0ip!M?ewYXufjf>ge9!qX-m7$jd}!WV7*)trb(JO&mEc=*(Ph>&qNmEB$zAfI*8*-mn=p?)!Ktw%M&UaM9;JGR$}{p3_c z8>d*1(;gIVF~Hdsa+X7@DM%bIqEp$UdaS9~{E?Sl^Jv-JviZe#u=27swz9z7k&123 zims8ibEUF16Z0c&+tlZ_BFG)|u3Yu1zyBTyl8PDxPo=PRC_%4cr9o0`5}oA?_&mEOH$D1z8S0;lAKb z@FHK#kK8}Ezy{s_M2dL zgI5m?Bu+bz*HZ!Xl28ufJn#?(8%QRWGcZ`wyn;ua@b1Ce(AfwN0%dAb3OR*92oOh! z&rTM5dyJJK( zz71~lVt4}q_@Y&r_7(6mpwVNan4It6jEB21V1gL>ABXJ(TgiZRiL&`)>Fyu3U9kjIwI}CfG5>SBXH%aA09+uX;IaQ-93>on4bm#$ccKA7Xvlz_ceqjG7#0vD$8ex) zYoh?L3L$bqG=vj%hF`I^X+V(;$hv{n0riB3mI+s)kd`L_Gb`PXr#0|J%)Ph}O!ea{ z1RI8KJH!@CB5Fs_A%csV^#CNqY&F;dCGmm>EAB_ZaDJw3IP?MptHMvC)QJM2n1W0& zEreUJO$WrR0cMhXV8H!PXHtWDNEU<_5(xxIa%#mFOq}?FzZ#W9^FT1tfP;}9&O7Ky zG}Yz=15WKSHM${_gSVF<6-UTNPN@7@69>LXHD_K_pqSJW+g{kyZ41vU8b-YZIyU%2l!{gHh78ovY|^jC&U zh@3Vg99vcdg7EK0dsglNxne?_t2h0dt&1M|?C4V?7e}(ZqxRnTWR|zN_OWwzUN3&l zR(IC^ZPP^;&Ic5@dfo9$sWf?Irg?2vv)7VjixhgZY&J`3XQtO>bHQ_4 zw%Tj`86c4I1BgGLYje4(^NaVpyv?H7^E?UR#|n7JFqe%PTuR$d|=T zy?Ek!_4(zkQMstJ_KL0BW_8-0d;XYB?!A8A9hW5C-rd=`ySi3PvgC_DFFRrry;;qT z`XAP8u;u4VnQ~!{m@kU$L@oSgewn{dOv%GN1x|TdJZ#O+M^s(%C^4Iz+forYvpMjP zD0;F-MzXY4OU?pG1Qx$Ge_iVK;+^T9lV7&|JBPZwr#a*>pTQx%RLsxHa=FyLtazFP zUe=1<+FDy@KC&se@~dsO?lxCGL_>B}x1B6@$%YBCOttaKwx70$)esR+iYWZ8t?gSG zoZSgLUp0o|1^-{8IJD&E0%PZ$k zGG~b`E|)CKEAY0+#lxE*iRmT?jzO-*yW(Ozt?zpgjV{ZfVcuTOB(oLb*_rs7JJj3D zB$+RW$4~jt;?86_H5H`G-{OnIEZZIQWrH|P6gzvpvR7uQSuWej`()82i~PbiQ5CavS>N<H`UT$Ox=*xq$HnGvtFxxN)SqPU0&lIY79AsN;tj4J#ltQ<6kkZKklL7k=dpC* ziZb7s&2)d3`p8CR+z#Y3uUs4N%y#&D&*Vsq4R9pCw!Fog?&t%uWm?{T#Io{WD>Hsc-`rpHaRKDW$OwuX#t+H3CN6aMRlB*fi5(=^~uItJM&lH zYRk`@Kh68E>5^V1+@7F&F^kHvNv9&+X>8^N09&VFUX$w zv96C9SQ>J+23amE)G~WDBd=E%o5dDk^}3erWBb`5_6R%1USO}Xci1QFpX^802WLhf zm&=uKleszELT)v89=DO(%iYI4%Dv9L3*GlSuAevXHr~YtpaGBM+xS_?@3opgpWn>? zh5s5bGmXLm;acG~;T^rGFVR=%JN2{lOZ4aI-_w7h|6Kof{kQrP`jetg6p(kqA%?|b zu?&hsyP?zAU~DqB7~71KjAt8rjprLT8@C&G8uuohF#Xl^uIXpfDG3>HB$H&8Qltzi zUn-W$r14UhG+pYJdZdNYQfalcPC8GzP}(Bxl=e#drCX)rR8q<_HdXXP#`#L zXTnn|R)kmATZp)f&v#5w4LBK8Cki|M4rFZn3o8dU5!6pVih@FMjD z%ibI~9g&`29@GGDuv?CJLnx#%sH1^w@^*zcK3-|SR(lZxG2{>jm@9IjLrf6VCMZo% zLa}1Nx`9lypeRJ1TI9h(R8mYIrCB_pn2fje88v5Ga5RBqX9-IE=*+RRH)o z@&I>~LoLRdg8;pt8b`65`K>!{w$lKVFJJ~kfyFJ_DgwmCp@`z0STSLH40n{3|27VR z^f#0ZZfk7?thsXN(NsHVgs?6krXhrQNMN2owS?t?w9S~_5Y(i>L53UJpQf-C(%GRj zV|m~V=71Ek-axCy0*hP&(6U(M-~jHzI1>%!WK(Dftsj8Ijn@WHWrR+U@|nqDMWcbP z8)9f?vbo|N5G^z@u?t(@AgCPcHZ-*MD2#s&9%B|*2hqVsGFXgL<&b2&mE5<%Guo7j zl?`b1BRwd|63U;v_Bq)R}1)0I}Pm7&{6V%}P7#$3}A6RM)kOR{Q)P9I5Ef&!y z!ASIW*r}{wF%~80r&RX=Ng;b4`kJ=r3pxma?zaS^oqw%PNb9)tj*x8_^AFf$XfL{q zj_Hl48UEuAik}iS9)J@3M~9-0ICw@g1yIExs_5yUpLjznVi=^bVm>wN}De_*zmM;caLf`b(%Q znyH4Sj1U%>G<4H{ngU>RLle(YVZ>jK&~+?V+=-{ zV64H)7B9v+m=?}JeJTTRgh7w8oml%~dU2LxfF#+1AOIdH6u=OLnxF=lCh%4?0gA@7 z6T{?0-)QL!7oCB=Kmj`xvxpJi5QE7S6*AC|X3N57c`6NTcpJ7#v5F58+++?K6&45- zNi5VXSeg|J!!B7Q==FFd1{1m?lf^&@CJOkX!o;Lu@W@H5kXMdMQ@!n| zc3FsaW>HQmXn{dD7BIl$6VqzYP#(kU0pwU7Bg;q>PmqrbF&(Z+Wf*|TPK1Y210ihy zpjM%2kTeOLfh6uC4pU}13A zgEdesQgM_DATDwcg53`i0-#U0$!YdLgz0IL195@iVQ$6`O&UN@GK4VS(=?xV6aG01*lauqY%U!`V#m>#yvO=({s3_e|K%t-MP zQw9wn2W6thIDxsh43ddX59x}5nczP9B&rS2JyaHwhgpDFj1DK0>vUmeIBpI zn3+TK5}p;aQ-{HB3K66M5u+eP7mjw|Xb^$mXcDquf>x3tj4~*Rct(*`5Ol3)rc#x) zf(Pa#SnQ}7Q3(=nNTikmlx$?>G3<&$>@m=$qS|1BmO&QO8BiioioQmq8mJ2wKnsTq zq8B>h(i9V^BB&skMwp+}s2F^rsI4ZL7+le_f(9DUP%LT(HF7M~XAlf&$+(l_T%uItIn1nxZaChe=-Xx#28USU$@kTyLy1u8{jl9>}+oM1GV>?aFEdJ2F89wU*W z=Y)?CZN)D}R0Wecpvn>zMmrw;`9+>HNyr~0i2X*dm;pkv2#LVR8T84hEhqZ0(WZ|v z#&05Eg-&`8H=hcY830P*EXZ*p_j4RK2)rR5WpY2wWQJs1Fd}rrAR)rdkYxA+p5iks zsfzehIR~04!Lx-!#0tpdnHgvFBEsMXD#J6*#!+1X+%QuxaL7>#5ZrXpp7hfZ=Hvv7 z<-nn3oZ%YDU@-_kwdt|RqX=9EcqDLwVI>Mn5}DrGF8=f(@M;Cal|*V0=T8#Sj3C&^ z6tf2vF~}yQ=1eo-ao%vU$cTHvZ-a2X-iyKkzsd=6664YXP6xA=0NEtHRG$J^UJDQ_ z^78GLRHpysT~^7nRDDR04OTu~kbn7xkRCvReu5yhBj278xV>oJ01FE)LZv`^fn@^L z*1XZc^hP=Lmux1x5zdk%_&!Ie)W)dE5RV_t#rf?SA<-z_Cno7bdcnXyoGNm1D%kVO zx3XZ6%wiJWs8=txI3*_IcaA|Y0v}Z{Fq67~IT$mQ>p3|I?Pf9Hn`lJg5NNcQl_6s{ zxWVBIGPIE#iCH)zs9}hsU={URBj5~YFnDpJz+WZG=nA|M!Zplb z6TGs%%t@A%KuaJIFMiz268A?Uk>a@I3^;&$We{qeuS;~px&WRDLb)!fB!ZRaISMovQ+VeELTeFWfUKf@w`-=RJl|NRMPW- zuI!jb2o47+475~E%n7u}k%+T3E*UP2%=fe`Jt&?VsjZ(~+0=-KoGs2yxq_Z>;9FM< zJ`58iCoUu+-zqI2&k>R3aFy(rq*PZTfLkS_S#mf7xRo9e{)wApr*@M>6iE@>MHO_o z{NrRaqa^W_QY7F|r$p8V9D&(jjdL@_nJ>k)zz(U!DLH<+o{UVgBVbxcRRaM5AtV#s z+VLmjD*}?@Zjq2FGa#e%%7ARcS1Bb2MLL}WS7ixTot6H=NM^cJ;Zfu+@VC&LE>(K0 zQl}h<(D_QYB3DKNvSk}_?c2i%^oMzMSjmp8sQ50mdxs++%jZ>Q+kbgK68XotKq4~1 ztE7wyBnn$IZKrfBJs@{Sh&?S4X9Ws=olPh#lr}7sl7z}AEaAhG2>rzB8rSS<_Z8%u zOoBh;k!;3dnA44fdP3mCc6`_YIjn&YOUYSd>9lM$W^{#;t&uYQOSh%1oZfg@NjT^T z)D>q{;xcZa50{i|v}%XOEED>sAf(Y1yB(bye3dZb9goF0VN$Y%3w6l&jihLB6lZ*f-rM$owvp zA;?C%QSjlQZt<-{1pAa+c?8HYnk-Th%`3L9DdPaO$VLsd|M(vCjuFP^jfn_$|0rB*NyqrV z(3KvlEXE!7WIa(?);nIXcTYCD^b+!Z7=>^|pCSkteDY9OZn${1)q3g=R--W&!_f|- zvF}Qc2M{C2N{86YenAMODw46dCgW5X)EH%lG*&3d6~57+p$%pA7VPF!tACf1&(BRt z6-+G8n9tiif*VaI(*P$33rny?;x5TewWWi{;JqhNi9a%GmqhjP;5bb(=h`c+NiL%` z4?JGv66|21-K8LlnviM*!%+1pxg~fGHIVH_yB~a_FL+H-nKh|Q`x11xL8KL8r56;b zI@#zOvA*4iAybc=-3r{*L$jeG zB)}H4ETrToA=L!VI9*_dV!Qn+kc@b{fGZpby}azl+L9}$d8lh{s(Wo0sW z!9D@U6eT&=YESFI7sDhjWPlx&#$hSfZ!W= zlr6~VHjh7{RQphdHA!HSn>t?daKT#1#u<$-%Mc><1JpO7X$~A5>=>~18S-Xh_Io}_ zvFV`}*kPK%oX1VjOEL!uw$w{0$fu~7WSjQW8J|bM(c~r-mZ2qv0?qeJGsg4IF#7G94gN zdQ$PkbZb&5DV3Q(n$7em6h&s>k~ro};sxk_q%d*ZKQSyCOwd;F1CS`j?l8zG2bszF zY=>9ikW42-hFSygD>G2jdGR}D&~&#{1FzSAg~^0Dk4oDQZMzB=0Sxsp| zlAb}mp%R5mkgsQZAks#F+^Y{m5fVicHue}u7kokh1m9*oa1xeP;`PJn==An*t3O_H3GY7r## zsWfa{q%+BoVoOiuxcShKQMiLEbGjucn_v&>XK*BQ=_$FWAgQLL6zc63lgjBO1rS{+ z>e*Z}$kP6KBo5FGXa_oV02>?B>c%7?nZui*6hi^$$6!-SLgN8>nCagl$Oov{`wdp3 zWHQ*GsqNtOt^CO@@8jML9PR2P>&4mW&oY2}%C`rh9D!gLISl2KnvM)OlIk;oJddjw~ zbF&oS_sLRA#J@Qrsoy$N-m2_Q6f!}&Kk(u~rxcMZ63C-uOgnhc>F9Jq7pPbo2t?H7 zfgP;Pab4xQi^m6kcHSQ-+$2LYOGxWUttkv(Q$_Yiq=Y1QO3;{~2TF;=b-h%gLvlWo z^Pv=QCLGSk6H;D|T;X)6TcI9>rpb;;X`NDKz$xtrEKLLg-5u%+%=#}}YSL|Dfa(vJ1pR@t__a1hUJZgIABcRHP{q^oGw(jAV?l03UGAwh|hr1b&kbuVs~ zEfeKLWoIBT`^BYY`oQ(5VVx|M=_^~L7Xyia_AbGrOwKhL}z7wk0W5nXsL{>ANYj2h>6*Oe^4*w^$uaa zEvx3u^9>nZv!SLaRS=zG*u)Daizl_n1OZGkXQF!tGL)1GgGmrG1s9epJnv1;%jFdz zASz9Uu{-W?xUDC{ks=h=NW38-T1+NWo?^~M$HT%I`8VK! zVzSz7ycL6z614M8PNC9m$}pvI7Nf`LH>PjuLpo%w0208(XMP#%vi>5G)0AC>Ud@}rbf|;vaCfv&lN(OMv zc&xGv09CXL#bJE#UX+S;PL1&#&}Gi-M|M$ zFXjN!q)j{yWIfd=TS+Rh2TU$3&~cZYH<`d}oHq+tRZ`>dcHB)ufj7_(&*NAR?tmsf zBbm=r?4n>YyG%lMDpf<06@vleBcF`iB}GO_5k;Pt+!@&lm>Kmdc%f5(82HN)W+!n0 z@OfrumDz4^)hJ2^pUGDs!HuEV%!}n+R%5b83=i7k0#S{Bsk~wpq-;LJnHk8y(8<8e z4293Iqe;z%WXWm*&E|}ZfRdSPDmKM2v!KcbliB11>?bkF;^OV87AFrTrX`^a1*E3r zLsZI5e70gvMHvQE6U;DV-~;6FhP+&0*1%wAaM^|AR&YoPif9~z%V4(i%!3RiU zgvLx=z>eiJ(&7OIljdjzJmyW=Ac6!{QA9g00+uofIs@i1QB?Fch#Afrc)03H$yom6 zDZGNY++|7?(BdY8v@02OfP898^cIqh(h}5537BBL0n?Mw_e^$lEMc!ne+XiQcEAT8 zHx#3Y218ghS&0#L*vR>S%Vfs_le7tZM5tu^#eE>&z?;#CXk(%snO?vd^d&SxgDGrK zOkTVM0lQ-V$Ng^zKkLs%{Aaq(2Sld1x|QUq>Qi)nN=|?*vo=`UC=FwIjKe9iYyUr` z)qoo*tdy`Y!WKwzaxl-)2Crs<|24%4FtW}_al#2JxpURe)%T97@2W@FetXl;KlMN8 zc9-u}pZ!t2u@$2O^uW4%Qd-Ht@%23aqkxI+V3{JcyU1>kQj^q=g1PHz?ZH2!)uJcj?yTkJ=M;9cp2Rhl*cLfGlkHrZg} zgryvtq-4IRwVTO zHx7)76c#!%_AJayJKt=s*;EfqvOnjW@4tU{U0Q0P2V1JPe`LGJp4iapzBs7p9n1~4w4_ivKB@CZsn0&PjdD>#YjWF!OMh;Up zRd--#<+S{eaUYu}*pMJoGK~N9ikwgmAqZ%DEukO`-DI;S6EKRxyF$c7Sm1A$7TEPB zdnW9jCVhtZ5|Tn8Iy=Q8hV)93*O)XqS#V+q z#6QI;j0wR;;&v)}y{O<$XL4$)O_ZFX=oXQ_64NYhlVkvwhM=`*NYDP*X# zSoJ2X4lL=Wp_UnzqBN`CVn@;G&}#MiI;lc0-D|PvMNDTU`2oE+ZJXZh_IM;wOtgq% zv{Ua;P>v!6^!mF3xr)a+-N{-qTh)`e!%fsl#&iV)=%LWdx$e(#=tf)d{a^Nf!OsD9 zxs1$qfS;oMmtUhUm>(QQ%3T=cfdT`@agQ{{hpCsHYK-q;@7Vf|a^Dpe_U)eB%bT2C zU42=Fg~HQu+bw;6Q^PUapZG~}^(rkBL*o~XjX2QC`?kMdC`Z1*vve~so=|)s+2_%= z*c=@g4zPKQ?I-M7qRp|JVMl&0+VI9^6wv3;bcm-Q>=_vEu~&gj2y7DI138h_QHzL*}aI|z6mZ5eAVBy4HxX9VO|WR-%MLoGl8A6kg>V{0A)I=FT06!mhP zwG^tq5C+y#Q|1o~uCa{RWYtitELN6USUA?A)~>7h!8%lU&q`?3R%=;BVG$yJGA|jM z5gd!x;Ay;ZXr_7x%U-Ac;swA~jb*_&8O1Ls1UOdl?=52|aN8|o0U`ZRW{tNzyKw8l zhsH987GL~l;UVB8P^2GWqeIjI#-r^5`?Ix3F4Y%cU<|uy#^VJ+|(+?ITuQo9b;+f+`Xc zPwMzYJvOgLvD|YhZdYP|+k71>dZ|9x_rbQg^8l9Pe!`}H*0`GQS-!q)dCV$lu# zz;qR!(D1@(Gcjm`p9Q%^mtk`lLS#2+fYoTka;bv9FI3!8T zmyibu$=lVtkoIH}<-csOsh#r>6*UdEdR%Ni6_l`9u;@^Gr7x%~NvS;?)a7%-Q?_6=b zveGQL%t`)P2M-o}v~S;c{g+LgXwFY5dr(PKs~>ng?n`{7>O1EhG97NHG=uhU`=7zi z(eE(-=If#Z{nHL#7H}LJ!xY~Kjs|#a8q%nU;g6&u1@QPmL9fH$S_<4rZfs>Ze1ZD@ zvg=Bml?AgV*ZL;hS?}qtuvpYZjhD|xXqWmm+o66it7?CY|6PH(IOf?g;kom#H&
Y< z1fYj+y@(UoBL?IJd0$~O6b?cT_WGUb*9PN{Ir?R}EJ=MoU|{8qGgvbd&fdrPnsuCd zqG}u;p4VIzVVruqq2aN2H|1_F$PRrzf5l^7?h-p6>WLW&$EwKi`a7;^`bw;s?+5{y zmFw?UuY(olYM^i+9w;RzWW)Y6f<)lYXGq~OEui?+a`Jk*Y-N%+?vS~xAa8UsG2eA# za=2EKhqUsNdR@~y z$VWgf{nzBx1E5*IO?y{Qw`32>64XJuuP6*&7q5^sZiA zHd0GDrje=Z#bWp4m^%*xuMtf8!;DAzNdCFPsliHRrM%*DbV`evxh2kE^o^X1dT z2Cx$t&0%${=hWwWx3Vnt?|dHOwKJyWW#layr2Q0sNdGPLB&QZ#?}TiTM;V9#7`;O7 zKLgq*`oN$zic?(b$#AlZAJh7ETT)e^ojv#L%bWlBz@|w(mtDE)+NI#qw8!?mq5q`b z$$8zAl18Q!^r$bNI-xeQ2bqtxF7CTX?P0T&x@dsi0-eUve^B2h9Kr}P0o)rz%<`wE zIB7(q^pTi@(A_XkY3>2qGzH&ED7F~1gE<5AfvNPLfRgb6^(LrgO(>@tupRv5+0oLd zI{MabG=19@vU>X^IjO5~KQ(~|3=9!kM_zY^Fq z_WG+|7Y>EhQ+1_pD*!s0t@zv2!^2V)rESw&fl?)~QvJYRimZFq(zDnUmdVC1?z>o> zkK}X#YdZkQqKbbN?rLKI7NT+I6a7(OjrbAyO|tDvfzG7?dtjrT=4U8c`Ya_z&Kc2x zfl@=1&}00;FbS^Y__2cejMZ$hX0B1)mg+TIR;`V!+xft>Hlf6vmgHb<*c&ar_|#$ zUd!4c4{Uy`ExgBKz5fBP7GJxBuj6Zg@#5AHC!jNe{qWa>MuP?dq!hFZeNa;+$orFA zKjQ4c+N^HN#<2Qgcu18uJc=C)Thvh-S4;}C2S=2Z4;?6DDCE$8vb`$RS{0xD>~5#t?=6A1Y08ilhpZ zg)}fB9L!)?@ZlOLDb}VYOOyId;aMjTD?X+sT+|g& z|GXI)J!$|?-2|9-!pc~)<<)sLT^8&MKb1L~Jrc2`jW4XhHlxK`WBG2?uo~Xl)g7qL zT$VfVM~m_2`G1OAfw4FY9_r&DWnfP#jWpysGmu>jydba;^ds`S;$!5ZIN*Z={thG- z!>S>cjfa3{isca)826v%3%BsqNq5JxKD$g&CSA6|rJSXn(-6!015QANm_O;B`Kxp9 zK5PAa=8k2>gMD8&tea<>z!^RHB~=X#dhB*My#Q^CDO>+)Q-8R7egmcIiUBil-VNNj zO6;s4RF9xKmt7sq%|wi9(3Zuv&YRkHnQh&?)wvB>o2VbXqRWK0`ypV6)oC^KWirU6 z9g%0*rSLS0l4Ci!$WTuX^qTy3C6*9<`pmuB9cT&~Gix@ZA(P3WxfJEpCz4$n7nomm}8#khH!2BA?Hen{F+R zvZSJtNg3lZeYwqsm$%*VdxzC$x@pGLzW4Rk!eQ3gvhVP8bO+cCPV}GB=R$`~(KLdg zXxmn5TW6|{huD(jVtgRCbU0cfN)%3p1T78^qcUh@bdH3a{Pp1DL#%;TY5PyNavwJu z)LXv-j z*rSiEce;ue7Fo6lfAac@`o2<4Q=sn`4Yd|=b&sz(XV28B>g&TNAAfRKeTQ|t-1DB} zFKed_t+{-t?=4?Z<{}HrFTx$K4WFXjVHuy?gL)Iz-RDA+xE*>&^hwzri zQ%e6Ms3x?{Gbdh(_{5MJck+Jd$K;O5+2rA!Ba+i^NxLSrJg4ra#(37q(1v)UdK-`w zvliOai{cCNlH*sX-`Q;Y8*PoY{Nk#_24?O=6yf%n!_FSJb7wrcv#Lj}Q!ma^-|x(d zAB#6ql0Do0`#KR^wt;{NAE;k&#j&DqJHO~Vp0)m5n{DYD9j{){--ivLTd_7wU~I!$ z?F_7=U#CDC$*^|UWcZ}m9Vzryjlk~k7=@7}yBnoy6D zBGp_$l}ez_n6E&~P)_|EKn_^YtX$ouKbX#4`!F9<0)o2V$~WPi8V-TZBzVQ8m|5`^-hWuy z14QXGMycVyW0amLa~Ly8$I`0BU1o{iHSY)RR+2HL?~ephx`(7B3Xsx=6=*+94W9-n zofj}PZ$@Pfw(IXLOuUn@sM9ssf?|#qJQ!8F1KsM;Xq2Q2@->%EM^~wz=3#kD7^Ure zD+G+{VE~SBR4j&E5mFc_1JDNj7`RE1)(l_?<;N3^nM zJQw>+b6G?xZR%L#o_)-ic_qn~q^ zbxf@9QLkiw%e*{jTJj+VMQndI1(lAzwNu@JtTF~;8`|latbjP7r0Wvbunym-o@?yq zDAUxvL0CH!+sMb2`g7DgTTgUki_uo!QO_;wF+d%mwd5Db@i0-mdaw-*As%+>|H-$1 zqYiDzc_%0@^5c|;*q1({>*E2s#s-y1eXGv~T|nTbQ|eRom+IGy;#r!lidSopAQV`! z^-tJ9P>!vqY$Tvw>nGK7t9lxO)c^b6?Ei@0D*Dm??Z8FCI&MI$&R7W9qNmOBfIt?2peM-}1_Xm(5t+z3-YEmZ9VIzKX=t>ZhW=CHbLP@*(x;sV~$z zc9dnYJNkC1_m5{i%2BpY0d%l7zrD#nCQikC;L>og28I!Bb^{pGnF}#8JYc1RZ2?-@ zy!D}+c2oMUoUN`7_WH|zx#YEnFX>t^=ele6uYlGx^XY5f6Q@4lty{Y`B9+XKsH;!C zrw(CHBOY<`vc9G2U9D`9QePHi_i6S2S;z6a#r)ItKLZQvbou|Y{*T;JWBE&0=I}Q5 z%(0g@y?p;A(^t;A`s!y~4Q@a5goZFB&rtb=y`e)e;*DLX{ZsqGl;2r9R zv`|baeZ0E0^bl`u$H1pV+Y^eq6{BJQ!w<6;qRbkNs(aaNb+5Yj`e}-A6??n8z5UdE z%$*J2C?y@$$9M)@bipIeLi^K-@JwwWeEtH!Qq{K_E*`p zx@M!Ck#vPHcLH;6ESXSJ@}ml;w3)SjHg8#Lt>5o&a5It>n-%s_$&bsd*jG`pZ{-xXAd5&fVqYn1 zIARnAngqsDBs33V0H(cyc+`(z2#U!&qiG2Kxg6^-3KO7aq16V053?~k536mfEuBzW za#DR_xF3SkFw_q;?g?zHzqGAx77|3))%{{s0MR|V`j!cO?R69CScfU39@mf8%U0{G z38hM(4I`Cl7dLEuX=y2LEya(LhP2(=OG`_Bxp)_5qT~JtXNKJVT532}>ubK#dNw3Z z2Q0P=#Nn{sPnfCa&?dD~&k{B7R$K*)-X5Y$6U+)hU+0BlD9 z2DApm6#>&m2>AoGC8d$Q6OQNBKFiMaH!S&`vg4sqQ~ibYr8jM{wyi2s?|p2+=U3IK zf2s9Pm{r&IvNf%@<}%r>=k^+eM@AQnF{GtEv!e8K<#)eN`E0^3rM{~c=d`kg(Oqv? zi)SATzQHb-z5>(MFr;-aMPBsRKCWXM`5vy)y|x0f1uHE-4j6`|nuQi|DMTB!Oj$Oq z0?27gq|uCiPjBL2(Ez#1$)+V=HX4)agD@-9XK_{4;VR3hQR?HPmipNTheytPYx{*3 z;PwI@He4K9*@!n=yy}=svL>_UXC>$2iuF>uVY9#|%w~UDm2c@JziCEMqqntW*`&|I zgd2Qxh1vVos2dwcS>E*A5P!XH9a~)CK9BJExc-Oxzu_+y*JB+E!(a+%|4&(_F~P%% zkEQItWtpbjnVK-hvMIQd!&1*6^6_IEvR>bxrLIul&-!`t6>)9$$Rc$;R^FG~$nN?a zihhH7FsswniC=&mu#etm_VgPw4h4jLg3w1wJ^hsQ9aNVi`}8rW{K$`e#TB17a4Xnc z9LF2uoeQuUz)kdR!&$0#0(T2{Sx5H8R4vSSI`vxq0}Xeu2@%6aUzio@dp-G~r!&ebOD6ZJ4Nfv~7iE;GKbsAPtSqP9O#ciyhRvjJ-A7SM0YIPcQL3 z$!4+Wc;9&6sn>m0*7l**7x>mU9#-ykKIbW4uXVWM6aF;fyW=H(7KElZ+-LQu=OX8! z&+qqHr;ji8t=LekAj#s=)+N)CQ*pX~nnq{;2mN>OqX+3E&G~<$lUCjaOwqJB&`d7W zw@@ou$)N!a)HtQsPI%zIP~DUg3K#s|WGYEXDbBImb37@}__yRsNT+*F6 zvALnD)x}+Nx`F=B`f)G9)3|6-+0>~QHOmEl{pfJ0sN9FlCcVbYOdrNVukLNWT_}X@ z6=<$0+WtN*(a?V|gkY+|_8DXf{R?R}Fm_OM)S&TX$${39j?6$c*%Y5C77cKX#n!e08vv>CN<+Eq`RxImTG{;v@r|am{;$B~@u-Mn#yLh3m zv1iWQ$QTedvvB6(g>#oLUs6?6v})C=!ZXRFOS;AIU|7SFVehBRnV;i#v94FQ5V_43 zV2@}y{w~tZ!e_5;7Jm(YH-D6W48ITYNBG;3hi8p$37(9=GCCbSCKvBmf&1s<9dorO z=i+KFzUAR+DX_hZA<3;cHwX7E(B9gHJ9~7?aJ?J%QSMHkZWXT3nVEQYHjbC$a~8g@ zz}X($GY7}@zusN<>s>^-4|7`~%AxmB$qRARh^OYDoCva^FUC<1-du=Zx;htcUyfQ- zp>G$VWcn%muk|?nR5P9(;1cnv=ifL{g&h>CH&IJ<@?mF{Xe$1%=Y6nE&IU!ZLD3BD zS>jhYM$UinjgG1IJ)pM-Tv>qc#GeKDC4S9D_Qumx(YZ5QPiw#5TwkPHgG}iqLX*%S z)Znx7*Q+1kjf=FmO^2u4ba0+zVg$v|}OKsu)}?#k^XE7LS8h70_I(u%4}kH^)%udBZ_-EmpS)=(F`$<2GWo zF&f!A#=r{PtZTuVcRXZq0@|(}?b4x}1aF>BT^Dl5PSH)(ovoXOdd@(b%mOdEHTmg* zT+9b|7HS+>f^6v2#>>H#m5`d%=nd!S*6P;j*6TLt&eff#J70H!?m}ctzesnnZj)}a zZVPg!U!uEIw@r7M?s8;N->$n-w*yE_n__}-4WfxK={;SZg~vx zp$6TPxE71813abobnojv(0!=;2*^sG=swkbru!R4hA(tq>b}zbUH7%_8{I#U`qIQC-S@f^ zx*v2u>VDGwjM?O*?v$=ir|SBFGRkyEb#Eec#SF`*l_|`|Qkb2k!ehe8T)=8dM~-MW z5SzTvoPEp>`$`bznryh~gkUqtWBIIrg;@k>PDQ`~D`BN93ZpRgTL7w3!7Aa`ID}QR z8l+nr#)h*ItQK4E2{w||vj*14MzPVXiH!l$X)|kK@q;TU%|GsE7=aVlU>Dj zvEA%ywufEA_9D3BT6P_~9_!W{flqZ4yO|w8_Lp1Ot?V{-5O`I$vpd+G>@d5F-OcV{ z_p|xjtj|-Efeab##e`BAs zFW8stEB1HxHT#DBgMG`s17_Cu>;(G($XP$JpV=?$Bs<0Wn9BNr9*aO9gun<0vKA3D zLLoFJPU2)PnKN@1gq*hS1m&*hSm!Au8K`x8S=5n|Y zFuC%$e6D~Ca}lnPE8>cQ(N)StxiT)s#kq2>f~(}JxFK9MSHlhEhH=BW5nL@-$0fLt zTs?4X8@W;3Xs(I-|GK);=D2Mnn(7&n9uBqHt;+96@tW(Stc-0XisZ;r6niV##gLdG zEe<&5A=%D-UzKDx$Nt^j20-*M>ngDyj()Ej07;N4(I@aWoWdDAgm>T^zixX0m+&rJ z!F%u+p1|MnyI=3a2k;O0Coot+0x57<1rt!h4xT{;H8jw|4cy|F z!d}2j_z*sVkKq&e6h4E`;S2Z@zJjme8~7H!gYV%7_z}OQ`xE>OzrerX-|#E^2EW4} z@E_}S(H6yDMJcwdy2hJ$#hbPguMC%MRXVN6@A-bv*-%F1f69U2G`bJYDTWi`2IeS@n(|OK-yrx&MFEmY~%7yD@^!;ZjvOj4L}ns|!|V8kc))HKi!)gEZ$A zuk%`!+{0p*;d22tjF>1Je3-yL17Ggy90gVNon_OtALB(tRe5LITwX+U;j-7+J=^k) z?)0}DyZ=ecQd+~iykXXTh9{-EUPX)8>GXzAuxWsL)Hm&fRE;YmId(W_lYQ+62* zgvWas-N8Wg@W^}=dac#%##L2X%73`R?$$;oXQjMonA33$Ut+^RHa03XNoHNE)C!fF z4)X31h!&pbPO0-7R9R`)GP%G~X3$`co&I2l`6c*+;aH^{cu&%ruwMPYFujgb&sx{0 z3a)ZiaKzSm9{7G388it9g@pF4G1W~7O$p5i%?T|Gn)-xDV`?;lnD_!>35X>imXKIN zVhIh4A5wfF@r4v$NX#KIhr}Eab41J$F-J585otuE5s^kj8WCwkq!E)wOd2t1#H10E z#(rF4(uheTCXJXhrWDnbJf`F^C66h2%qXfE#WJHQG8Vk}`kj8>E7NoHtjRk2eNMp8VF&cP=bZa;8O>5vA zjlee=fp0Vd-)IEB(FlB_5%?yLz$cA>H1;u1Or5`&+gF>4;oj|5n=5s}w{MjhU75?k z=++?nvF`Xn85KXC=>Ghm6)C=WO3I3-tK#5Ui?;!}DXeOq+S^>n9Z1cSQlumPJi*Hq z_L7&Y(2thd&h8D1ykho_*LaCj%+nz3)*_;AZl*yr2}itca0i8Nr$)HNs><7fuNz09cIPTvNeCHFwLZ z$>gn?6<#hdS4^(K*KT9}tj%v&!9~KYtZZw3fWuJ8H)O29d!s8m7X>cv^11y02WJ-94}v*cU*Lp^)|=l$`X$2?1l=1x)t^s zpN9%Qln1NcVZ_6vXjfLo8H%Pk+v4dq-s1aPz4rKDdbm6P@DVVU=O0@9T^;@p@c(@U F{s+KL)(rpv diff --git a/testsuite/etc/logo.png b/testsuite/etc/logo.png deleted file mode 100644 index 0e15632f7ef039d604d4f7d21440dce52e8eeb76..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22059 zcmV*GKxw~;P)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RT3myv=0OewTX#fBq07*naRCwB~ zeRsTNMUj71cb{<6%Wvkr$ultItf(LcKmipA5)=d!1w_`>^@nR%l@(o442YOe7e!rA zf(VL$h$~SrkQp+Q^SpTDN!|7PtMIjc03L1ONoeUOyf%CePMJZ_Y&ePXu$^+G`|pL-DsMn~!+% zvKwIb(0pnU2`GILB!sS!lksKJ|A3Ir1W`I}V-5Aola)@sk~}rrCumF+C4WHD?x0Un z_R{IE@&(9mhCP1eoChv(>DdA$_Q!+l%_Au)55Z_rO zD{pQD(%!MK*OO;LVRneJD+u&m(lu~GikCTgZzSW3$*Lr`r(U-J1fckP@hcKg_IBws zaC{(}(Hm5C9NqdqCSA zn)D?Lo9T_BxRNo8+A>n61Q0=!opcQ_{Rz9pfpRTE#g!m!+LEqD}8OZWk)*7z5iz z#*qu4WXHrRhjax8#tuOQbR1+H90xf^%j!0ai?#tG5GHGHyNoe#N+<+cHs*VVFdKeB zisF+O0c?jOg5xsInXq-!j^aa!vaO9!)IWrX;J9d-GZfz$^O@_OI79$s=#&ycOg0>0 za&*LJ3n9f88+MYh{`pfF6Abbf4` zl`DL9CIV?%<+?&MwP^M?$7Qven4S!z%=Mn(Zsf*6z6pel!y|SWDq*oGP+Z`SZDD%L z+}zAuox)L%gp~^K8xT9T22qqBo0+&G>hGDKQ?`xs#_TX4$6-OB7-L&D`LS@OW^{6z zW(G%g58KA^MUL+)$6>y&1jmi*e8S8tp@OQ-4^%+M#)V5A-&3~Bd`}61>(}@_R87%g zBp|R|T(r#gJ#t;<`y@E5eaQ#t<@^|@7Xg7=VoR3Weutbg`W+G$u71wT*F^I&$vn)7 z&@JKem3F5?ZVCMk2^(L0+S5L?qV++NL&-g*0t0wH>&vYH_L!*4b zV(Yo5{4k)zgwM=N$n3Gq%`-%Cctk8*CZ2gB2z=_ATuP+7%dv7RY>)k%R#R1~tldvn`$4&Wm3`X zOc;eE1V#}t&TNNCNiw3I9;E*Eh)}BVFeKkgpW8i3GLGmSRj6&G4HS~RDIV|!O z7pz=iG6FwP0H%#=Vm*~A4L#4)}kbGaw1p>sMY9Eh6wRuD6b(B+Lfk(oEYK?`V_S`O*335vq1_rtB$z}t? zkYH||(Y?nIVPJ@RZPlo&Fz8-M)ICJ=7KjK#!`$nr#w>*a?Nnq_!6qPt{y`r4wDyIt z-BRT$^V&*S*gwF6fI4jg=xzG*3O^Gt_5ToU%cWL9>KBa6a}cpvmr6kx5=oSi8kuKv&b+qL_Xt7}p_xB!13+U|Dg{A6 zQi6ywlaMKrd+=7P0bQk&jfDANNS7c8+6V0Kzc znOqx&v|y3V1-6?~gS4<{Faa!FW&^;?WF!>@KFN^6fEF&bEeo4<)rHJxkS^kq6*dA) zO-Lyr@RW=o3TV-?_*x0wW{_q$0fKGe(p??SvDU|@CnTwYuFVV(!Y)CxY_@!NM{rgf zU{l+rQX0Mk8csmP@yFKqu6tNaVBY|r+$NRk8l`E6X9L=GPX~c^Y>A=>ytaxW2s~P` zhi%z7JFQIMIfrBg=+FRxieP#=WDG%pl8Q2HO9#i0dIG{q(&S{wVzWXL>Hu(0EH|A1 z&r?A_nHIGj6oPpjDI@3!f9CcGA`pa9ZQJ5A232K|(mhK`oI)~;t;{R#w50FlTG=U~ z5UkTuL8x^9vg@co0BE(O*O8e%DV3S9vDK8i3hV+>Vs{;y4e50hq+nC2!h~SWx(tF; z)(xx?G)E2^b=hglY*eKRvnM`iAe5p>AF zF`)X4Y&WyH!fK7#4y(^fzeC-CG-bF@YIg?#Jol6@1StY3DS<_btJ-boA^?PedggIY z2vj7Lq8w}{GS8SCddOh-FrcR%4Y(juAf++68Tb`rD1?SNJ~JU3b;^`t-vBx#HZ>Xf z?ZVS^R2zhbSEh5J068!Lz%oE3pQ+R}=xw(tMZ#i?vjpcd;3!R}Wc!@}5QKn8Q*c6{ zWw9L3E}+>N3XEx(iBL%poC#qRM&Bo`<(GSTQ-J4{vFF=jtc-FB^rSm?W~|wiCYfizRL88Wiz*oK%k`Nz#6DY zt}o*ilad^lIf+A$DO9b;T8g$>HX<33<1*V}F|d_93k!Q14TzSsYfEUmF3;P0&raPybPEI+hwH+LjV;i$nX%&=!zj~Ru|EzyKl5;%;Cda4l6%M~dBD1zAPux&Ov z&u%rNsfj=-4f`%aK-qGUQl#Q*9X5Y~-KxvUZJ|;?iU_Hoq0Hlr;zz`bG`7%g&PF@7 zg;K>Eg>oFtj6cQtKq-}s*~LXm?Ae)U+m=u%B1K)E0crqu4Euv31VE?67B97Dr=zVK zLnVO}X}}4ko12WFgH)uD0NfHTS#HlvMVr@$Qi4*NBav+wEQBMHsZ$CvbBj!XEZsUL z9*rr;2}lHrND=4(5!3}S53d-893{_9WQn>$7P;c1T#rSPVog!1;L5$M$*pqzi}Cvw zj{gkDML=M495PVw+lp}j>fy*?dTkUcc-fD3c^FU(At(izPaK9S!y&paMxsJoC-x;M zM|P z;Hk_<-^s`CP?%oZX{-3|er7HAT$uFZbrN@6N@}-~_w_s3wY^6he>&~>T8tv?T2F;O z;Z6svn{rY@yV*nT=d_NsZ$=Td8;W)gCG-yaLljc8uHa=xYTUBjQh`rM5Jl9itJqNL zw3LST^q49}`AGzs-bMzl%TrQHg8foAeWQ3|1OUdscDcE#l*F~PAfSG!TVldy^TuuD z&}t@_pyM*jE<)l-1cg#@$>Y($IhHFN$-vS{=k}29`vMZVC1yLgV6n{*>h;)B!*UsI zCyxslL0-)_FI9x&!s4Y4<1jN7#qV3L3ddm@bkoIPLI8kDmD>(3UEy*@GgD#w>q?c| zF4wk&KIxFghp}Aaj*BZ+x}4+GMB-dlYTPX`EQam%@COk47N4Gi*1g%xZKq+CpSm7P%tRwgGs#U}>p{Em#xU@|#0&kBGDl%hEJgBg5Ra6*zu z0VckWf*>T{Cm9i>Xqb`^g-R*vMyQN^oefb@6d#OcfE-O?0TRa603wni-zTp_k#;O7 z34yPa%AH0ud1LW?l_JllPFv;2IHHUw2$WJqu&7`d#0OfZrGg+e<`JYMzZ0WYnBYIC z@h8*il#;v-wOT3;p-^%%lkX`dVJ?LiPf-*>r%kPf@_m({G6*svzZ0WQ$z)+lRnz?p zDWTJn%|;B~B)^ssbvjDQR8?sXcA`kg@H|7BpI2Anl|68;PNCw zCeSVM%n`v1jwIa+NrRP`krjvlxP_If$Tc)&=C9#~#4+voP0A(i*cb&$cYR6~W;q<@ z4p(K$IH5}k?CaxQ#se1745U=;Q?)7_2LsUQI2^)|DY&Wghm`h!i+JSdlQ4hqFiB}mN8cYBcf8}88H=8ZVVZ) zXwD9_9X5YlXmMLKogfq6C*5{M$6;d&MY{PjG7x4@gN)&nZM<|eb_vH8T0MeH@rW}U zCv_ePU}TKfYS?b71U^O}fRXv4Z;)x53kz@n&a-a^Gpc4eEDDvdn3Tj2qA+Kgb}_C| z52O)*>#!hHLa;~zV<1D7v$6|^Mgsp(3b>%_vcOZe%c77N#~>i=O%c%Pi^-$i5(|7~ zJ1h)<@x;p4Io>((Q6uQa+pbjRzNZ|Q1p#r+0$=K^=OSA^W33_pK)J%awsKq+1;hjf zeukeU+V`9b8t0y0|I)<+H~+Fycz(Ge{El=>Ebxg7=6lj0sS|YBiI_1Q%Qe<%sZyDT zzTyJCjw*0dicy7JsbICrTWwV;v%ptepx;);+Z9kuG8Vel$6GB`t}x#tA<%0p$hhhR z1@d!tI>k4x^@~O~m6bl;X_IARr`gNk(HZuX>Jor~LD8Izs{OpvR+hk4lc0;Ao0>cTJy{X9&6FhD z#=r+IFbXB;;TKC=S*bb8s zISwOI!eU_{ckbdI`nq7z;%N0tV(n@XNlQlLmKYIP$+bK4TJ_~C-P`Sd}ByF1ozDzs^1OONtm;ZLrv=DH`_b2z>FZ}Z72c`B-)f$r#l`EV` z+79~O%lOkeAXKgL(5Fh3HHjRDrQ$RPuYimby*9wZkg7G-0zD<1h^5JWLJfNtHxo3pXDW~N?7j=RP$y8{(YC3gfxEiP}B^mU2 z4adkrXGq;-4b9ciPE?#{-;tzr7=-f-gv*mBigd-)W~0J9WZ`u^%_KFE4m>dYuj^)q zN9NQJVSs=5y>rb^-T%JJA%bNq-T7KNo!#j2`01zC0d)MT?`-kwq`+Vl( z16N;F*@-x5vzl6#1szgAXtXRN1o+kuCXRnc7Db1*zj^%eN3CLllo39`1P?3bTte4k z?j-AxdgX^0(R)NPEbu%En7U;gm9 z3#X^JE)vwuDJMcGGIR~&xS%S!(mF}E>bi)+xl2Z6o{9Kf7+`?guB|aCq?9&owF)0)VB< z2FY%h!C^8G~b+hizks1}P)LQBI!Ev4(C7Ck@skAYAzA zi3c8CbK`9r{(e~kfI|*z|MK>=OP2YW+iCcJ8m%LZHl`e=BzRBV5smDXvO}E;sbNOk zKbe0bG?J3iAaVj1X;cUexigZcSq~q>AQL1i0Wk+rDFZXHi-@nmE7au+T=Bj7$Y}40 zetMO-{^zAXKWJyaS-D$u?TypNye)BMrl;7mPYW^*+W+lm!9w}Ycem#VN~;M+)4vf~;Yl^PE!v z^TP%;+ctqba~ch%`A9vdJm*jV0pNm#(Lc5jOO}P#{bs{1y9XL=x?sNPR0q`ccb+Mm zB|@Rj%%Gl`!H@-Tu*>{S&nQTvX9j8AV34I~S-MR`_q%b9N)czE6d}VTn$fMBsJEq! z2%KXWCEG2tI*vQhf7NT|JW}S*SNy=Rua963u;Wh;SedcXY_LcEW*zj(Iq&xD)8Yr; ztE_s`8egCmERbX4a(tmW_V`YzWZb_#%eHLh@q>hgT?GC)d`=_-8wKs!G?)_`Q?r^l zJ^iF*N+jAgMiFs=kp#g&qeK*H2lU1j5W%uBCNvbSK0PR}Q2^7_9706fWqzmIp-;|6 z>`oW}03gR@VW2FVM|q8!|5k1S^c8jl`@gMuTx^3 z4s}Co)C&wGlhfzA;B|T@7!}g*^f@V2(DTT3Fbs)t2z;CyhDh?A3^$cC);AzzL_(m9 zfH9bv4vl>GOc@y-Z@&Jv%WE}0JsCz)$0P6~7q;diG;LVVZ5wvqV`sYUPxm=FpGe{P z?;rU6J(K+dN+%h;_=0`^y`}r_E!}xLi%qXpPb334_5|;gcNquKQ%_juojtUEjR1s7 zl@APATQ>PJ$~Gev{gXKun$;Q~8nHI5_aj+6AL{j7oSu5p*d?}u3l=-Rubeb}aKjou z=P#h<+3Rsf%5C$$0X{O%TEE7RdQg+Jx)41!3ic0*c?+yH&wD{gScDr2G?ZMu+b&za z((yW~ROX$w5&~a**36epy==B;frtn0vvv4m<3@hgy9W2#Ckg{L zJ&hYTS}(r95Lpxu|kLnNeB- z$tc!<0F@dKLh7_tO#auMWh5|8r>CrRlBir^Q3yM>g@G?Ehle2u4&^EfBMdx+U=#<@ z>*vHU1tQe?MBuAVTj}YQP%P?WD7~~U5%%@-z*nuNj6&-AYMP9L2|kraxBGY$P@F#K zmZCWBRIRZ{qSsMCgan}vXPmU??I$*lKDN309>Ktntj{oHuz91s`bB$s+B)}w9bL|< z@fjG97k+BTJKw(4FiQ~zhFGUft-6ZhXk_NK<^jZ>+AdBXWStJxXJQTt6msix(>yh4 z<6HL+@F1YIFGZcEELFJIQ9?lf00sdh>4Ts}uTvQiy2Dg1iy%;)mIMSpP6Jld-)~2e z3Zn?1kPcG-Bmo&#`b6j}%SMcC#WZ93lv8sxHnD?!?laY2-RP1^KokT(#ysEt&9BA& z2dQ_T*(f|?#}4-VbHX(I;je$O|N0ke{+QG0sF-nwpa1|unm6VIzM7c{0nxJQ`#+k_ z&M~Fnl8gJU{B|{-2*%mau+?cxx5OEXq$0Ck5%vgVq{s{xYgKYD8$v z=9jKYPA39@p;4>VR59Z&2qYi`0E?G8owl0vlmP%2W8XjoLg2`lXf>!*NuO(AVblxo zOLmZD!`MQrSy$yUV*+JJ#ZYGEtLSl6qOk>5V@7V@8UjEN01)_qD|T}_ZM9{iO2A}9 zrQqfp`)&7G?ac~veYX=);581siHO;RKTIP48ccH7pi?AW6Q zH*FL~85RVoXvg+|Gx*viwS!;P*==`Y(jR-+!lG}GOlhZ+Fwo*3U6*lzowf?Ygl&EC zf7B0pW%6{-!wb$Cyz{nFHXBi-HgEJfOQ?ojAvmI}I$e>QiBUuw*ZXn0q)ID->6xOL zu{W`UZ{NlxP_c+ zp$NdiA+^VziFo|`ztuKv;N}L=t`#hh?1FRqfBX9>$H^ZNgeE^dW0(POfh^3GI2X*a z834tR!S|ipde8e4pKW%AopsvK-~QqN8lI|>WLYe&2ZVZzpU8A+yL7&iBpfE}#9W>~ zUsn4PhNz5ic7|o!Y|p*JXI8}r6j~MwLjq6;$QcX_!r-6`L$+-znt*x6y z%6GxQ&jB^dMz<8drXKV(pP8<$ZKH1?0P4$9J%edQbVm!bK@xwa~a zhFLcD^@*v8h?=s0h!f$_Z}mQRN#Zh1O|nx@9Da6{W%}5;Bx^KqqtqwpH6KAwSWEaK z1V9Q-Jf;1SkIgP$VH~QD{LM*WEBV{Iw%&WUYui+*P_3UzB~>b^N|iFLF*D6iJ$}*b zj2R)sVnmEHIP$3WyU%XC{*4``ZO+LF{+EZ`TYg#n?>oyufY1|!NJGU+k}_&x0OxST zQLT5sxBj{}whOoY>qDhmep!9ynSpqU+!C9=z*@iBpPt0|3oSB|V0vp=B}pj+fL4?5 zM4a|N&^!6G*&Bb}Z&F~>pA;xXDl>Q}E^ic#j>^?9I$?;=8x;ot7#fLOmk}X#=R9f1 zVV?et5Jf~1h@ets3zvu&pAAUKMawNEjW8o=RU`>W=y#1Gh^rSVg49akc$_wiID+(!; zNtKJVRt!Wwyyj5v%B!Yh9#f;vPJ74jGpj5#y`S8AQxGKSgG611mbn*;5gsY&vlnkW z{vFNk{c+GMea!)&Bi_6-$ZKFwUjDr;CmucSdA%7L$l%QP)i3z)?D&En?#J+me9N(| zw;bD=oZ#R5+Q79x>5C%7qPw0P9ZFFbKLfr zW1Ghu+nk!*@!iWuuD+@oM$~G`ILF8DsKBSRaA}_Hn&36*;M;AALIR@fxGsGD`}fs& zo347&>BXG`0*WNH8;S_W7DPwCt$xU1?Sl?(4-HEI2t$1C8Rx0T-Bpj5o_ySW;aN8b z^5mG^_w;^x<2nF1@u(G>H(6auIjcEAS)>&R+RfzE!ibuUq*hF)rD9B>7fMe6ATTq@ z32d7Mpa%Oyy&kn&i3~U{1_6(P$iT%-K8O*vVpuBiPAlg}uh=E|H)HUD3)=tw>1tli zg|fIh0s!Mss_>nFfN50u7g5TZdXlJtV0&a-`(D1N5wIgUWqF7jtJlg-p zwbNbouwTzwP+u%ieZqSbpXU+s-;?=7RSvc>2j?5~_Wo(^Bo0f{fQjJ&3&hgRD`f zrMuXTMvv2J+g5Sz84G(Ax-mkpmcUmY5}a`2rZ*hfWTsSF7VWd2x6ghaoH?UOS83G~ z?$i|j!dJEq4MzZQ-iIbG{?vF8fKQ5Q<*G8?n}fp~QSQ2@{roC_%Z6kF_Yd(_LpAG4 zdq2|mBro$bP}?d|L}8S`uW`&+Nr+@I@4(P`1?eJ^M6t^lyNe&VptE>MGNAnq2=BPf z*|D8vd1^W$ObTS2dWfJD+st->#^DP4C>xr>9D#wPp z6;YG_e&ah6$GxNV_up!GM4j}m=JU_mFTG@;j=9IGUr`uP;M3Q?v;DYtH2?nF_7BJt zPOd-yoE-;SrG!pL$tVd8r-^Z?L7XJ^fDEDCQq8(L@SyfV2OAR=hWN1$jjwsh&S(lf z=!pmj_}=p-KL4c+D|QX=8CK=@g3Shw@ zLjW$YQV~HX$VzmEp{_{a>T70?J9=o_HobskqT0B?YM<~rL43b3#DD*E&5$88&;sSoD1UzZM-_n(cgB$>?bbTuK(_@50&q@wYFiMJ-LGuq3uwqtj5N}Wh=r1U)et3 z6&>5AE-F*N1w|3|DpjW?Bmxk^$QYlVkkb=VcbrG&^JZP`*eanrpch{|&%Of;XA&%% zMUfH~QxPFzBvn=+xC<{Y=Bqgl^8;mDERw`nB1m8TM*Yn9>fv+PvYG$$`Lze`v$GVC zINdVvmE|&-qz@|WiE0|$k>p>c5*r!{zxDmu*BrXDbfpdJ`CWIoKfAj6{Bt}LjZy`D zkK*(}!7($Avr3&kT02ml0_b_taakA;V;BZ1J--MLe}2>U*SZL3UF$N< z{SVystW%df{E%&^oWw!vikM+iZQ`OYZhFV52Ez2lSFPB%&eA(v^!(g$|C(J_CM&yf zz5OreFL>{^dzI^th2=2SPr158|T4ln*QiWSKx-QcI5H0#daN#Fs=8~lso^!tW zwV}^^eC*4gAG+}u)itZF?(euRee=rg<&w($&T2mw7Jl@TQ*+7EGpp>ce`)x_f13a0 z&kx`Di@vpU-1ci%Y;j8jrbn*V3eJnlQS3qS0w;5Ff^S@F8QIyr#={t{*d>V^u71%u z`?Osjf4GEtDX(mgrVnm5*o{9Qc-tFyx#;8L?H1SYo}R_+4BUw{`u)H#w_MgY#9bFF zHS8uEYfdx}5YDsiFnduI0#j)<9_sRypcjN+>`sILz&YJ|=ga{I?JT7t3}I@L?byz? zY~jy7Eml1#9=PA0oZxfv4f8@OK!6WFwS9cSoY}iXJqF&Wv-96Kc+Vf*-Xz!59nj5y zv;&!z1C3g9pDr@+J%8DpLFbP?Y#)F0xJkFfVjUSW`L#%k7R!78x{-HHLm1$vKR$fJ zwKY=uq|@H}1|Rv@%u#PGT83BuXaDCuHJsgj(Gq#jU)J@!?PotRdi}M1g>AFXzQI3# zZ1SkL6mR>J{~Y-Ir$>`$w#?aGd-3^d zO+_Id_1axFthb7lA&Z@Oj1yw{F2N7~YyHyY0RS9+@NSzn2t6@E%fQUYC0ef%?QXh! z=+O+!8wh3z8kjQDO#)=wm>Q^Oub+c+=}gFaKI?W;%z2YJKV_Kc9N*u^s5P*(q;+Mx+-gPu~pkSUhB>GscZK zFgss#W&F?st`^WrYM3LD(OU~MMtZWk@B2W#d(%fDzV}`8ufL|6mVwVV|EeeL^WHQ6 zYnKcaE;;kP^<~Q=Nb{xMciv3T+n#;e_;uIRh$!oR7Jq-$lg zmxWp5!7K*}GiGD{as)tyJg6<3tvo@3%wA4lR2$#0&W?i(V++D8!-?{uX9mYVo!#W8 zwJ+JHpRjD>dh4P4D?7Fct%U^@%WV@9pcyKm*b77=n1jLSd}hqzmwJqA$a~d@s48$w`^traM8uh=_&l;#kI9-_|EZ}Lk|mnaP`dX zf0$ghJknquqvTEe;2VBky6_|YM9`??EjN`u`2K-?SI$5E9L?IzCXR+0ReTFd4lX2q<0F_GRUw8TUKL$BcX2W{pBI2Xtl=GV$D`c zlpE7%S#Z*+J)nZmesbu6`&=zkv6#Z~?Qiry_OM}Kb3qq=YAU|OvgsYCHo9+n(I-bA zxUURaepKPU-~PtXV~@Cn+kW(ulSWIH!icg2E!J5EF76!XLeDd~SD#1YcTcYpZ+Ok{)mN4F+9&$@6^*NYHnVKG{Cm9K{SF9k zyK8FCy`r61m;i9yHRZz(8Qo{)*ahbf{QBlnquJv-(^Krc_YPh8?W*nrTyf>J?d+s9 zOLv@|=8XSq3f6W=2bbC{oO7ys(jWnjJgPG=*u%ZK^_D8_d@cZfcYC?;nFZu#;J7H1Ja&tG()gk#AmBS+|z|{W#LVpt|x$GwG&B(oCuZ0)swPUTT zZnw~&vOF$fJFHaUvojGzpqKO0j3GS-+Zem!@@kDwPli$^?%X-&dQUwi{`_F#L2li` zzWjyCm%mWiZTIN)hll$g5bm>IwCl=9h@DB)%H5)4-s1oIW;gH9lo}P5g(LIrFd*B( zARx|Q+txt${>UsHf`C_jzk24``K}sYi1QaXfv-4cTQ+%m^K`x33Yl%WZ41X2IdOfz zr2F7{KO>+QP^@g$92X8f%=_bAB_gQx@!@&a#&v!e5@a4rD$Iu;**i3B+c-LI2Oc?E z_dzVA(e(oDmM#ybCQ5HSvOU+~1H{Y;Gnk5w%SPsjUbhV=Ck1KMkdjsv8o|VM*^1qq za>bYv-`8um>RY=|X$X+uZ+{;6$|cr^KbnC1OO^!}eQ|36yWg_WTC>`D>3RFw@{QtPLv2_bOXzvj+sc+9eFP?wSLIB%A zVbSh;$O8`wj(oHKx;OegMeC?UoQ&d4uFE1xGgDz0lCV&Q#00u7my*JOdX~59s#pT; zwwjsIw=p5Fn0v2nHQ%IJmXMcVeB@=Q~t zTA%It>d}Yc(T7Vv`eA9oLiPVX-aP%xPIuJE3^)M*)dslNp~h?k2ywC&gfws54t$!O zj!4JIF{cg}U<9Qo<|u{$V+$-lfa%FF&U~aoGt)GNJ2Y&y+iH3$0)jA-0DvUNmpFb$ zO-+PA*wt%->p5dsE*M1atz z#mgP9Lp!zx8XKs`g_Zze;p~j)@0WA>e0eD!BfLLvfz_Cij@?6?grE$NBh)Gfrk9T` zwCXcz+m@hcD3&=Q^zZ{sB9IWcaGBMZk(EBSajkgnX-?>XR<}Lgx|J#|TM?{S>95?~ z-+Nzg-vc`13&PXRnmX;QsSWGg8?RgNgDc8WL_NHpykeC>47F@ryuzBDR<(ZQ0;9-? zG#9fy5I|#;CnHmmW~M^MlXO7rS%D+r3>Q;#l%%Q2FkXhZRDb^f9eP-h3WIv#`EDvd z0f6Vrz|WiXOiWl8{oCM!54e|qdzuTKpdFm|!PXD1tmuRBIx6sUri9}%!Lid)Vbq~v8cMnku2>oD>}2aviVuNcc3Kt)T#4`0?k?0_boa*=L?@g)>j(U|xAZCH zMv*dwE`MID$yPn#u6okVJo}Y<_(vSoIOX)2#Y?@9eth-5`_(SIU}2}tc8<7Y0v7o6 z=t36Ao`2f8@|&Ya9lFcyzZn34HyzXX)_-p7l8g9VpkE!KO_EwOy-Y`5KiLy9*4eWcLsN9IRCWJK{|NXeMgjZ92{jlSBZPg+s$u zyD4X8q$0ffs@k?~1{Z%YS@Im#Qz{v#W#jOu)o!bq>5!60Bq8PGteD_nS#;&qQw0hI z=Quh)?mieQl_j}T?xLX{Iwt^9sc+*rY;1wunw66~B2tuL00k&w{W<{vzrVxXYx$yE zZ!Rb5gh0uJGk^#S7Dn$nqd7Q~Eb0CCx_aTu>@zkN0fOVQMN8~@(tR*VKaMB)-nLeq@(&kOv3`7G`-hJ@IQ(C{fck@G!Z@lyylb^qI5&(`pzV+Z^8%3|WKPz_i ziAaXjY3E%4gCNi0y6{uey&1r31prJ<^56Wb*8SJr_wtSAtzay$=cx-mGBF3S*rF_s zqBVud)R5PvFi?d(+-|WrSuifY*=v^p07W6(dUIl=?6ZGerO6VMogg_FShzaR6{%J8 zefa9vwm@$|i&_ikETF5MMNvepx>DUCbB*ngH9ln-VNWNCa0af!(nnx1tVVM&=>Y*4 z1GmKDFBXb{UGkO28;e_!xk3m44B@0xidN8%emHpdU8UKXSpJ|?jvG$vG}OF9YldQ$D#CGa z@iK=onwg1WBd1(tj?3b2)wI*8*<{_3mAKGFZGaC9sxN$bYL7jWll-_7TgM#Ry7lJD z_rKl0eOr&{+#3$>EZ@y#0^(Xo2vDtY*Jbe#KYZczD-QO$uX^?wd+Qb}p0pqTU@(e` z3;4P>b~1~Da^h5M*1(58I(gtLdKi!|KJRSZEHav##$e3v-#5rgWxmJ0hLO4I$x5=v zOk}7VcfIMlf%qT$?(gTPN5(B8?BNPg#^C`X9M=xjnQ4xC?Flcy2~D>X z+btRRYGx`*tHu!kwHq=F-aW^L9t?&V)eChV(jWdLq=TMz)&L=+Tl+#Ro2E z?zy+&qu=q{(icBhO#vSSz-C=_+Ooj@=<2UcjoGN(mfN?4QJCYq^;y|zCTWcNyD#ct zBM|@vzM9$*ed04S1>E9YXEXufl8Xlq-e>W5F7Gc~`I^_Z2m70PL)+QuD9#=D;D={F z`ib60)6F;3V)GXOo_$(e{hxz{FC6xU_Fnt=W|Oky-{*gL`t0*|^uFzlwLDf%arC)y z+dbk>@M2CIl3k8c~mI+qkR|WiucFq zgT5E+wtElstWu$$|8n}gv-|J5)6Om|m$~O90+9K9r3!ohxsAQ|%{SuD|ErW4{gm=G zDplTYORueTF)vqn;3*l(q^&OkAe?w|=TrY~B-=dkxbv_7GU9v2I4d>o`>Gkq9M~#W z>IERE^@&zXwi+^PJqUoc8uvX2>Jou&xvBilcPEUyzJ7J+>%2eR=N76WSvEWBt- z`>*6sdc%>)G>$EZpwRmxr+kCKCD%5?p8It6*}sz=O;0>p>WVvMStMQ6jsYDP7c3T2 z6SBC`sYq!+035aMKojw{93}(BlkS5kV`#Df5gJW(^1nF_3j(q&3>BR7f%fPBqtSC2 zl!7ZRt6cWAau~1->vY_>gCGw>#WX{a4xABkzM|bRCTJ^ZR#huoDkT_>S zC{tcT>@&d!{;Bnui|cxgfz>aG6W%g9HOWe4^gZRcJP3$03<71wwloJSO$y1#`sFJ3 zIxR(J8@up-U|8$>w`bqQj$ZWYH7BBH``1Q7dL8;5q8K=yz&$1k5 z1=wX}c++iLdd8sso=+6wneQBb@V-(Ex>aku)mEhvE?VSY|LaZt1M(ldZG`VRb>V~e zmmsYyw#Qz<&3CTv8A1M^eu&`Jd#{|D?alis8*Gie{W+ZOJRsNTssd7UG3B6Hi#)&h>J{)!zwkdLHeYnG{gXM+y&W#5`vv{)9xsiz)4ympNR-Hb!6|6jLg6KtyuI|@%lj8EiH%*C&rTw*B{S%84EP zg7?q=%Y$y3mZPWhC5mBaNM7=_9mgEo`XAl)L+6bD>}5!vE3duHRM9 zKWh;H9CK{r+gEKW{I%0zSAKK&nyUwMPCBAiFcXJ{oAyPjplVM)3{(tSH(QM>rZQw^5Xv&pAVe1=nucE0>GEPvGtVGXA*$4!G80r{&$`+9UB^+ zhmZcPym^C_u}VrMs`km{D}p%pR{#9&*1nINw^*rOO@0tH1I=v8n3))AeaNBR3bKWxuy?Kk!-P9*3Nhjc}v)ljrsYW+%?cTec^WQ&I zpV10m_A0N7MFhrp2$~`9LQSWly9F>hDnED0^l>M)cP>lKI-|S;I$#$|GEGuLLOA%< z-aF4|9P`$8HcL-{zIC^51zz{phzgd*1f?Yx}Oh zwtvee0mV510DSO?)$_;xZ|V&`{-}HMTb9MX$)6s7ao$)20IQyG|M}d7>(|+@JG^z- zcQy?U{k<-q=iv{(J^G!?Ml;8idW#bjYdP!n&*x+#p-#%cdDiVcSgmAq!lm9NX6A0` z1%QATeZG1AhuSya%;W_6w-gbPeP)Fs_hn+J{o!6+d7~LTv>``nwg0Z3T%nnS(UzP|IyS9gHm zfqTpM-Cch4VOOa_=VV=_;<8woa7VR`Sxuh2(Z?ts#R6%Q`MTP z)}-qiGaw~QY!_R%3Z?MCS9DgtXut2Y#Tz#y1vhW{-P(N*@UHps;3XH$kMsWsU~ou& z{IlCnIAylGy^V~p(O|PPe0GL!-E95k!SZi!?wgzt31$&bgjtxq#)y>v5i!IUC~s1z}dN~Xw3PhW`&xxpBBACwW;4z1ixu6jy9p2ve(&^T`{ zNwmwZa_@bjp%I#$!8NPJGf!Jl1XA{F+JOKxf1Wzw0~P20vOBq2~LKtvT48fg*b5jGuA%9B%)?t@u%EUht7VesWL?R$V9+bY6h zGE&bz}yjx1T561nDhzuamIH7Xm35CGn5j%{OZLu&UWH7Nk=x!K3Jfc4NndYhQ>R5jV zf^^RxoiBf(?}^7OvlEX|*}}5f*tnB|?22>TvN1^a5CL^Z{EBghzCJ!UYHeEQN0Cvw zok%+(pimS7Y=w9Oiq<61aY?Zy+G18bpJH3xv==z(?j=>tFK2unQ?NyhtNv z=D2L>3die^>!R!oKqK~RKmwC?DI0_*6OaK-M9 z*Cw}&UWbH*&#&_4C@0+Y21rCOJSxVQSkFJ@g?=7!)fb{CJEBx(yX|eaTU4oFyG53b zPe0~&*%M|bfb_m@neDNk-D#>)mA9M9w&AHqe9&9*<`!>i10>>v2o!f8BrY(Dn|VrN zNk8d?7ngN#*90tsH4qUThe=75s+gUQioO`TAccmF-MK0#!bKLdbrnMEO377GGpJ1kBg1OzFw0?R$ku5=z5 zR4P0SRsW#1V_T3^3ZtyUm}VxM$49xsqmXQu`Cb|;6M%q`q|ncqtUCOIX0gLs9}5E* z9Ohd$g+y~j)QjD1YBdpt%68D}P+Trka6pDpNWQ1cCS)fs8>AP}+SE zb=)p%m!^>k$~*`XicO#h03=cWfGs73%?R{@b=@Xpnm!l=%CZ?_Xr}RWH(Wz)x`pSf z=4=E2GU=#G!=rW>sQPSV63<@7wtM<1>(WbVPdwficnTma5{}JJ|De@rtJ$dt0HP>P zAEdDbw(rCAWcV^7i@-S!jl`V`8Dpp<6{Co~Pf#2lwOUO%H5max7)U^X02VEAJx@+< zk9Jxika zaD0)~m{sK}V;o~f*c=Vrjg=l-WX0VF0nqm)0QdmA?CG?dv}HrsmHJZ@Ez81%OYO#V z+4e01$YR0H8yA?(4LgYQq{eH)99@V9|0hGa)yxmw@E8 z<1AV-yAM)#MIJ#KcoBq3M!8nBEXEmyp^8FfSPrPwx6(>T#p#3b>!q@kSJcu2HK>t7 z2*LAYTo^rxBRePr^Ey(7MRaTJYH`=^oHrfa114UxT730OwL5PsGl7h$FjN}wSkz(h zFFQ@;YtMB{ObFJA(+9P|YOW0Fgrg{ejT`(p*-ELrEk{Po%rRCY2oJ4>>~v)2DU~Zs zSgh5E0zWSoQNYOc4HJg6F77_4gj9K@w(i2)=FeEK%T6oLudY_nc6g&MeUCJVKS$GZ zgr==Qr7E=(!wNt z{md!j^rZA!g+n3((+L0tX4Rny7@|s&YzYV%111MFfAquRsv6aNw{p9^dV$x~h~cRFYCs_w(Jo z8?XCztE5!s$&=^&^)KsbrhWr!6%j4AR!Ap43~eFC1Y|7w~r{`9J zTHe-ZCexifSX&5En+4Z3o|OBC3G39+Sc1Hi3VTM-An&uPi(UdY@=!i6Uu=W z2riqo7>)8CULZ17w;h(XVo5Abtr5tSj(DD2)2gOb(M$+b`i~+QQpC{HwCZ|s@#1b; zbZd~G^%4pk$cp?djLZ|aooYITU{RMXZS8>u0;uiOi0GMYK4OSMT<^Djl zt+Gv$Yi>KN)*zyJtPmJ+Q<)`4U)({Rf3?+EtMkc|8$3Um@4p?{ z&HE>l?M6^KNkZ)1@giu^G;057zx`%Byi5;IhLchyN+ko9#Ws9&WfuoIB^J$h8Vuwd zIc6esoqBvWUw?Harsr3dU6qgXjmFoq5Dt#{4w^%ikPfg)zs*Tn1u`I|+y61f(!7zj#@ZeDE1yOnG92YuDrBPn&=LWB88gqtBXoawwo_hP5 zpZvo9^Y2&wnm_%eb!DlV^<528oleuVBx%mkG`TQvL2h=SB4OfuE-~ufkaL)>bNN)I zr*1Cc#zf=DUYCMrTCzT-@yhc|_sfvwuTLY-BeQoOyetnojsS8_F-Fu41ToR|>fA&` zm>mt5C0(1Y!FjRToXg6wpg98b#KYoTD78=Tf>j_I1~*N_#8Vg_B%G)nV(0`<9cEZV-+aA%-`#^=uT}A3`K|{uS&Qe_rgC+?zW_}8C%xX_H^1++rvHAq z{N4SN?lOYhJ?fYK~BHimhrhCTGJkoV?^Ay<@%#Nm+U7XSY zs)kn)6xq!b2l7xr^f@uY_o$tt>&U~v^OU+|Zl-B+0n(`SqC@UBJGu@$3{Gf1!7+_j zcF_evDszO1XK!{m4177|ubufiIOnAAOz3HF_I;PQJ(K?op;(D?+NB~}e6xAx;p|RF z9>>JB#YF)H?luDOL({~IHEm6}y0tIeOy`q3yzn@<*u9u19?oud;+rmg zb|6WqrHLFj^ElFQ;kt{PZ~$cG<~-5q!U+TBGl^kewAmJ)9Sgf!??7b=zVZ zooMdTYU6JS++ZDw;$*YYZHMdN*rlYsU#qr7(2~zup%N0=Y*pK;>vd)kVEc`a)=7S! zi)%P)Jz%+=9v)46j-}qy4JB>t_O@H;>&1~vH2=-sta}}F?{i9y3wO6)?_x|qDtT#GL)8;K5K?z-)Oo^ISy?$${T8yN4Pt$o{xry!~DS;yV>f)3sVZp zcz&g~JC&_b%3M?EeD%g|wm81bL0_7g==rVQJ*ap~aKSuc?>?B{lD1L4ojT79*icT(Zvl_N8734>e^OKBP*@OtGi9&CDaY#Wqi=8 zabm62b(mAIh2(=oC?PmE0jst0imG9JkTWbHBQrpATpu-Lzg4_DTlkj-3kJWTyYUoLDcE=%!-$ki&b5AlWe5Bl)J z{Z^e%&RQ%t2C5c!MbP%IeR-Mtt@e1;@$mB`qxhB&Px|nq>-RXHs4+MmTt!pK*rZAX z<{)F=Dy7M`N-0>QLi;4^7iFM|*eFe_)G|H@YogCc2$E$-!Q*U;@j|y!+MsQfHV~DM z(40wSeg+XRQMb}sqgyLu&@{>62PicTeX$*;Pe6Y3aB&}m93PC{l;Tz>i<5P)jK*rM zZG%;(3ZE-tnF?n>Hi-I-GFGj7Z5p&I6`aPxb;5~&8@-}^#m%72R$EQ|R@nxj^5M*{ zCwy{>#^m8=_n@s++pTIEta_CeDDYbrGA`vJA;e6)-5YDwcBk4FU9aNx7#Phc4Is|>!#HY zkGfl_exs1oFrpFH4c8B!x<5QA+hPA`x)%LbNt-8l^RJkz2F~8!vW+ep0a3YG)&O?vZ5J&@ zM5|1i&!SQZ5TlWT^99_;%R@aUTd!2Hyl=9rP}Q%@@7ZjUd@SYv<+3r|a+_r2{!T@A zBU$B9=|DyNKA?^&2RF_lLe^P{#(iag;e6$ZYOFpgwW1v3VX6@1g7+Z2a3N*tif70-)+BqNC^5CF(q++k1QYH-wsI|D%05tW zgF?3IlGKTCOH45{J%Wtw%P5$qVF5T(vCW2aiX0ks+rOZ-WLp9fAO?go2PA!&AqdcS zjuvF@5e&jP8*PH@HU<0KaQyC2@k{n5i`s>wzzdaK)S6EtvhR3*FeTUCiV!Zh9Ow3| z5NI5O)3i8O?fSBTAj@jbsf+2(;#&-v^k1t4sk9vf3wjF*O>gN;QC~)a;1K}o>>$La z#_7={)&duRQVop-hLmjMidHS>@?WT0KPi_hmH(^xGAWu%61Gk$45!UU@nMP^fphLJ zG7Xq~TL>@bBtnV=(|J+5t$# zSvKNYhnSKFgd4epz-iJcXU?6TZ^y+91znVk3RracR$jJ0eDf&K%*aVEK`}I;f(A;F zF$$VslG-ztp9F<+m3tWrMPw|X)Z$-QdJ4;SKf(@)b~t2hmZ?Oc^#JN;V&VkWA1iwt1Z#gW>;{Y)A{?_mruK diff --git a/testsuite/etc/more/give-me-more.txt b/testsuite/etc/more/give-me-more.txt deleted file mode 100644 index e69de29..0000000 diff --git a/testsuite/exclude.encodings b/testsuite/exclude.encodings deleted file mode 100644 index eab0709..0000000 --- a/testsuite/exclude.encodings +++ /dev/null @@ -1 +0,0 @@ -encodings/* From ae393e2ff3dabccbd2a9eb4b826dd54338df930a Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sat, 17 Mar 2012 11:55:15 -0700 Subject: [PATCH 10/16] Trying to make parallel work. --- s3cmd | 67 ++++++++++++++++++++++++++++--------- testsuite/blahBlah/blah.txt | 1 - 2 files changed, 51 insertions(+), 17 deletions(-) delete mode 100644 testsuite/blahBlah/blah.txt diff --git a/s3cmd b/s3cmd index 32ba859..31c6153 100755 --- a/s3cmd +++ b/s3cmd @@ -1906,20 +1906,13 @@ def cmd_upload(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = [] if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) info(u"Summary: %d local files to upload, %d remote files already present" % (local_count, remote_count)) - work_info = { - 'remote_list': remote_list - } - - - - # if local_count > 0: # if not destination_base.endswith("/"): # if not single_file_local: @@ -2150,7 +2143,7 @@ def cmd_inflate(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = [] if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) @@ -2158,7 +2151,8 @@ def cmd_inflate(args): info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) work_info = { - 'remote_list': remote_list + 'remote_list': remote_list, + 'destination_base': destination_base, } if local_count > 0: @@ -2336,7 +2330,8 @@ def cmd_copydeflate(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) + local_count = len(local_list) remote_count = len(remote_list) @@ -2345,7 +2340,8 @@ def cmd_copydeflate(args): work_info = { 'remote_list': remote_list, - 'local_destination': local_destination + 'local_destination': local_destination, + 'destination_base' : destination_base, } @@ -2572,15 +2568,34 @@ def cmd_manifest(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) +# remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) + + if local_count > 0: + if not destination_base.endswith("/"): + if not single_file_local: + raise ParameterError( + "Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).") + local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base) + else: + for key in local_list: + item = local_list[key] + remote_path = item['file_name'] if cfg.flatten else key + item['remote_uri'] = unicodise(destination_base + remote_path) + item['destination_base'] = destination_base + + + s3 = S3(Config()) work_info = { - 'remote_list': remote_list + 'remote_list': remote_list, + 'destination_base': destination_base, + 's3': s3, } @@ -2629,6 +2644,7 @@ def manifest_worker(): def calc_manifest_info(item, work_info): + size = item['size'] filename = item['full_name_unicode'] @@ -2649,14 +2665,33 @@ def calc_manifest_info(item, work_info): uri = content uri1 = uri uri2 = u"%s__%s" % (uri1, item['file_name_extension']) + + remote_list = work_info['remote_list'] if remote_list.has_key(uri): remote_file = remote_list[uri] size = remote_file['size'] return uri1, uri2, size, True else: - info(u"Skipped %s with no remote file at %s" % (filename, uri)) - return uri1, uri2, size, False +# info(u"Fetching %s for %s" % (item['remote_uri'], filename)) + #Maybe we just didn't preget the list... so get it now + try: + remote_uri = S3Uri(item['destination_base']+uri) + item_list = _get_filelist_remote(remote_uri, False) +# item_info = work_info['s3'].object_info(remote_uri) + if len(item_list) > 0: + item_info = item_list[item_list.keys()[0]] +# info(u"Received %s for %s" % (item['remote_uri'], filename)) + size = item_info['size'] + return uri1, uri2, size, True + else: + warning(u"Skipped %s with no remote file at %s" % (filename, uri)) + return uri1, uri2, size, False + except S3Error, e: + warning(u"Skipped %s with no remote file at %s" % (filename, uri)) + return uri1, uri2, size, False + + def do_manifest_work(item, seq, total, work_info): diff --git a/testsuite/blahBlah/blah.txt b/testsuite/blahBlah/blah.txt deleted file mode 100644 index 907b308..0000000 --- a/testsuite/blahBlah/blah.txt +++ /dev/null @@ -1 +0,0 @@ -blah From 56d686ebb5dac329cafa8465ba56ea5def48c738 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sat, 17 Mar 2012 15:57:32 -0700 Subject: [PATCH 11/16] A different path of making system faster, local caching of the remote directory. --- S3/Config.py | 4 ++ s3cmd | 194 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 135 insertions(+), 63 deletions(-) diff --git a/S3/Config.py b/S3/Config.py index 14c70a0..4454c5b 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -34,7 +34,11 @@ class Config(object): skip_existing = False recursive = False flatten = False + deflate = False + use_directory = False + blob_directory = 'blob_directory.txt' + acl_public = None acl_grants = [] acl_revokes = [] diff --git a/s3cmd b/s3cmd index 31c6153..3cdba9c 100755 --- a/s3cmd +++ b/s3cmd @@ -856,7 +856,7 @@ def _get_filelist_local(local_uri): for root, dirs, files in filelist: rel_root = root.replace(local_path, local_base, 1) for f in files: -# info(u"File %s (%s) -> '%s'" % (root, rel_root, f)) + # info(u"File %s (%s) -> '%s'" % (root, rel_root, f)) fileNamePrefix, fileExtension = os.path.splitext(f) full_name = os.path.join(root, f) if not os.path.isfile(full_name): @@ -1906,7 +1906,13 @@ def cmd_upload(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = [] if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} + + if cfg.use_directory: + remote_list = parse_directory_file(destination_base_uri) + + if len(remote_list) < 1: + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) @@ -1948,7 +1954,6 @@ def cmd_upload(args): else: output(u"skipped: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) - warning(u"Exitting now because of --dry-run") return @@ -2035,19 +2040,22 @@ def do_upload_put(item, seq, total, work_info, uri): # output('key: %s' % key) destination_base = item['destination_base'] - item['remote_uri'] = unicodise(destination_base + uri) + full_uri = unicodise(destination_base + uri) + item['remote_uri'] = full_uri if remote_list.has_key(uri): remote_file = remote_list[uri] - info(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + info(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + elif remote_list.has_key(full_uri): + remote_file = remote_list[full_uri] + info(u"Skipped[2] (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) else: - info(u"upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) + info(u"Upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) if True: do_put_work(item, seq, total) # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) # time.sleep(5) - def do_upload_deflate(item, seq, total, work_info, uri): size = item['size'] filename = item['full_name_unicode'] @@ -2066,6 +2074,7 @@ def do_upload_deflate(item, seq, total, work_info, uri): return True + def do_copy_deflate_into(item, seq, total, work_info, uri, destfilename): dir = os.path.dirname(destfilename) if not os.path.exists(dir): @@ -2090,8 +2099,6 @@ def do_copy_deflate_into(item, seq, total, work_info, uri, destfilename): return True - - def do_upload_work(item, seq, total, work_info): cfg = Config() s3 = S3(cfg) @@ -2099,17 +2106,17 @@ def do_upload_work(item, seq, total, work_info): uri1, uri2, should_upload = calc_upload_remote_uris(item) if should_upload: if os.path.exists('.blobcache'): - local_filename = os.path.join('.blobcache',uri1) + local_filename = os.path.join('.blobcache', uri1) if not os.path.exists(local_filename): if cfg.can_link: try: os.remove(local_filename) - os.link(item['full_name_unicode'],local_filename) + os.link(item['full_name_unicode'], local_filename) except OSError, e: cfg.can_link = False - shutil.copyfile(item['full_name_unicode'],local_filename) + shutil.copyfile(item['full_name_unicode'], local_filename) else: - shutil.copyfile(item['full_name_unicode'],local_filename) + shutil.copyfile(item['full_name_unicode'], local_filename) do_upload_put(item, seq, total, work_info, uri1) do_upload_put(item, seq, total, work_info, uri2) @@ -2117,10 +2124,9 @@ def do_upload_work(item, seq, total, work_info): do_upload_deflate(item, seq, total, work_info, uri1) - CONST_sharef_size = 60 #A bit larger than... 40+4+5+2 -CONST_sharef_prefix='sha1' -CONST_sharef_suffix='.blob' +CONST_sharef_prefix = 'sha1' +CONST_sharef_suffix = '.blob' #============================================================================================================================== @@ -2143,7 +2149,13 @@ def cmd_inflate(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = [] if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} + + if cfg.use_directory: + remote_list = parse_directory_file(destination_base_uri) + + if len(remote_list) < 1: + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) @@ -2153,7 +2165,7 @@ def cmd_inflate(args): work_info = { 'remote_list': remote_list, 'destination_base': destination_base, - } + } if local_count > 0: if not destination_base.endswith("/"): @@ -2231,7 +2243,6 @@ def calc_inflate_remote_uri(item): info(u"Skipped %s with size %d" % (filename, size)) return "", False - content = open(filename).read().strip(' \t\n\r') m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) @@ -2239,7 +2250,7 @@ def calc_inflate_remote_uri(item): info(u"Skipped %s with content %s" % (filename, content)) return "", False -# output(u"Blob-reference %s with content %s" % (filename, content)) + # output(u"Blob-reference %s with content %s" % (filename, content)) return content, True @@ -2258,29 +2269,36 @@ def do_inflate_download(item, seq, total, work_info, uri): destination_base = item['destination_base'] - item['remote_uri'] = unicodise(destination_base + uri) + full_uri = unicodise(destination_base + uri) + item['remote_uri'] = full_uri item['local_filename'] = item['full_name_unicode'] item['object_uri_str'] = item['remote_uri'] if remote_list.has_key(uri): remote_file = remote_list[uri] - info(u"Inflating from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + info(u"Inflating[1] from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + if True: + do_get_work(item, seq, total) + return True + elif remote_list.has_key(full_uri): + remote_file = remote_list[full_uri] + info(u"Inflating[2] from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) if True: do_get_work(item, seq, total) return True else: error(u"Unable to inflate (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) return False - # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) - # time.sleep(5) + # output(u"DO THE upload (%d): %s (sha1_%s__%s) -> %s %s" % (seq, item['full_name_unicode'], sha1, item['file_name_extension'], item['remote_uri'], item['file_name_extension'])) + # time.sleep(5) def do_inflate_work(item, seq, total, work_info): uri, should_inflate = calc_inflate_remote_uri(item) if should_inflate: - local_filename = os.path.join('.blobcache',uri) + local_filename = os.path.join('.blobcache', uri) #local_filename.split("/")) if os.path.exists(local_filename): - info(u"Inflating from local (%d) (%s): %s <- %s (%s)" % (seq, uri, item['full_name_unicode'], local_filename, ("Link" if cfg.can_link else "Copy")) ) + info(u"Inflating from local (%d) (%s): %s <- %s (%s)" % (seq, uri, item['full_name_unicode'], local_filename, ("Link" if cfg.can_link else "Copy"))) if cfg.can_link: try: os.remove(item['full_name_unicode']) @@ -2305,7 +2323,7 @@ def do_inflate_work(item, seq, total, work_info): shutil.copyfile(item['full_name_unicode'], local_filename) - #MLF:TODO:Should also have a local inflate location? + #MLF:TODO:Should also have a local inflate location? #============================================================================================================================== @@ -2330,8 +2348,13 @@ def cmd_copydeflate(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) - remote_list = {} if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} + + if cfg.use_directory: + remote_list = parse_directory_file(destination_base_uri) + if len(remote_list) < 1: + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) @@ -2341,8 +2364,8 @@ def cmd_copydeflate(args): work_info = { 'remote_list': remote_list, 'local_destination': local_destination, - 'destination_base' : destination_base, - } + 'destination_base': destination_base, + } @@ -2382,7 +2405,6 @@ def cmd_copydeflate(args): else: output(u"skipped: %s -> %s" % (local_list[key]['full_name_unicode'], uri1)) - warning(u"Exitting now because of --dry-run") return @@ -2446,10 +2468,6 @@ def cmd_copydeflate(args): do_copydeflate_work2(exclude_list[key], seq, len(exclude_list), work_info) - - - - def copydeflate_worker(): while True: try: @@ -2478,7 +2496,6 @@ def copydeflate_worker2(): q.task_done() - def do_linkorcopy_from_to(source_filename, dest_filename): dir = os.path.dirname(dest_filename) if not os.path.exists(dir): @@ -2486,12 +2503,12 @@ def do_linkorcopy_from_to(source_filename, dest_filename): if cfg.can_link: try: os.remove(dest_filename) - os.link(source_filename,dest_filename) + os.link(source_filename, dest_filename) except OSError, e: cfg.can_link = False - shutil.copyfile(source_filename,dest_filename) + shutil.copyfile(source_filename, dest_filename) else: - shutil.copyfile(source_filename,dest_filename) + shutil.copyfile(source_filename, dest_filename) def do_copy_from_to(source_filename, dest_filename): @@ -2499,8 +2516,7 @@ def do_copy_from_to(source_filename, dest_filename): if not os.path.exists(dir): os.makedirs(dir) - shutil.copyfile(source_filename,dest_filename) - + shutil.copyfile(source_filename, dest_filename) def do_copydeflate_work(item, seq, total, work_info): @@ -2510,9 +2526,9 @@ def do_copydeflate_work(item, seq, total, work_info): uri1, uri2, should_upload = calc_upload_remote_uris(item) if should_upload: if os.path.exists('.blobcache'): - local_filename = os.path.join('.blobcache',uri1) + local_filename = os.path.join('.blobcache', uri1) if not os.path.exists(local_filename): - do_copy_from_to(item['full_name_unicode'],local_filename) + do_copy_from_to(item['full_name_unicode'], local_filename) do_upload_put(item, seq, total, work_info, uri1) do_upload_put(item, seq, total, work_info, uri2) @@ -2521,7 +2537,7 @@ def do_copydeflate_work(item, seq, total, work_info): source_filename = item['full_name_unicode'] dest_filename = os.path.join(work_info['local_destination'], item['key']) info(u"Deflate copy (%s): %s to %s key: %s => %s" % (seq, source_filename, work_info['local_destination'], item['key'], dest_filename)) -# info(u"Deflate copy: '%s' to '%s' ", (item['full_name_unicode'], work_info['local_destination'])) + # info(u"Deflate copy: '%s' to '%s' ", (item['full_name_unicode'], work_info['local_destination'])) do_copy_deflate_into(item, seq, total, work_info, uri1, dest_filename) else: @@ -2552,6 +2568,44 @@ def do_copydeflate_work2(item, seq, total, work_info): #=== cmd manifest #============================================================================================================================== + + + +def parse_directory_file(destination_base_uri): + blob_directory_file = cfg.blob_directory + local_filename = blob_directory_file #os.path.join('.blobcache', blob_directory_file) + + remote_list = SortedDict(ignore_case=False) + +# if not os.path.exists(local_filename): +# #Check for the file up on the server +# item = { +# 'object_uri_str': destination_base_uri + blob_directory_file, +# 'local_filename': local_filename, +# } +# do_get_work(item, 1, 1) +# + info(u"Trying Directory File: '%s'" % (local_filename)) + + if os.path.exists(local_filename): + info(u"Found Directory File: '%s'" % (local_filename)) + + for line in open(local_filename, 'r'): + day, time, size, name = line.split() +# info(u"%s, %s, size=%s, name=%s" % (day,time,size,name)) + + remote_item = { + 'key' : name, + 'base_uri': destination_base_uri, + 'object_uri_str': unicode(name), + 'size': size, + } + + remote_list[name] = remote_item + + return remote_list + + def cmd_manifest(args): if (len(args) < 2): raise ParameterError("Too few parameters! Expected: %s" % commands['inflate']['param']) @@ -2568,15 +2622,22 @@ def cmd_manifest(args): local_list, exclude_list = _filelist_filter_exclude_include(local_list) -# remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) - remote_list = {} if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) + remote_list = {} + + if cfg.use_directory: + remote_list = parse_directory_file(destination_base_uri) + + if len(remote_list) < 1: + remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + + # remote_list = fetch_remote_list(destination_base, recursive=True, require_attribs=True) + # remote_list = {} if cfg.parallel else fetch_remote_list(destination_base, recursive=True, require_attribs=True) local_count = len(local_list) remote_count = len(remote_list) info(u"Summary: %d local files to fetch, %d remote files present" % (local_count, remote_count)) - if local_count > 0: if not destination_base.endswith("/"): if not single_file_local: @@ -2590,14 +2651,12 @@ def cmd_manifest(args): item['remote_uri'] = unicodise(destination_base + remote_path) item['destination_base'] = destination_base - s3 = S3(Config()) work_info = { 'remote_list': remote_list, 'destination_base': destination_base, 's3': s3, - } - + } if cfg.parallel and len(local_list) > 1: #Disabling progress metter for parallel downloads. @@ -2644,7 +2703,6 @@ def manifest_worker(): def calc_manifest_info(item, work_info): - size = item['size'] filename = item['full_name_unicode'] @@ -2664,24 +2722,31 @@ def calc_manifest_info(item, work_info): uri = content uri1 = uri - uri2 = u"%s__%s" % (uri1, item['file_name_extension']) - + uri2 = u"%s__%s" % (uri1, item['file_name_extension']) remote_list = work_info['remote_list'] + + #MLF:Temporarily can be either full or suffix key + full_uri = item['destination_base'] + uri if remote_list.has_key(uri): remote_file = remote_list[uri] size = remote_file['size'] return uri1, uri2, size, True + elif remote_list.has_key(full_uri): + remote_file = remote_list[full_uri] + size = remote_file['size'] + return uri1, uri2, size, True else: -# info(u"Fetching %s for %s" % (item['remote_uri'], filename)) + # info(u"Fetching %s for %s" % (item['remote_uri'], filename)) #Maybe we just didn't preget the list... so get it now + warning(u"File is not in directory_listing: '%s' , remote file at '%s'" % (filename, uri)) try: - remote_uri = S3Uri(item['destination_base']+uri) + remote_uri = S3Uri(full_uri) item_list = _get_filelist_remote(remote_uri, False) -# item_info = work_info['s3'].object_info(remote_uri) + # item_info = work_info['s3'].object_info(remote_uri) if len(item_list) > 0: item_info = item_list[item_list.keys()[0]] -# info(u"Received %s for %s" % (item['remote_uri'], filename)) + # info(u"Received %s for %s" % (item['remote_uri'], filename)) size = item_info['size'] return uri1, uri2, size, True else: @@ -2692,8 +2757,6 @@ def calc_manifest_info(item, work_info): return uri1, uri2, size, False - - def do_manifest_work(item, seq, total, work_info): uri1, uri2, size, should_output = calc_manifest_info(item, work_info) filename = item['full_name_unicode'] @@ -2702,7 +2765,6 @@ def do_manifest_work(item, seq, total, work_info): output(u"%s\t%s\t%s\t%s" % (filename, uri1, uri2, size)) - #============================================================================================================================== #=== cmd cleanup #============================================================================================================================== @@ -2731,7 +2793,6 @@ def cmd_cleanup(args): 'remote_list': remote_list } - if cfg.parallel and len(remote_list) > 1: #Disabling progress metter for parallel downloads. cfg.progress_meter = False @@ -2796,8 +2857,9 @@ def calc_cleanup_info(item, work_info): uri = content uri1 = uri - uri2 = u"%s__%s" % (uri1, item['file_name_extension']) + uri2 = u"%s__%s" % (uri1, item['file_name_extension']) remote_list = work_info['remote_list'] +# full_uri = if remote_list.has_key(uri): remote_file = remote_list[uri] size = remote_file['size'] @@ -2810,7 +2872,7 @@ def calc_cleanup_info(item, work_info): def do_cleanup_work(item, seq, total, work_info): size = item['size'] should_output = size <= CONST_sharef_size -# uri1, uri2, size, should_output = calc_manifest_info(item, work_info) + # uri1, uri2, size, should_output = calc_manifest_info(item, work_info) if should_output: filename = item['base_uri'] @@ -2959,8 +3021,14 @@ def main(): help="Recursive upload, download or removal.") optparser.add_option("--flatten", dest="flatten", action="store_true", help="Flatten a recursive operation (makes target name space flat even if source was recursive).") + optparser.add_option("--deflate", dest="deflate", action="store_true", help="Stub files during an upload.") + optparser.add_option("--use-directory", dest="use_directory", action="store_true", + help="Read from a directory-file (named 'blob_directory.txt' unless overridden below)") + optparser.add_option("--directory-name", dest="blob_directory", #action="append", metavar="FILE", + help="Set the directory-file's name to FILE") + optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option("--acl-private", dest="acl_public", action="store_false", From b502ff67827d486f19148d3e0bd6c207cbe408f5 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sat, 17 Mar 2012 16:40:53 -0700 Subject: [PATCH 12/16] Dropped work_info accidentally --- s3cmd | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/s3cmd b/s3cmd index 3cdba9c..22432d7 100755 --- a/s3cmd +++ b/s3cmd @@ -1919,6 +1919,10 @@ def cmd_upload(args): info(u"Summary: %d local files to upload, %d remote files already present" % (local_count, remote_count)) + work_info = { + 'remote_list': remote_list + } + # if local_count > 0: # if not destination_base.endswith("/"): # if not single_file_local: From ada8701f02e7a6509e54c549ccfac797c1846635 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Sun, 18 Mar 2012 09:29:12 -0700 Subject: [PATCH 13/16] Removed timestamp reference since local directory doesn't include it currently. --- s3cmd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/s3cmd b/s3cmd index 22432d7..4cae14f 100755 --- a/s3cmd +++ b/s3cmd @@ -2048,10 +2048,10 @@ def do_upload_put(item, seq, total, work_info, uri): item['remote_uri'] = full_uri if remote_list.has_key(uri): remote_file = remote_list[uri] - info(u"Skipped (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + info(u"Skipped (%d) (%s): %s -> %s " % (seq, uri, item['full_name_unicode'], item['remote_uri'])) #, remote_file['timestamp'])) elif remote_list.has_key(full_uri): remote_file = remote_list[full_uri] - info(u"Skipped[2] (%d) (%s): %s -> %s (%s) " % (seq, uri, item['full_name_unicode'], item['remote_uri'], remote_file['timestamp'])) + info(u"Skipped[2] (%d) (%s): %s -> %s " % (seq, uri, item['full_name_unicode'], item['remote_uri'])) #, remote_file['timestamp'])) else: info(u"Upload (%d): %s -> %s " % (seq, item['full_name_unicode'], item['remote_uri'])) if True: @@ -2279,13 +2279,13 @@ def do_inflate_download(item, seq, total, work_info, uri): item['object_uri_str'] = item['remote_uri'] if remote_list.has_key(uri): remote_file = remote_list[uri] - info(u"Inflating[1] from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + info(u"Inflating[1] from S3 (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) #, remote_file['timestamp'])) if True: do_get_work(item, seq, total) return True elif remote_list.has_key(full_uri): remote_file = remote_list[full_uri] - info(u"Inflating[2] from S3 (%d) (%s): %s <- %s (%s) " % (seq, uri, item['local_filename'], item['object_uri_str'], remote_file['timestamp'])) + info(u"Inflating[2] from S3 (%d) (%s): %s <- %s " % (seq, uri, item['local_filename'], item['object_uri_str'])) #, remote_file['timestamp'])) if True: do_get_work(item, seq, total) return True From 39d6b99de48f798cf2343a1ad575a2ab20440958 Mon Sep 17 00:00:00 2001 From: Mark Fussell Date: Tue, 9 Oct 2012 22:30:32 -0700 Subject: [PATCH 14/16] Partitioning functionality --- S3/Config.py | 2 + s3cmd | 268 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 206 insertions(+), 64 deletions(-) diff --git a/S3/Config.py b/S3/Config.py index 4454c5b..939c83a 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -38,6 +38,8 @@ class Config(object): deflate = False use_directory = False blob_directory = 'blob_directory.txt' + blob_prefix = '' + blob_prefix_length = 0 acl_public = None acl_grants = [] diff --git a/s3cmd b/s3cmd index 4cae14f..a802881 100755 --- a/s3cmd +++ b/s3cmd @@ -284,6 +284,23 @@ def fetch_local_list(args, recursive=None): return local_list, is_single_file + +def fetch_remote_list_worker(): + while True: + try: + (uri, seq, out_queue) = in_queue.get_nowait() + except Queue.Empty: + return + try: + object_list = _get_filelist_remote(uri) + out_queue.put([uri, seq, object_list]) + except Exception, e: + report_exception(e) + exit + in_queue.task_done() + + + def fetch_remote_list(args, require_attribs=False, recursive=None): remote_uris = [] remote_list = SortedDict(ignore_case=False) @@ -294,55 +311,103 @@ def fetch_remote_list(args, require_attribs=False, recursive=None): if recursive == None: recursive = cfg.recursive - for arg in args: - uri = S3Uri(arg) - if not uri.type == 's3': - raise ParameterError("Expecting S3 URI instead of '%s'" % arg) - remote_uris.append(uri) - - if recursive: - for uri in remote_uris: - objectlist = _get_filelist_remote(uri) - for key in objectlist: - remote_list[key] = objectlist[key] + if cfg.blob_prefix_length > 0: + ## + #Disabling progress metter for parallel downloads. + cfg.progress_meter = False + #Initialize Queue + + #Doing this before workers would collide + global in_queue + in_queue = Queue.Queue() + out_queue = Queue.Queue() + + blob_prefix_max = pow(16,cfg.blob_prefix_length) + + for i in range(blob_prefix_max): + prefix = hex(i)[2:] + + for arg in args: + prefix_uri = S3Uri(arg+prefix+"/") + if not prefix_uri.type == 's3': + raise ParameterError("Expecting S3 URI instead of '%s'" % prefix_uri) + + in_queue.put([prefix_uri, i, out_queue]) + + for i in range(cfg.workers): + t = threading.Thread(target=fetch_remote_list_worker) + t.daemon = True + t.start() + + #Necessary to ensure KeyboardInterrupt can actually kill + #Otherwise Queue.join() blocks until all queue elements have completed + while threading.activeCount() > 1: + time.sleep(.1) + + in_queue.join() + + while True: + try: + (uri, seq, object_list) = out_queue.get_nowait() + for key in object_list: + remote_list[key] = object_list[key] + except Queue.Empty: + info(u"Merged results") + break + + else: - for uri in remote_uris: - uri_str = str(uri) - ## Wildcards used in remote URI? - ## If yes we'll need a bucket listing... - if uri_str.find('*') > -1 or uri_str.find('?') > -1: - first_wildcard = uri_str.find('*') - first_questionmark = uri_str.find('?') - if first_questionmark > -1 and first_questionmark < first_wildcard: - first_wildcard = first_questionmark - prefix = uri_str[:first_wildcard] - rest = uri_str[first_wildcard + 1:] - ## Only request recursive listing if the 'rest' of the URI, - ## i.e. the part after first wildcard, contains '/' - need_recursion = rest.find('/') > -1 - objectlist = _get_filelist_remote(S3Uri(prefix), recursive=need_recursion) + for arg in args: + uri = S3Uri(arg) + if not uri.type == 's3': + raise ParameterError("Expecting S3 URI instead of '%s'" % arg) + remote_uris.append(uri) + + + if recursive: + for uri in remote_uris: + objectlist = _get_filelist_remote(uri) for key in objectlist: - ## Check whether the 'key' matches the requested wildcards - if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): - remote_list[key] = objectlist[key] - else: - ## No wildcards - simply append the given URI to the list - key = os.path.basename(uri.object()) - if not key: - raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) - remote_item = { - 'base_uri': uri, - 'object_uri_str': unicode(uri), - 'object_key': uri.object() - } - if require_attribs: - response = S3(cfg).object_info(uri) - remote_item.update({ - 'size': int(response['headers']['content-length']), - 'md5': response['headers']['etag'].strip('"\''), - 'timestamp': Utils.dateRFC822toUnix(response['headers']['date']) - }) - remote_list[key] = remote_item + remote_list[key] = objectlist[key] + else: + for uri in remote_uris: + uri_str = str(uri) + ## Wildcards used in remote URI? + ## If yes we'll need a bucket listing... + if uri_str.find('*') > -1 or uri_str.find('?') > -1: + first_wildcard = uri_str.find('*') + first_questionmark = uri_str.find('?') + if first_questionmark > -1 and first_questionmark < first_wildcard: + first_wildcard = first_questionmark + prefix = uri_str[:first_wildcard] + rest = uri_str[first_wildcard + 1:] + ## Only request recursive listing if the 'rest' of the URI, + ## i.e. the part after first wildcard, contains '/' + need_recursion = rest.find('/') > -1 + objectlist = _get_filelist_remote(S3Uri(prefix), recursive=need_recursion) + for key in objectlist: + ## Check whether the 'key' matches the requested wildcards + if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): + remote_list[key] = objectlist[key] + else: + ## No wildcards - simply append the given URI to the list + key = os.path.basename(uri.object()) + if not key: + raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) + remote_item = { + 'base_uri': uri, + 'object_uri_str': unicode(uri), + 'object_key': uri.object() + } + if require_attribs: + response = S3(cfg).object_info(uri) + remote_item.update({ + 'size': int(response['headers']['content-length']), + 'md5': response['headers']['etag'].strip('"\''), + 'timestamp': Utils.dateRFC822toUnix(response['headers']['date']) + }) + remote_list[key] = remote_item + return remote_list @@ -1891,13 +1956,22 @@ def process_patterns(patterns_list, patterns_from, is_glob, option_txt=""): #============================================================================================================================== def cmd_upload(args): + cmd_upload_prefix(args, cfg.blob_prefix) + + + +def cmd_upload_prefix(args, blob_prefix): if (len(args) < 2): raise ParameterError("Too few parameters! Expected: %s" % commands['foo']['param']) if len(args) == 0: raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) + destination_base_uri_arg = args.pop() + if blob_prefix: + destination_base_uri_arg = destination_base_uri_arg + "/" + blob_prefix + "/"; + + destination_base_uri = S3Uri(destination_base_uri_arg) if destination_base_uri.type != 's3': raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) destination_base = str(destination_base_uri) @@ -2016,6 +2090,12 @@ def calc_upload_remote_uris(item): return uri, "", False sha1 = hash_file_sha1(item['full_name_unicode']) + + if cfg.blob_prefix: + if not sha1.startswith(cfg.blob_prefix): + return "","",False + + # destination_base = item['destination_base'] uri1 = u"sha1_%s.blob" % sha1 uri2 = u"sha1_%s.blob__%s" % (sha1, item['file_name_extension']) @@ -2044,7 +2124,12 @@ def do_upload_put(item, seq, total, work_info, uri): # output('key: %s' % key) destination_base = item['destination_base'] - full_uri = unicodise(destination_base + uri) + if cfg.blob_prefix_length > 0: + prefix = uri[5:(5+cfg.blob_prefix_length)]+"/" + full_uri = unicodise(destination_base + prefix + uri) + else: + full_uri = unicodise(destination_base + uri) + item['remote_uri'] = full_uri if remote_list.has_key(uri): remote_file = remote_list[uri] @@ -2064,9 +2149,14 @@ def do_upload_deflate(item, seq, total, work_info, uri): size = item['size'] filename = item['full_name_unicode'] - if size <= CONST_sharef_size: - info(u"Skipped %s with size %d" % (filename, size)) - return False +# if size <= CONST_sharef_size: +# uri, is_sharef = calc_inflate_remote_uri(item) +# if is_sharef: +# info(u"Skipped sharef file %s with size %d and sharef %s" % (filename, size, uri)) +# return uri, "", False +# +# info(u"Skipped %s with size %d" % (filename, size)) +# return False #MLF:Need to detach a possible simlink @@ -2082,14 +2172,18 @@ def do_upload_deflate(item, seq, total, work_info, uri): def do_copy_deflate_into(item, seq, total, work_info, uri, destfilename): dir = os.path.dirname(destfilename) if not os.path.exists(dir): - os.makedirs(dir) + try: + os.makedirs(dir) + except OSError, e: + warning(u"Path creation collision for %s" % dir ) size = item['size'] filename = item['full_name_unicode'] - if size <= CONST_sharef_size: - info(u"Skipped %s with size %d" % (filename, size)) - return False + #Don't skip even suspicious items.... we need a copy +# if size <= CONST_sharef_size: +# info(u"Skipped %s with size %d" % (filename, size)) +# return False #MLF:Need to detach a possible simlink @@ -2124,6 +2218,7 @@ def do_upload_work(item, seq, total, work_info): do_upload_put(item, seq, total, work_info, uri1) do_upload_put(item, seq, total, work_info, uri2) + if cfg.deflate: do_upload_deflate(item, seq, total, work_info, uri1) @@ -2144,7 +2239,12 @@ def cmd_inflate(args): raise ParameterError("Nothing to inflate. Expecting a local file or directory and a S3 URI destination.") ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) + destination_base_uri_arg = args.pop() + if cfg.blob_prefix: + destination_base_uri_arg = destination_base_uri_arg + "/" + cfg.blob_prefix + "/"; + + destination_base_uri = S3Uri(destination_base_uri_arg) + if destination_base_uri.type != 's3': raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) destination_base = str(destination_base_uri) @@ -2251,9 +2351,14 @@ def calc_inflate_remote_uri(item): m = re.search('^sha1_([0-9a-fA-F]+).blob$', content) if m is None: - info(u"Skipped %s with content %s" % (filename, content)) + info(u"Skipped %s with content that doesn't match blob pattern" % filename) return "", False + if cfg.blob_prefix: + if not content.startswith("sha1_"+cfg.blob_prefix): + info(u"Skipped %s with content that doesn't match deflate_prefix %s" % (filename, cfg.blob_prefix) ) + return "", False + # output(u"Blob-reference %s with content %s" % (filename, content)) return content, True @@ -2273,7 +2378,12 @@ def do_inflate_download(item, seq, total, work_info, uri): destination_base = item['destination_base'] - full_uri = unicodise(destination_base + uri) + if cfg.blob_prefix_length > 0: + prefix = uri[5:(5+cfg.blob_prefix_length)]+"/" + full_uri = unicodise(destination_base + prefix + uri) + else: + full_uri = unicodise(destination_base + uri) + item['remote_uri'] = full_uri item['local_filename'] = item['full_name_unicode'] item['object_uri_str'] = item['remote_uri'] @@ -2341,7 +2451,12 @@ def cmd_copydeflate(args): raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.") ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) + destination_base_uri_arg = args.pop() + if cfg.blob_prefix: + destination_base_uri_arg = destination_base_uri_arg + "/" + cfg.blob_prefix + "/"; + + destination_base_uri = S3Uri(destination_base_uri_arg) + if destination_base_uri.type != 's3': raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri) destination_base = str(destination_base_uri) @@ -2503,7 +2618,12 @@ def copydeflate_worker2(): def do_linkorcopy_from_to(source_filename, dest_filename): dir = os.path.dirname(dest_filename) if not os.path.exists(dir): - os.makedirs(dir) + try: + os.makedirs(dir) + except OSError, e: + warning(u"Path creation collision for %s" % dir ) + + if cfg.can_link: try: os.remove(dest_filename) @@ -2518,7 +2638,11 @@ def do_linkorcopy_from_to(source_filename, dest_filename): def do_copy_from_to(source_filename, dest_filename): dir = os.path.dirname(dest_filename) if not os.path.exists(dir): - os.makedirs(dir) + try: + os.makedirs(dir) + except OSError, e: + warning(u"Path creation collision for %s" % dir ) + shutil.copyfile(source_filename, dest_filename) @@ -2617,7 +2741,12 @@ def cmd_manifest(args): raise ParameterError("Nothing to manifest. Expecting a local file or directory and a S3 URI for additional information.") ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) + destination_base_uri_arg = args.pop() + if cfg.blob_prefix: + destination_base_uri_arg = destination_base_uri_arg + "/" + cfg.blob_prefix + "/"; + + destination_base_uri = S3Uri(destination_base_uri_arg) + if destination_base_uri.type != 's3': raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) destination_base = str(destination_base_uri) @@ -2730,6 +2859,7 @@ def calc_manifest_info(item, work_info): remote_list = work_info['remote_list'] + #MLF:TODO #MLF:Temporarily can be either full or suffix key full_uri = item['destination_base'] + uri if remote_list.has_key(uri): @@ -2778,7 +2908,12 @@ def cmd_cleanup(args): raise ParameterError("Too few parameters! Expected: %s" % commands['cleanup']['param']) ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash) - destination_base_uri = S3Uri(args.pop()) + destination_base_uri_arg = args.pop() + if cfg.blob_prefix: + destination_base_uri_arg = destination_base_uri_arg + "/" + cfg.blob_prefix + "/"; + + destination_base_uri = S3Uri(destination_base_uri_arg) + if destination_base_uri.type != 's3': raise ParameterError("Source must currently be an S3Uri. Got: %s" % destination_base_uri) destination_base = str(destination_base_uri) @@ -3032,6 +3167,11 @@ def main(): help="Read from a directory-file (named 'blob_directory.txt' unless overridden below)") optparser.add_option("--directory-name", dest="blob_directory", #action="append", metavar="FILE", help="Set the directory-file's name to FILE") + optparser.add_option("--blob-prefix-filter", dest="blob_prefix", + help="Set the prefix-filter for the blobs to be inflated/deflated (e.g. 'a') ") + optparser.add_option("--blob-prefix-length", dest="blob_prefix_length", + help="The number of digits to use as a directory prefix for content/blob-based addressing") + optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") From ff21db38c70db485bb6c188221c8a6a2407d87b2 Mon Sep 17 00:00:00 2001 From: Brenden Matthews Date: Fri, 19 Apr 2013 16:28:29 -0700 Subject: [PATCH 15/16] Add --no-sync-check parameter. --- S3/Config.py | 1 + s3cmd | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/S3/Config.py b/S3/Config.py index 939c83a..18594db 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -71,6 +71,7 @@ class Config(object): default_mime_type = "binary/octet-stream" guess_mime_type = True # List of checks to be performed for 'sync' + no_check_md5 = False sync_checks = ['size', 'md5'] # 'weak-timestamp' # List of compiled REGEXPs exclude = [] diff --git a/s3cmd b/s3cmd index a802881..a689bbe 100755 --- a/s3cmd +++ b/s3cmd @@ -1062,7 +1062,7 @@ def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote, src_a u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) attribs_match = False - if attribs_match and 'md5' in cfg.sync_checks: + if attribs_match and 'md5' in cfg.sync_checks and not cfg.no_check_md5: ## ... same size, check MD5 if src_and_dst_remote: src_md5 = src_list[file]['md5'] @@ -3270,6 +3270,8 @@ def main(): help="Number of retry before failing GET or PUT.") optparser.add_option("--retry-delay", dest="retry_delay", type="int", action="store", default=3, help="Time delay to wait after failing GET or PUT.") + optparser.add_option("--no-check-md5", dest="no_check_md5", action="store_false", + help="Skip md5 integrity check when using sync.") optparser.set_usage(optparser.usage + " COMMAND [parameters]") optparser.set_description('S3cmd is a tool for managing objects in ' + From 496247ed1cbcb9b1a59285e21d7d413e7de28501 Mon Sep 17 00:00:00 2001 From: Brenden Matthews Date: Mon, 22 Apr 2013 10:35:50 -0700 Subject: [PATCH 16/16] Don't try to treat directories like files. --- s3cmd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/s3cmd b/s3cmd index a689bbe..357d927 100755 --- a/s3cmd +++ b/s3cmd @@ -1300,6 +1300,11 @@ def do_remote2local_work(item, seq, total): s3 = S3(Config()) uri = S3Uri(item['object_uri_str']) dst_file = item['local_filename'] + + # is this actually a directory? skip it. + if os.path.isdir(dst_file): + return + seq_label = "[%d of %d]" % (seq, total) try: dst_dir = os.path.dirname(dst_file)