Mercurial > repos > shellac > guppy_basecaller
comparison env/bin/lss3 @ 0:26e78fe6e8c4 draft
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
| author | shellac |
|---|---|
| date | Sat, 02 May 2020 07:14:21 -0400 |
| parents | |
| children |
comparison
equal
deleted
inserted
replaced
| -1:000000000000 | 0:26e78fe6e8c4 |
|---|---|
| 1 #!/Users/pldms/Development/Projects/2020/david-matthews-galaxy/guppy_basecaller/env/bin/python3 | |
| 2 import boto | |
| 3 from boto.exception import S3ResponseError | |
| 4 from boto.s3.connection import OrdinaryCallingFormat | |
| 5 | |
| 6 | |
| 7 def sizeof_fmt(num): | |
| 8 for x in ['b ', 'KB', 'MB', 'GB', 'TB', 'XB']: | |
| 9 if num < 1024.0: | |
| 10 return "%3.1f %s" % (num, x) | |
| 11 num /= 1024.0 | |
| 12 return "%3.1f %s" % (num, x) | |
| 13 | |
| 14 | |
| 15 def list_bucket(b, prefix=None, marker=None): | |
| 16 """List everything in a bucket""" | |
| 17 from boto.s3.prefix import Prefix | |
| 18 from boto.s3.key import Key | |
| 19 total = 0 | |
| 20 | |
| 21 if prefix: | |
| 22 if not prefix.endswith("/"): | |
| 23 prefix = prefix + "/" | |
| 24 query = b.list(prefix=prefix, delimiter="/", marker=marker) | |
| 25 print("%s" % prefix) | |
| 26 else: | |
| 27 query = b.list(delimiter="/", marker=marker) | |
| 28 | |
| 29 num = 0 | |
| 30 for k in query: | |
| 31 num += 1 | |
| 32 mode = "-rwx---" | |
| 33 if isinstance(k, Prefix): | |
| 34 mode = "drwxr--" | |
| 35 size = 0 | |
| 36 else: | |
| 37 size = k.size | |
| 38 for g in k.get_acl().acl.grants: | |
| 39 if g.id == None: | |
| 40 if g.permission == "READ": | |
| 41 mode = "-rwxr--" | |
| 42 elif g.permission == "FULL_CONTROL": | |
| 43 mode = "-rwxrwx" | |
| 44 if isinstance(k, Key): | |
| 45 print("%s\t%s\t%010s\t%s" % (mode, k.last_modified, | |
| 46 sizeof_fmt(size), k.name)) | |
| 47 else: | |
| 48 #If it's not a Key object, it doesn't have a last_modified time, so | |
| 49 #print nothing instead | |
| 50 print("%s\t%s\t%010s\t%s" % (mode, ' ' * 24, | |
| 51 sizeof_fmt(size), k.name)) | |
| 52 total += size | |
| 53 print ("=" * 80) | |
| 54 print ("\t\tTOTAL: \t%010s \t%i Files" % (sizeof_fmt(total), num)) | |
| 55 | |
| 56 | |
| 57 def list_buckets(s3, display_tags=False): | |
| 58 """List all the buckets""" | |
| 59 for b in s3.get_all_buckets(): | |
| 60 print(b.name) | |
| 61 if display_tags: | |
| 62 try: | |
| 63 tags = b.get_tags() | |
| 64 for tag in tags[0]: | |
| 65 print(" %s:%s" % (tag.key, tag.value)) | |
| 66 except S3ResponseError as e: | |
| 67 if e.status != 404: | |
| 68 raise | |
| 69 | |
| 70 | |
| 71 def main(): | |
| 72 import optparse | |
| 73 import sys | |
| 74 | |
| 75 usage = "usage: %prog [options] [BUCKET1] [BUCKET2]" | |
| 76 description = "List all S3 buckets OR list keys in the named buckets" | |
| 77 parser = optparse.OptionParser(description=description, usage=usage) | |
| 78 parser.add_option('-m', '--marker', | |
| 79 help='The S3 key where the listing starts after it.') | |
| 80 parser.add_option('-t', '--tags', action='store_true', | |
| 81 help='Display tags when listing all buckets.') | |
| 82 options, buckets = parser.parse_args() | |
| 83 marker = options.marker | |
| 84 | |
| 85 if not buckets: | |
| 86 list_buckets(boto.connect_s3(), options.tags) | |
| 87 sys.exit(0) | |
| 88 | |
| 89 if options.tags: | |
| 90 print("-t option only works for the overall bucket list") | |
| 91 sys.exit(1) | |
| 92 | |
| 93 pairs = [] | |
| 94 mixedCase = False | |
| 95 for name in buckets: | |
| 96 if "/" in name: | |
| 97 pairs.append(name.split("/", 1)) | |
| 98 else: | |
| 99 pairs.append([name, None]) | |
| 100 if pairs[-1][0].lower() != pairs[-1][0]: | |
| 101 mixedCase = True | |
| 102 | |
| 103 if mixedCase: | |
| 104 s3 = boto.connect_s3(calling_format=OrdinaryCallingFormat()) | |
| 105 else: | |
| 106 s3 = boto.connect_s3() | |
| 107 | |
| 108 for name, prefix in pairs: | |
| 109 list_bucket(s3.get_bucket(name), prefix, marker=marker) | |
| 110 | |
| 111 | |
| 112 if __name__ == "__main__": | |
| 113 main() |