Mercurial > repos > shellac > guppy_basecaller

comparison env/bin/lss3 @ 5:9b1c78e6ba9c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author shellac
date Mon, 01 Jun 2020 08:59:25 -0400
parents 79f47841a781
children
comparison
equal deleted inserted replaced
4:79f47841a781 5:9b1c78e6ba9c
1 #!/Users/pldms/Development/Projects/2020/david-matthews-galaxy/guppy_basecaller/env/bin/python3
2 import boto
3 from boto.exception import S3ResponseError
4 from boto.s3.connection import OrdinaryCallingFormat
5
6
7 def sizeof_fmt(num):
8 for x in ['b ', 'KB', 'MB', 'GB', 'TB', 'XB']:
9 if num < 1024.0:
10 return "%3.1f %s" % (num, x)
11 num /= 1024.0
12 return "%3.1f %s" % (num, x)
13
14
15 def list_bucket(b, prefix=None, marker=None):
16 """List everything in a bucket"""
17 from boto.s3.prefix import Prefix
18 from boto.s3.key import Key
19 total = 0
20
21 if prefix:
22 if not prefix.endswith("/"):
23 prefix = prefix + "/"
24 query = b.list(prefix=prefix, delimiter="/", marker=marker)
25 print("%s" % prefix)
26 else:
27 query = b.list(delimiter="/", marker=marker)
28
29 num = 0
30 for k in query:
31 num += 1
32 mode = "-rwx---"
33 if isinstance(k, Prefix):
34 mode = "drwxr--"
35 size = 0
36 else:
37 size = k.size
38 for g in k.get_acl().acl.grants:
39 if g.id == None:
40 if g.permission == "READ":
41 mode = "-rwxr--"
42 elif g.permission == "FULL_CONTROL":
43 mode = "-rwxrwx"
44 if isinstance(k, Key):
45 print("%s\t%s\t%010s\t%s" % (mode, k.last_modified,
46 sizeof_fmt(size), k.name))
47 else:
48 #If it's not a Key object, it doesn't have a last_modified time, so
49 #print nothing instead
50 print("%s\t%s\t%010s\t%s" % (mode, ' ' * 24,
51 sizeof_fmt(size), k.name))
52 total += size
53 print ("=" * 80)
54 print ("\t\tTOTAL: \t%010s \t%i Files" % (sizeof_fmt(total), num))
55
56
57 def list_buckets(s3, display_tags=False):
58 """List all the buckets"""
59 for b in s3.get_all_buckets():
60 print(b.name)
61 if display_tags:
62 try:
63 tags = b.get_tags()
64 for tag in tags[0]:
65 print(" %s:%s" % (tag.key, tag.value))
66 except S3ResponseError as e:
67 if e.status != 404:
68 raise
69
70
71 def main():
72 import optparse
73 import sys
74
75 usage = "usage: %prog [options] [BUCKET1] [BUCKET2]"
76 description = "List all S3 buckets OR list keys in the named buckets"
77 parser = optparse.OptionParser(description=description, usage=usage)
78 parser.add_option('-m', '--marker',
79 help='The S3 key where the listing starts after it.')
80 parser.add_option('-t', '--tags', action='store_true',
81 help='Display tags when listing all buckets.')
82 options, buckets = parser.parse_args()
83 marker = options.marker
84
85 if not buckets:
86 list_buckets(boto.connect_s3(), options.tags)
87 sys.exit(0)
88
89 if options.tags:
90 print("-t option only works for the overall bucket list")
91 sys.exit(1)
92
93 pairs = []
94 mixedCase = False
95 for name in buckets:
96 if "/" in name:
97 pairs.append(name.split("/", 1))
98 else:
99 pairs.append([name, None])
100 if pairs[-1][0].lower() != pairs[-1][0]:
101 mixedCase = True
102
103 if mixedCase:
104 s3 = boto.connect_s3(calling_format=OrdinaryCallingFormat())
105 else:
106 s3 = boto.connect_s3()
107
108 for name, prefix in pairs:
109 list_bucket(s3.get_bucket(name), prefix, marker=marker)
110
111
112 if __name__ == "__main__":
113 main()