Mercurial > repos > shellac > guppy_basecaller

comparison env/bin/dynamodb_dump @ 5:9b1c78e6ba9c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author shellac
date Mon, 01 Jun 2020 08:59:25 -0400
parents 79f47841a781
children
comparison
equal deleted inserted replaced
4:79f47841a781 5:9b1c78e6ba9c
1 #!/Users/pldms/Development/Projects/2020/david-matthews-galaxy/guppy_basecaller/env/bin/python3
2
3 import argparse
4 import errno
5 import os
6
7 import boto
8 from boto.compat import json
9 from boto.compat import six
10
11
12 DESCRIPTION = """Dump the contents of one or more DynamoDB tables to the local filesystem.
13
14 Each table is dumped into two files:
15 - {table_name}.metadata stores the table's name, schema and provisioned
16 throughput.
17 - {table_name}.data stores the table's actual contents.
18
19 Both files are created in the current directory. To write them somewhere else,
20 use the --out-dir parameter (the target directory will be created if needed).
21 """
22
23
24 def dump_table(table, out_dir):
25 metadata_file = os.path.join(out_dir, "%s.metadata" % table.name)
26 data_file = os.path.join(out_dir, "%s.data" % table.name)
27
28 with open(metadata_file, "w") as metadata_fd:
29 json.dump(
30 {
31 "name": table.name,
32 "schema": table.schema.dict,
33 "read_units": table.read_units,
34 "write_units": table.write_units,
35 },
36 metadata_fd
37 )
38
39 with open(data_file, "w") as data_fd:
40 for item in table.scan():
41 # JSON can't serialize sets -- convert those to lists.
42 data = {}
43 for k, v in six.iteritems(item):
44 if isinstance(v, (set, frozenset)):
45 data[k] = list(v)
46 else:
47 data[k] = v
48
49 data_fd.write(json.dumps(data))
50 data_fd.write("\n")
51
52
53 def dynamodb_dump(tables, out_dir):
54 try:
55 os.makedirs(out_dir)
56 except OSError as e:
57 # We don't care if the dir already exists.
58 if e.errno != errno.EEXIST:
59 raise
60
61 conn = boto.connect_dynamodb()
62 for t in tables:
63 dump_table(conn.get_table(t), out_dir)
64
65
66 if __name__ == "__main__":
67 parser = argparse.ArgumentParser(
68 prog="dynamodb_dump",
69 description=DESCRIPTION
70 )
71 parser.add_argument("--out-dir", default=".")
72 parser.add_argument("tables", metavar="TABLES", nargs="+")
73
74 namespace = parser.parse_args()
75
76 dynamodb_dump(namespace.tables, namespace.out_dir)