annotate ensure_synced.py @ 8:aaa6ae7c64de draft default tip

Uploaded
author greg
date Tue, 25 Jan 2022 14:24:54 +0000
parents 59edc91d0bea
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
9180906544b6 Uploaded
greg
parents:
diff changeset
1 #!/usr/bin/env python
2
3dc919d53939 Uploaded
greg
parents: 0
diff changeset
2
0
9180906544b6 Uploaded
greg
parents:
diff changeset
3 import argparse
9180906544b6 Uploaded
greg
parents:
diff changeset
4 import sys
9180906544b6 Uploaded
greg
parents:
diff changeset
5
3
2656d87abfee Uploaded
greg
parents: 2
diff changeset
6 import psycopg2
7
59edc91d0bea Uploaded
greg
parents: 6
diff changeset
7 from sqlalchemy import create_engine
3
2656d87abfee Uploaded
greg
parents: 2
diff changeset
8 from sqlalchemy import MetaData
0
9180906544b6 Uploaded
greg
parents:
diff changeset
9 from sqlalchemy.engine.url import make_url
9180906544b6 Uploaded
greg
parents:
diff changeset
10
9180906544b6 Uploaded
greg
parents:
diff changeset
11 metadata = MetaData()
9180906544b6 Uploaded
greg
parents:
diff changeset
12
9180906544b6 Uploaded
greg
parents:
diff changeset
13 SKIP_VALS = ['#CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT']
9180906544b6 Uploaded
greg
parents:
diff changeset
14
9180906544b6 Uploaded
greg
parents:
diff changeset
15
9180906544b6 Uploaded
greg
parents:
diff changeset
16 class EnsureSynced(object):
9180906544b6 Uploaded
greg
parents:
diff changeset
17 def __init__(self):
9180906544b6 Uploaded
greg
parents:
diff changeset
18 self.args = None
9180906544b6 Uploaded
greg
parents:
diff changeset
19 self.conn = None
9180906544b6 Uploaded
greg
parents:
diff changeset
20 self.parse_args()
9180906544b6 Uploaded
greg
parents:
diff changeset
21 self.outfh = open(self.args.output, "w")
9180906544b6 Uploaded
greg
parents:
diff changeset
22 self.connect_db()
9180906544b6 Uploaded
greg
parents:
diff changeset
23 self.engine = create_engine(self.args.database_connection_string)
9180906544b6 Uploaded
greg
parents:
diff changeset
24 self.metadata = MetaData(self.engine)
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
25 self.coral_mlg_rep_sample_ids_from_db = []
0
9180906544b6 Uploaded
greg
parents:
diff changeset
26 self.affy_ids_from_file = []
9180906544b6 Uploaded
greg
parents:
diff changeset
27
9180906544b6 Uploaded
greg
parents:
diff changeset
28 def connect_db(self):
9180906544b6 Uploaded
greg
parents:
diff changeset
29 url = make_url(self.args.database_connection_string)
9180906544b6 Uploaded
greg
parents:
diff changeset
30 args = url.translate_connect_args(username='user')
9180906544b6 Uploaded
greg
parents:
diff changeset
31 args.update(url.query)
9180906544b6 Uploaded
greg
parents:
diff changeset
32 assert url.get_dialect().name == 'postgresql', 'This script can only be used with PostgreSQL.'
9180906544b6 Uploaded
greg
parents:
diff changeset
33 self.conn = psycopg2.connect(**args)
9180906544b6 Uploaded
greg
parents:
diff changeset
34
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
35 def get_coral_mlg_rep_sample_ids_from_db(self):
7
59edc91d0bea Uploaded
greg
parents: 6
diff changeset
36 cmd = "SELECT coral_mlg_rep_sample_id, coral_mlg_clonal_id FROM genotype WHERE coral_mlg_rep_sample_id IS NOT NULL AND coral_mlg_rep_sample_id != '' AND coral_mlg_clonal_id != 'failed' ORDER BY coral_mlg_rep_sample_id;"
0
9180906544b6 Uploaded
greg
parents:
diff changeset
37 cur = self.conn.cursor()
9180906544b6 Uploaded
greg
parents:
diff changeset
38 cur.execute(cmd)
9180906544b6 Uploaded
greg
parents:
diff changeset
39 rows = cur.fetchall()
9180906544b6 Uploaded
greg
parents:
diff changeset
40 for row in rows:
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
41 self.coral_mlg_rep_sample_ids_from_db.append(row[0])
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
42 self.coral_mlg_rep_sample_ids_from_db.sort()
0
9180906544b6 Uploaded
greg
parents:
diff changeset
43
9180906544b6 Uploaded
greg
parents:
diff changeset
44 def get_affy_ids_from_file(self, f):
9180906544b6 Uploaded
greg
parents:
diff changeset
45 with open(f) as fh:
9180906544b6 Uploaded
greg
parents:
diff changeset
46 for line in fh:
9180906544b6 Uploaded
greg
parents:
diff changeset
47 line = line.strip()
9180906544b6 Uploaded
greg
parents:
diff changeset
48 if line in SKIP_VALS:
9180906544b6 Uploaded
greg
parents:
diff changeset
49 # Skip the first 9 lines in the file.
9180906544b6 Uploaded
greg
parents:
diff changeset
50 continue
9180906544b6 Uploaded
greg
parents:
diff changeset
51 self.affy_ids_from_file.append(line)
9180906544b6 Uploaded
greg
parents:
diff changeset
52 self.affy_ids_from_file.sort()
9180906544b6 Uploaded
greg
parents:
diff changeset
53
9180906544b6 Uploaded
greg
parents:
diff changeset
54 def get_difference(self, list1, list2):
9180906544b6 Uploaded
greg
parents:
diff changeset
55 if len(list1) > len(list2):
9180906544b6 Uploaded
greg
parents:
diff changeset
56 return list(set(list1) - set(list2))
9180906544b6 Uploaded
greg
parents:
diff changeset
57 return list(set(list2) - set(list1))
9180906544b6 Uploaded
greg
parents:
diff changeset
58
9180906544b6 Uploaded
greg
parents:
diff changeset
59 def log(self, msg):
9180906544b6 Uploaded
greg
parents:
diff changeset
60 self.outfh.write("%s\n" % msg)
9180906544b6 Uploaded
greg
parents:
diff changeset
61
9180906544b6 Uploaded
greg
parents:
diff changeset
62 def parse_args(self):
9180906544b6 Uploaded
greg
parents:
diff changeset
63 parser = argparse.ArgumentParser()
9180906544b6 Uploaded
greg
parents:
diff changeset
64 parser.add_argument('--database_connection_string', dest='database_connection_string', help='Postgres database connection string'),
9180906544b6 Uploaded
greg
parents:
diff changeset
65 parser.add_argument('--affy_ids_from_file', dest='affy_ids_from_file', help='Affy ids taken from all previously genotyped samples vcf file')
9180906544b6 Uploaded
greg
parents:
diff changeset
66 parser.add_argument('--output', dest='output', help='Output dataset'),
9180906544b6 Uploaded
greg
parents:
diff changeset
67 self.args = parser.parse_args()
9180906544b6 Uploaded
greg
parents:
diff changeset
68
9180906544b6 Uploaded
greg
parents:
diff changeset
69 def run(self):
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
70 self.get_coral_mlg_rep_sample_ids_from_db()
0
9180906544b6 Uploaded
greg
parents:
diff changeset
71 self.get_affy_ids_from_file(self.args.affy_ids_from_file)
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
72 if self.coral_mlg_rep_sample_ids_from_db == self.affy_ids_from_file:
0
9180906544b6 Uploaded
greg
parents:
diff changeset
73 in_sync = True
9180906544b6 Uploaded
greg
parents:
diff changeset
74 self.log("The selected file is in sync with the database.\n\n")
9180906544b6 Uploaded
greg
parents:
diff changeset
75 else:
9180906544b6 Uploaded
greg
parents:
diff changeset
76 in_sync = False
9180906544b6 Uploaded
greg
parents:
diff changeset
77 self.log("The selected file is not in sync with the database.\n\n")
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
78 num_coral_mlg_rep_sample_ids_from_db = len(self.coral_mlg_rep_sample_ids_from_db)
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
79 self.log("Number of coral mlg rep sample ids in the database: %d\n" % num_coral_mlg_rep_sample_ids_from_db)
0
9180906544b6 Uploaded
greg
parents:
diff changeset
80 num_affy_ids_from_file = len(self.affy_ids_from_file)
9180906544b6 Uploaded
greg
parents:
diff changeset
81 self.log("Number of Affymetrix ids in the file: %d\n" % num_affy_ids_from_file)
9180906544b6 Uploaded
greg
parents:
diff changeset
82 if not in_sync:
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
83 if num_coral_mlg_rep_sample_ids_from_db > num_affy_ids_from_file:
0
9180906544b6 Uploaded
greg
parents:
diff changeset
84 self.log("The database contains the following Affymetrix ids that are not in the file.\n")
9180906544b6 Uploaded
greg
parents:
diff changeset
85 else:
9180906544b6 Uploaded
greg
parents:
diff changeset
86 self.log("The file contains the following Affymetrix ids that are not in the database.\n")
8
aaa6ae7c64de Uploaded
greg
parents: 7
diff changeset
87 diff_list = self.get_difference(self.coral_mlg_rep_sample_ids_from_db, self.affy_ids_from_file)
0
9180906544b6 Uploaded
greg
parents:
diff changeset
88 for affy_id in diff_list:
9180906544b6 Uploaded
greg
parents:
diff changeset
89 self.log("%s\n" % affy_id)
9180906544b6 Uploaded
greg
parents:
diff changeset
90 self.outfh.flush()
9180906544b6 Uploaded
greg
parents:
diff changeset
91 self.outfh.close()
9180906544b6 Uploaded
greg
parents:
diff changeset
92 sys.exit(1)
9180906544b6 Uploaded
greg
parents:
diff changeset
93
9180906544b6 Uploaded
greg
parents:
diff changeset
94 def shutdown(self):
9180906544b6 Uploaded
greg
parents:
diff changeset
95 self.outfh.flush()
9180906544b6 Uploaded
greg
parents:
diff changeset
96 self.outfh.close()
9180906544b6 Uploaded
greg
parents:
diff changeset
97 self.conn.close()
9180906544b6 Uploaded
greg
parents:
diff changeset
98
9180906544b6 Uploaded
greg
parents:
diff changeset
99
9180906544b6 Uploaded
greg
parents:
diff changeset
100 if __name__ == '__main__':
9180906544b6 Uploaded
greg
parents:
diff changeset
101 es = EnsureSynced()
9180906544b6 Uploaded
greg
parents:
diff changeset
102 es.run()
9180906544b6 Uploaded
greg
parents:
diff changeset
103 es.shutdown()