|
0
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 import argparse
|
|
|
4 import multiprocessing
|
|
|
5 import os
|
|
|
6 import pandas
|
|
|
7 import pysam
|
|
|
8 import queue
|
|
|
9 import re
|
|
|
10 import shutil
|
|
|
11 from numpy import mean
|
|
|
12 from Bio import SeqIO
|
|
|
13
|
|
|
14 INPUT_BAM_DIR = 'input_bam_dir'
|
|
|
15 INPUT_VCF_DIR = 'input_vcf_dir'
|
|
|
16 OUTPUT_VCF_DIR = 'output_vcf_dir'
|
|
|
17 OUTPUT_METRICS_DIR = 'output_metrics_dir'
|
|
|
18
|
|
|
19
|
|
|
20 def get_base_file_name(file_path):
|
|
|
21 base_file_name = os.path.basename(file_path)
|
|
|
22 if base_file_name.find(".") > 0:
|
|
|
23 # Eliminate the extension.
|
|
|
24 return os.path.splitext(base_file_name)[0]
|
|
|
25 elif base_file_name.find("_") > 0:
|
|
|
26 # The dot extension was likely changed to
|
|
|
27 # the " character.
|
|
|
28 items = base_file_name.split("_")
|
|
|
29 return "_".join(items[0:-1])
|
|
|
30 else:
|
|
|
31 return base_file_name
|
|
|
32
|
|
|
33
|
|
|
34 def get_coverage_and_snp_count(task_queue, reference, output_metrics, output_vcf, timeout):
|
|
|
35 while True:
|
|
|
36 try:
|
|
|
37 tup = task_queue.get(block=True, timeout=timeout)
|
|
|
38 except queue.Empty:
|
|
|
39 break
|
|
|
40 bam_file, vcf_file = tup
|
|
|
41 # Create a coverage dictionary.
|
|
|
42 coverage_dict = {}
|
|
|
43 coverage_list = pysam.depth(bam_file, split_lines=True)
|
|
|
44 for line in coverage_list:
|
|
|
45 chrom, position, depth = line.split('\t')
|
|
|
46 coverage_dict["%s-%s" % (chrom, position)] = depth
|
|
|
47 # Convert it to a data frame.
|
|
|
48 coverage_df = pandas.DataFrame.from_dict(coverage_dict, orient='index', columns=["depth"])
|
|
|
49 # Create a zero coverage dictionary.
|
|
|
50 zero_dict = {}
|
|
|
51 for record in SeqIO.parse(reference, "fasta"):
|
|
|
52 chrom = record.id
|
|
|
53 total_len = len(record.seq)
|
|
|
54 for pos in list(range(1, total_len + 1)):
|
|
|
55 zero_dict["%s-%s" % (str(chrom), str(pos))] = 0
|
|
|
56 # Convert it to a data frame with depth_x
|
|
|
57 # and depth_y columns - index is NaN.
|
|
|
58 zero_df = pandas.DataFrame.from_dict(zero_dict, orient='index', columns=["depth"])
|
|
|
59 coverage_df = zero_df.merge(coverage_df, left_index=True, right_index=True, how='outer')
|
|
|
60 # depth_x "0" column no longer needed.
|
|
|
61 coverage_df = coverage_df.drop(columns=['depth_x'])
|
|
|
62 coverage_df = coverage_df.rename(columns={'depth_y': 'depth'})
|
|
|
63 # Covert the NaN to 0 coverage and get some metrics.
|
|
|
64 coverage_df = coverage_df.fillna(0)
|
|
|
65 coverage_df['depth'] = coverage_df['depth'].apply(int)
|
|
|
66 total_length = len(coverage_df)
|
|
|
67 average_coverage = coverage_df['depth'].mean()
|
|
|
68 zero_df = coverage_df[coverage_df['depth'] == 0]
|
|
|
69 total_zero_coverage = len(zero_df)
|
|
|
70 total_coverage = total_length - total_zero_coverage
|
|
|
71 genome_coverage = "{:.2%}".format(total_coverage / total_length)
|
|
|
72 # Process the associated VCF input.
|
|
|
73 column_names = ["CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", "Sample"]
|
|
|
74 vcf_df = pandas.read_csv(vcf_file, sep='\t', header=None, names=column_names, comment='#')
|
|
|
75 good_snp_count = len(vcf_df[(vcf_df['ALT'].str.len() == 1) & (vcf_df['REF'].str.len() == 1) & (vcf_df['QUAL'] > 150)])
|
|
|
76 base_file_name = get_base_file_name(vcf_file)
|
|
|
77 if total_zero_coverage > 0:
|
|
|
78 header_file = "%s_header.csv" % base_file_name
|
|
|
79 with open(header_file, 'w') as outfile:
|
|
|
80 with open(vcf_file) as infile:
|
|
|
81 for line in infile:
|
|
|
82 if re.search('^#', line):
|
|
|
83 outfile.write("%s" % line)
|
|
|
84 vcf_df_snp = vcf_df[vcf_df['REF'].str.len() == 1]
|
|
|
85 vcf_df_snp = vcf_df_snp[vcf_df_snp['ALT'].str.len() == 1]
|
|
|
86 vcf_df_snp['ABS_VALUE'] = vcf_df_snp['CHROM'].map(str) + "-" + vcf_df_snp['POS'].map(str)
|
|
|
87 vcf_df_snp = vcf_df_snp.set_index('ABS_VALUE')
|
|
|
88 cat_df = pandas.concat([vcf_df_snp, zero_df], axis=1, sort=False)
|
|
|
89 cat_df = cat_df.drop(columns=['CHROM', 'POS', 'depth'])
|
|
|
90 cat_df[['ID', 'ALT', 'QUAL', 'FILTER', 'INFO']] = cat_df[['ID', 'ALT', 'QUAL', 'FILTER', 'INFO']].fillna('.')
|
|
|
91 cat_df['REF'] = cat_df['REF'].fillna('N')
|
|
|
92 cat_df['FORMAT'] = cat_df['FORMAT'].fillna('GT')
|
|
|
93 cat_df['Sample'] = cat_df['Sample'].fillna('./.')
|
|
|
94 cat_df['temp'] = cat_df.index.str.rsplit('-', n=1)
|
|
|
95 cat_df[['CHROM', 'POS']] = pandas.DataFrame(cat_df.temp.values.tolist(), index=cat_df.index)
|
|
|
96 cat_df = cat_df[['CHROM', 'POS', 'ID', 'REF', 'ALT', 'QUAL', 'FILTER', 'INFO', 'FORMAT', 'Sample']]
|
|
|
97 cat_df['POS'] = cat_df['POS'].astype(int)
|
|
|
98 cat_df = cat_df.sort_values(['CHROM', 'POS'])
|
|
|
99 body_file = "%s_body.csv" % base_file_name
|
|
|
100 cat_df.to_csv(body_file, sep='\t', header=False, index=False)
|
|
|
101 if output_vcf is None:
|
|
|
102 output_vcf_file = os.path.join(OUTPUT_VCF_DIR, "%s.vcf" % base_file_name)
|
|
|
103 else:
|
|
|
104 output_vcf_file = output_vcf
|
|
|
105 with open(output_vcf_file, "w") as outfile:
|
|
|
106 for cf in [header_file, body_file]:
|
|
|
107 with open(cf, "r") as infile:
|
|
|
108 for line in infile:
|
|
|
109 outfile.write("%s" % line)
|
|
|
110 else:
|
|
|
111 if output_vcf is None:
|
|
|
112 output_vcf_file = os.path.join(OUTPUT_VCF_DIR, "%s.vcf" % base_file_name)
|
|
|
113 else:
|
|
|
114 output_vcf_file = output_vcf
|
|
|
115 shutil.copyfile(vcf_file, output_vcf_file)
|
|
|
116 bam_metrics = [base_file_name, "", "%4f" % average_coverage, genome_coverage]
|
|
|
117 vcf_metrics = [base_file_name, str(good_snp_count), "", ""]
|
|
|
118 if output_metrics is None:
|
|
|
119 output_metrics_file = os.path.join(OUTPUT_METRICS_DIR, "%s.tabular" % base_file_name)
|
|
|
120 else:
|
|
|
121 output_metrics_file = output_metrics
|
|
|
122 metrics_columns = ["File", "Number of Good SNPs", "Average Coverage", "Genome Coverage"]
|
|
|
123 with open(output_metrics_file, "w") as fh:
|
|
|
124 fh.write("# %s\n" % "\t".join(metrics_columns))
|
|
|
125 fh.write("%s\n" % "\t".join(bam_metrics))
|
|
|
126 fh.write("%s\n" % "\t".join(vcf_metrics))
|
|
|
127 task_queue.task_done()
|
|
|
128
|
|
|
129
|
|
|
130 def set_num_cpus(num_files, processes):
|
|
|
131 num_cpus = int(multiprocessing.cpu_count())
|
|
|
132 if num_files < num_cpus and num_files < processes:
|
|
|
133 return num_files
|
|
|
134 if num_cpus < processes:
|
|
|
135 half_cpus = int(num_cpus / 2)
|
|
|
136 if num_files < half_cpus:
|
|
|
137 return num_files
|
|
|
138 return half_cpus
|
|
|
139 return processes
|
|
|
140
|
|
|
141
|
|
|
142 if __name__ == '__main__':
|
|
|
143 parser = argparse.ArgumentParser()
|
|
|
144
|
|
|
145 parser.add_argument('--output_metrics', action='store', dest='output_metrics', required=False, default=None, help='Output metrics text file')
|
|
|
146 parser.add_argument('--output_vcf', action='store', dest='output_vcf', required=False, default=None, help='Output VCF file')
|
|
|
147 parser.add_argument('--reference', action='store', dest='reference', help='Reference dataset')
|
|
|
148 parser.add_argument('--processes', action='store', dest='processes', type=int, help='User-selected number of processes to use for job splitting')
|
|
|
149
|
|
|
150 args = parser.parse_args()
|
|
|
151
|
|
|
152 # The assumption here is that the list of files
|
|
|
153 # in both INPUT_BAM_DIR and INPUT_VCF_DIR are
|
|
|
154 # equal in number and named such that they are
|
|
|
155 # properly matched if the directories contain
|
|
|
156 # more than 1 file (i.e., hopefully the bam file
|
|
|
157 # names and vcf file names will be something like
|
|
|
158 # Mbovis-01D6_* so they can be # sorted and properly
|
|
|
159 # associated with each other).
|
|
|
160 bam_files = []
|
|
|
161 for file_name in sorted(os.listdir(INPUT_BAM_DIR)):
|
|
|
162 file_path = os.path.abspath(os.path.join(INPUT_BAM_DIR, file_name))
|
|
|
163 bam_files.append(file_path)
|
|
|
164 vcf_files = []
|
|
|
165 for file_name in sorted(os.listdir(INPUT_VCF_DIR)):
|
|
|
166 file_path = os.path.abspath(os.path.join(INPUT_VCF_DIR, file_name))
|
|
|
167 vcf_files.append(file_path)
|
|
|
168
|
|
|
169 multiprocessing.set_start_method('spawn')
|
|
|
170 queue1 = multiprocessing.JoinableQueue()
|
|
|
171 num_files = len(bam_files)
|
|
|
172 cpus = set_num_cpus(num_files, args.processes)
|
|
|
173 # Set a timeout for get()s in the queue.
|
|
|
174 timeout = 0.05
|
|
|
175
|
|
|
176 # Add each associated bam and vcf file pair to the queue.
|
|
|
177 for i, bam_file in enumerate(bam_files):
|
|
|
178 vcf_file = vcf_files[i]
|
|
|
179 queue1.put((bam_file, vcf_file))
|
|
|
180
|
|
|
181 # Complete the get_coverage_and_snp_count task.
|
|
|
182 processes = [multiprocessing.Process(target=get_coverage_and_snp_count, args=(queue1, args.reference, args.output_metrics, args.output_vcf, timeout, )) for _ in range(cpus)]
|
|
|
183 for p in processes:
|
|
|
184 p.start()
|
|
|
185 for p in processes:
|
|
|
186 p.join()
|
|
|
187 queue1.join()
|
|
|
188
|
|
|
189 if queue1.empty():
|
|
|
190 queue1.close()
|
|
|
191 queue1.join_thread()
|
