Mercurial > repos > iss > eurl_vtec_wgs_pt

annotate scripts/ReMatCh/modules/download.py @ 6:20ff3dca457f draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 6857c749c21f580c828aba3543e294b69d32b662
author iss
date Mon, 23 Oct 2023 11:45:36 +0000
parents c6bab5103a14
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
1 import os.path
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
2 import multiprocessing
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
3 import sys
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
4 import functools
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
5 import time
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
6 import subprocess
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
7
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
8 try:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
9 import modules.utils as utils
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
10 except ImportError:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
11 from ReMatCh.modules import utils as utils
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
12
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
13
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
14 def get_read_run_info(ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
15 import urllib.request
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
16
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
17 url = 'http://www.ebi.ac.uk/ena/data/warehouse/filereport?accession=' + ena_id + '&result=read_run'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
18
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
19 read_run_info = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
20 try:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
21 url = urllib.request.urlopen(url)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
22 read_run_info = url.read().decode("utf8").splitlines()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
23 if len(read_run_info) <= 1:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
24 read_run_info = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
25 except Exception as error:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
26 print(error)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
27
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
28 return read_run_info
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
29
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
30
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
31 def get_download_information(read_run_info):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
32 header_line = read_run_info[0].split('\t')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
33 info_line = read_run_info[1].split('\t')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
34
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
35 download_information = {'fastq': None, 'submitted': None, 'cram_index': None}
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
36 download_types = ['aspera', 'ftp']
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
37
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
38 for i in range(0, len(header_line)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
39 header = header_line[i].lower().rsplit('_', 1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
40 if header[0] in list(download_information.keys()):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
41 if header[1] in download_types:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
42 if len(info_line[i]) > 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
43 files_path = info_line[i].split(';')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
44 if len(files_path) > 2:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
45 print('WARNING: Were found more files than expected in'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
46 ' {download_information}-{download_types} download'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
47 ' links!'.format(download_information=header[0], download_types=header[1]))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
48 if download_information[header[0]] is None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
49 download_information[header[0]] = {}
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
50 download_information[header[0]][header[1]] = files_path
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
51
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
52 return download_information
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
53
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
54
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
55 def get_sequencing_information(read_run_info):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
56 header_line = read_run_info[0].split('\t')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
57 info_line = read_run_info[1].split('\t')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
58
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
59 sequencing_information = {'run_accession': None, 'instrument_platform': None, 'instrument_model': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
60 'library_layout': None, 'library_source': None, 'extra_run_accession': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
61 'nominal_length': None, 'read_count': None, 'base_count': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
62 'date_download': time.strftime("%Y-%m-%d")}
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
63
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
64 for i in range(0, len(header_line)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
65 header = header_line[i].lower()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
66 if header in list(sequencing_information.keys()):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
67 if len(info_line[i]) > 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
68 sequencing_information[header] = info_line[i]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
69
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
70 if len(read_run_info) > 2:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
71 extra_run_accession = []
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
72 for i in range(2, len(read_run_info)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
73 info = read_run_info[i].split('\t')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
74 for j in range(0, len(header_line)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
75 header = header_line[j].lower()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
76 if header == 'run_accession':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
77 if len(info[j]) > 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
78 extra_run_accession.append(info[j])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
79 if len(extra_run_accession) >= 1:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
80 sequencing_information['extra_run_accession'] = ','.join(extra_run_accession)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
81
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
82 return sequencing_information
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
83
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
84
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
85 @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
86 def download_with_aspera(aspera_file_path, aspera_key, outdir, pickle_prefix, sra, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
87 command = ['ascp', '-QT', '-l', '300m', '', '-i', aspera_key, '', outdir]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
88 if not sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
89 command[4] = '-P33001'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
90 command[7] = str('era-fasp@' + aspera_file_path)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
91 pickle = pickle_prefix + '.' + aspera_file_path.rsplit('/', 1)[1]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
92 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
93 command[7] = 'anonftp@ftp.ncbi.nlm.nih.gov:/sra/sra-instant/reads/ByRun/sra/{a}/{b}/{c}/{c}.sra'.format(
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
94 a=ena_id[:3], b=ena_id[:6], c=ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
95 pickle = pickle_prefix + '.' + ena_id
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
96
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
97 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, 3600, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
98
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
99 utils.save_variable_to_pickle(run_successfully, outdir, pickle)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
100
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
101
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
102 @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
103 def download_with_wget(ftp_file_path, outdir, pickle_prefix, sra, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
104 command = ['wget', '--tries=1', '', '-O', '']
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
105 if not sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
106 command[2] = ftp_file_path
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
107 file_download = ftp_file_path.rsplit('/', 1)[1]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
108 command[4] = os.path.join(outdir, file_download)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
109 pickle = pickle_prefix + '.' + file_download
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
110 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
111 command[2] = 'ftp://ftp-trace.ncbi.nih.gov/sra/sra-instant/reads/ByRun/sra/{a}/{b}/{c}/{c}.sra'.format(
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
112 a=ena_id[:3], b=ena_id[:6], c=ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
113 command[4] = os.path.join(outdir, ena_id + '.sra')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
114 pickle = pickle_prefix + '.' + ena_id
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
115 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, 3600, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
116
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
117 utils.save_variable_to_pickle(run_successfully, outdir, pickle)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
118
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
119
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
120 @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
121 def download_with_sra_prefetch(aspera_key, outdir, pickle_prefix, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
122 command = ['prefetch', '', ena_id]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
123
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
124 if aspera_key is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
125 _, ascp, _ = utils.run_command_popen_communicate(['which', 'ascp'], False, None, False)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
126 command[1] = '-a {ascp}|{aspera_key}'.format(ascp=ascp.splitlines()[0], aspera_key=aspera_key)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
127
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
128 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, 3600, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
129 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
130 _, prefetch_outdir, _ = utils.run_command_popen_communicate(['echo', '$HOME/ncbi/public/sra'], True, None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
131 False)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
132
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
133 try:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
134 os.rename(os.path.join(prefetch_outdir.splitlines()[0], ena_id + '.sra'),
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
135 os.path.join(outdir, ena_id + '.sra'))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
136 except OSError as e:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
137 print('Found the following error:'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
138 '{}'.format(e))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
139
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
140 from shutil import copy as shutil_copy
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
141
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
142 shutil_copy(os.path.join(prefetch_outdir.splitlines()[0], ena_id + '.sra'),
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
143 os.path.join(outdir, ena_id + '.sra'))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
144 os.remove(os.path.join(prefetch_outdir.splitlines()[0], ena_id + '.sra'))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
145
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
146 utils.save_variable_to_pickle(run_successfully, outdir, pickle_prefix + '.' + ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
147
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
148
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
149 @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
150 def download_with_curl(ftp_file_path, outdir, pickle_prefix, sra, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
151 command = ['curl', '--retry', '1', '', '-o', '']
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
152 if not sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
153 command[3] = ftp_file_path
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
154 file_download = ftp_file_path.rsplit('/', 1)[1]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
155 command[5] = os.path.join(outdir, file_download)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
156 pickle = pickle_prefix + '.' + file_download
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
157 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
158 command[3] = 'ftp://ftp-trace.ncbi.nih.gov/sra/sra-instant/reads/ByRun/sra/{a}/{b}/{c}/{c}.sra'.format(
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
159 a=ena_id[:3], b=ena_id[:6], c=ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
160 command[5] = os.path.join(outdir, ena_id + '.sra')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
161 pickle = pickle_prefix + '.' + ena_id
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
162 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, 3600, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
163
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
164 utils.save_variable_to_pickle(run_successfully, outdir, pickle)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
165
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
166
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
167 def get_pickle_run_successfully(directory, pickle_prefix):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
168 run_successfully = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
169 read_pickle = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
170
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
171 files = find_files(directory, pickle_prefix, '.pkl')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
172 if files is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
173 for file_found in files:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
174 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
175 run_successfully = utils.extract_variable_from_pickle(file_found)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
176 read_pickle = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
177
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
178 os.remove(file_found)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
179
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
180 if not read_pickle:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
181 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
182
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
183 return run_successfully
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
184
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
185
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
186 def curl_installed():
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
187 command = ['which', 'curl']
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
188 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, None, False)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
189 return run_successfully
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
190
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
191
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
192 def download(download_information_type, aspera_key, outdir, sra, sra_opt, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
193 pickle_prefix = 'download'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
194
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
195 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
196 download_sra = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
197
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
198 if not sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
199 if aspera_key is not None and download_information_type['aspera'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
200 pool = multiprocessing.Pool(processes=2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
201 for file_download in download_information_type['aspera']:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
202 pool.apply_async(download_with_aspera, args=(file_download, aspera_key, outdir, pickle_prefix, sra,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
203 ena_id,))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
204 pool.close()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
205 pool.join()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
206 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
207 if not run_successfully and download_information_type['ftp'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
208 if curl_installed():
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
209 pool = multiprocessing.Pool(processes=2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
210 for file_download in download_information_type['ftp']:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
211 pool.apply_async(download_with_curl, args=(file_download, outdir, pickle_prefix, sra, ena_id,))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
212 pool.close()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
213 pool.join()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
214 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
215 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
216 pool = multiprocessing.Pool(processes=2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
217 for file_download in download_information_type['ftp']:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
218 pool.apply_async(download_with_wget, args=(file_download, outdir, pickle_prefix, sra, ena_id,))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
219 pool.close()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
220 pool.join()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
221 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
222
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
223 if not run_successfully and (sra or sra_opt):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
224 if aspera_key is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
225 download_with_aspera(None, aspera_key, outdir, pickle_prefix, sra or sra_opt, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
226 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
227 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
228 download_with_sra_prefetch(aspera_key, outdir, pickle_prefix, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
229 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
230 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
231 if curl_installed():
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
232 download_with_curl(None, outdir, pickle_prefix, sra or sra_opt, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
233 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
234 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
235 download_with_wget(None, outdir, pickle_prefix, sra or sra_opt, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
236 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
237
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
238 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
239 download_sra = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
240
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
241 return run_successfully, download_sra
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
242
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
243
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
244 def download_files(download_information, aspera_key, outdir, download_cram_bam_true, sra, sra_opt, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
245 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
246 cram_index_run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
247 download_sra = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
248
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
249 if download_information['fastq'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
250 run_successfully, download_sra = download(download_information['fastq'], aspera_key, outdir, sra, sra_opt,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
251 ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
252
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
253 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
254 if download_information['submitted'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
255 if not download_cram_bam_true:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
256 cram_bam = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
257 for i in download_information['submitted']:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
258 if download_information['submitted'][i][0].endswith(('.cram', '.bam')):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
259 cram_bam = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
260 break
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
261 if not cram_bam:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
262 run_successfully, download_sra = download(download_information['submitted'], aspera_key, outdir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
263 False, False, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
264
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
265 elif download_cram_bam_true:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
266 run_successfully, download_sra = download(download_information['submitted'], aspera_key, outdir, False,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
267 False, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
268 if run_successfully and download_information['cram_index'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
269 cram_index_run_successfully = download(download_information['cram_index'], aspera_key, outdir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
270 False, False, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
271
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
272 if not run_successfully and (sra or sra_opt):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
273 run_successfully, download_sra = download(download_information['fastq'], aspera_key, outdir, True, sra_opt,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
274 ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
275
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
276 return run_successfully, cram_index_run_successfully, download_sra
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
277
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
278
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
279 def sort_alignment(alignment_file, output_file, sort_by_name_true, threads):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
280 out_format_string = os.path.splitext(output_file)[1][1:].lower()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
281 command = ['samtools', 'sort', '-o', output_file, '-O', out_format_string, '', '-@', str(threads), alignment_file]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
282 if sort_by_name_true:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
283 command[6] = '-n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
284 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, None, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
285
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
286 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
287 output_file = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
288
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
289 return run_successfully, output_file
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
290
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
291
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
292 def alignment_to_fastq(alignment_file, threads, pair_end_type):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
293 fastq_basename = os.path.splitext(alignment_file)[0]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
294 outfiles = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
295 bam_file = fastq_basename + '.temp.bam'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
296 # sort cram
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
297 run_successfully, bam_file = sort_alignment(alignment_file, bam_file, True, threads)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
298 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
299 command = ['samtools', 'fastq', '', bam_file]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
300 if pair_end_type.lower() == 'paired':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
301 command[2] = '-1 ' + str(fastq_basename + '_1.fq') + ' -2 ' + str(fastq_basename + '_2.fq')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
302 elif pair_end_type == 'single':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
303 command[2] = '-0 ' + str(fastq_basename + '.fq')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
304
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
305 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, None, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
306 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
307 if pair_end_type.lower() == 'paired':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
308 outfiles = [str(fastq_basename + '_1.fq'), str(fastq_basename + '_2.fq')]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
309 elif pair_end_type.lower() == 'single':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
310 outfiles = [str(fastq_basename + '.fq')]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
311
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
312 if bam_file is not None and os.path.isfile(bam_file):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
313 os.remove(bam_file)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
314
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
315 return run_successfully, outfiles
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
316
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
317
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
318 def formart_fastq_headers(in_fastq_1, in_fastq_2):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
319
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
320 out_fastq_1 = in_fastq_1 + '.temp'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
321 out_fastq_2 = in_fastq_2 + '.temp'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
322 writer_in_fastq_1 = open(out_fastq_1, 'wt')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
323 writer_in_fastq_2 = open(out_fastq_2, 'wt')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
324 outfiles = [out_fastq_1, out_fastq_2]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
325 with open(in_fastq_1, 'rtU') as reader_in_fastq_1, open(in_fastq_2, 'rtU') as reader_in_fastq_2:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
326 plus_line = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
327 quality_line = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
328 number_reads = 0
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
329 for in_1, in_2 in zip(reader_in_fastq_1, reader_in_fastq_2):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
330 if len(in_1) > 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
331 in_1 = in_1.splitlines()[0]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
332 in_2 = in_2.splitlines()[0]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
333 if in_1.startswith('@') and plus_line and quality_line:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
334 if in_1 != in_2:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
335 sys.exit('The PE fastq files are not aligned properly!')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
336 in_1 += '/1' + '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
337 in_2 += '/2' + '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
338 writer_in_fastq_1.write(in_1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
339 writer_in_fastq_2.write(in_2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
340 plus_line = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
341 quality_line = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
342 elif in_1.startswith('+') and not plus_line:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
343 in_1 += '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
344 writer_in_fastq_1.write(in_1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
345 writer_in_fastq_2.write(in_1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
346 plus_line = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
347 elif plus_line and not quality_line:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
348 in_1 += '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
349 in_2 += '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
350 writer_in_fastq_1.write(in_1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
351 writer_in_fastq_2.write(in_2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
352 writer_in_fastq_1.flush()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
353 writer_in_fastq_2.flush()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
354 number_reads += 1
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
355 quality_line = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
356 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
357 in_1 += '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
358 in_2 += '\n'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
359 writer_in_fastq_1.write(in_1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
360 writer_in_fastq_2.write(in_2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
361 return number_reads, outfiles
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
362
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
363
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
364 @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
365 def gzip_files(file_2_compress, pickle_prefix, outdir):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
366 if file_2_compress.endswith('.temp'):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
367 out_file = os.path.splitext(file_2_compress)[0]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
368 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
369 out_file = file_2_compress
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
370
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
371 command = ['gzip', '--stdout', '--best', file_2_compress, '>', str(out_file + '.gz')]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
372 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, True, None, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
373 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
374 os.remove(file_2_compress)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
375
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
376 utils.save_variable_to_pickle(run_successfully, outdir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
377 str(pickle_prefix + '.' + os.path.basename(file_2_compress)))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
378
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
379
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
380 def find_files(directory, prefix, suffix):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
381 list_files_found = []
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
382 files = [f for f in os.listdir(directory) if not f.startswith('.') and os.path.isfile(os.path.join(directory, f))]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
383 for file_found in files:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
384 if file_found.startswith(prefix) and file_found.endswith(suffix):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
385 file_path = os.path.join(directory, file_found)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
386 list_files_found.append(file_path)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
387
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
388 if len(list_files_found) == 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
389 list_files_found = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
390
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
391 return list_files_found
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
392
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
393
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
394 def compress_files(fastq_files, outdir, threads):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
395 pickle_prefix = 'compress'
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
396 compressed_fastq_files = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
397
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
398 pool = multiprocessing.Pool(processes=threads)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
399 for fastq in fastq_files:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
400 pool.apply_async(gzip_files, args=(fastq, pickle_prefix, outdir,))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
401 pool.close()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
402 pool.join()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
403
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
404 run_successfully = get_pickle_run_successfully(outdir, pickle_prefix)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
405 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
406 compressed_fastq_files = find_files(outdir, '', '.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
407
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
408 return run_successfully, compressed_fastq_files
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
409
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
410
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
411 def bam_cram_2_fastq(alignment_file, outdir, threads, pair_end_type):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
412 run_successfully, fastq_files = alignment_to_fastq(alignment_file, threads, pair_end_type)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
413 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
414 if pair_end_type.lower() == 'paired':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
415 number_reads, fastq_files = formart_fastq_headers(fastq_files[0], fastq_files[1])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
416
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
417 run_successfully, fastq_files = compress_files(fastq_files, outdir, threads)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
418
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
419 return run_successfully, fastq_files
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
420
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
421
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
422 def check_correct_links(download_information):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
423 for i in download_information:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
424 if download_information[i] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
425 if download_information[i]['aspera'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
426 for j in range(0, len(download_information[i]['aspera'])):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
427 if download_information[i]['aspera'][j].startswith('fasp.sra.ebi.ac.uk/'):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
428 download_information[i]['aspera'][j] = download_information[i]['aspera'][j].replace(
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
429 'fasp.sra.ebi.ac.uk/', 'fasp.sra.ebi.ac.uk:/', 1)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
430 if download_information[i]['ftp'] is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
431 for j in range(0, len(download_information[i]['ftp'])):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
432 if '#' in download_information[i]['ftp'][j]:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
433 download_information[i]['ftp'][j] = download_information[i]['ftp'][j].replace('#', '%23')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
434 return download_information
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
435
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
436
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
437 def get_fastq_files(download_dir, cram_index_run_successfully, threads, download_paired_type):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
438 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
439 downloaded_files = find_files(download_dir, '', '')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
440 if cram_index_run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
441 cram_file = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
442 for i in downloaded_files:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
443 if i.endswith('.cram'):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
444 cram_file = i
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
445 run_successfully, downloaded_files = bam_cram_2_fastq(cram_file, download_dir, threads, download_paired_type)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
446 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
447 if downloaded_files is not None and len(downloaded_files) > 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
448 run_successfully = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
449
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
450 return run_successfully, downloaded_files
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
451
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
452
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
453 def rename_move_files(list_files, new_name, outdir, download_paired_type):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
454 list_new_files = {}
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
455 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
456
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
457 for i in range(0, len(list_files)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
458 temp_name = utils.rchop(os.path.basename(list_files[i]), 'astq.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
459 if len(temp_name) == len(os.path.basename(list_files[i])):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
460 temp_name = utils.rchop(os.path.basename(list_files[i]), 'q.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
461 if download_paired_type.lower() == 'paired':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
462 if temp_name.endswith(('_R1_001.f', '_1.f')):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
463 list_new_files[i] = os.path.join(outdir, new_name + '_1.fq.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
464 elif temp_name.endswith(('_R2_001.f', '_2.f')):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
465 list_new_files[i] = os.path.join(outdir, new_name + '_2.fq.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
466 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
467 if not temp_name.endswith(('_R1_001.f', '_R2_001.f')):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
468 list_new_files[i] = os.path.join(outdir, new_name + '.fq.gz')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
469 if temp_name.endswith(('_1.f', '_2.f')):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
470 print('WARNING: possible single-end file conflict with pair-end (' + list_files[i] + ')!')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
471
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
472 if len(list_new_files) == 2 and download_paired_type.lower() == 'paired':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
473 run_successfully = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
474 elif len(list_new_files) == 1 and download_paired_type.lower() == 'single':
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
475 run_successfully = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
476
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
477 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
478 try:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
479 for i in range(0, len(list_files)):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
480 if i not in list_new_files:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
481 if os.path.isfile(list_files[i]):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
482 os.remove(list_files[i])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
483 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
484 os.rename(list_files[i], list_new_files[i])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
485 list_new_files = list(list_new_files.values())
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
486 except Exception as e:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
487 print(e)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
488 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
489
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
490 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
491 list_new_files = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
492
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
493 return run_successfully, list_new_files
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
494
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
495
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
496 # @utils.trace_unhandled_exceptions
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
497 def rename_header_sra(fastq):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
498 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
499 try:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
500 command = ['gawk', '\'{if(NR%4==1) $0=gensub(/\./, \"/\", 2); print}\'', fastq, '|', 'gzip', '-1', '>',
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
501 str(fastq + '.gz')]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
502 print('Running: ' + str(' '.join(command)))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
503 return_code = subprocess.call(' '.join(command), shell=True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
504 if return_code == 0:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
505 run_successfully = True
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
506 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
507 print('Something went wrong with command: {command}'.format(command=' '.join(command)))
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
508 except Exception as e:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
509 print(e)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
510
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
511 return run_successfully
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
512
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
513
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
514 def sra_2_fastq(download_dir, ena_id):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
515 command = ['fastq-dump', '-I', '-O', download_dir, '--split-files', '{download_dir}{ena_id}.sra'.format(
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
516 download_dir=download_dir, ena_id=ena_id)]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
517 run_successfully, stdout, stderr = utils.run_command_popen_communicate(command, False, 3600, True)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
518 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
519 files = [os.path.join(download_dir, f) for f in os.listdir(download_dir)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
520 if not f.startswith('.') and os.path.isfile(os.path.join(download_dir, f)) and f.endswith('.fastq')]
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
521
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
522 pool = multiprocessing.Pool(processes=2)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
523 results = []
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
524 p = pool.map_async(rename_header_sra, files, callback=results.extend)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
525 p.wait()
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
526
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
527 run_successfully = all(results)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
528
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
529 return run_successfully
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
530
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
531
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
532 download_timer = functools.partial(utils.timer, name='Download module')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
533
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
534
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
535 @download_timer
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
536 def run_download(ena_id, download_paired_type, aspera_key, outdir, download_cram_bam_true, threads, instrument_platform,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
537 sra, sra_opt):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
538 download_dir = os.path.join(outdir, 'download', '')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
539 utils.remove_directory(download_dir)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
540 os.mkdir(download_dir)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
541
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
542 run_successfully = False
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
543 downloaded_files = None
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
544 sequencing_information = {'run_accession': None, 'instrument_platform': None, 'instrument_model': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
545 'library_layout': None, 'library_source': None, 'extra_run_accession': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
546 'nominal_length': None, 'read_count': None, 'base_count': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
547 'date_download': time.strftime("%Y-%m-%d")}
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
548
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
549 read_run_info = get_read_run_info(ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
550 if read_run_info is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
551 download_information = get_download_information(read_run_info)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
552 download_information = check_correct_links(download_information)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
553 sequencing_information = get_sequencing_information(read_run_info)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
554
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
555 if instrument_platform.lower() == 'all' or \
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
556 (sequencing_information['instrument_platform'] is not None and
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
557 sequencing_information['instrument_platform'].lower() == instrument_platform.lower()):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
558 if download_paired_type.lower() == 'both' or \
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
559 (sequencing_information['library_layout'] is not None and
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
560 sequencing_information['library_layout'].lower() == download_paired_type.lower()):
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
561 run_successfully, cram_index_run_successfully, download_sra = download_files(download_information,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
562 aspera_key, download_dir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
563 download_cram_bam_true,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
564 sra, sra_opt, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
565 if download_sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
566 run_successfully = sra_2_fastq(download_dir, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
567 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
568 run_successfully, downloaded_files = get_fastq_files(download_dir, cram_index_run_successfully,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
569 threads,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
570 sequencing_information['library_layout'])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
571 if run_successfully and downloaded_files is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
572 run_successfully, downloaded_files = rename_move_files(downloaded_files,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
573 sequencing_information['run_accession'],
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
574 outdir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
575 sequencing_information['library_layout'])
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
576 else:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
577 if sra or sra_opt:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
578 run_successfully, cram_index_run_successfully, download_sra = download_files({'fastq': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
579 'submitted': None,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
580 'cram_index': None},
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
581 aspera_key, download_dir,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
582 download_cram_bam_true, sra,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
583 sra_opt, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
584 if download_sra:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
585 run_successfully = sra_2_fastq(download_dir, ena_id)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
586 if run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
587 run_successfully, downloaded_files = get_fastq_files(download_dir, cram_index_run_successfully, threads,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
588 'paired')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
589 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
590 run_successfully, downloaded_files = get_fastq_files(download_dir, cram_index_run_successfully,
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
591 threads, 'single')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
592 if run_successfully and downloaded_files is not None:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
593 run_successfully, downloaded_files = rename_move_files(downloaded_files, ena_id, outdir, 'paired')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
594 if not run_successfully:
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
595 run_successfully, downloaded_files = rename_move_files(downloaded_files, ena_id, outdir, 'single')
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
596
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
597 utils.remove_directory(download_dir)
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
598
c6bab5103a14 "planemo upload commit 6abf3e299d82d07e6c3cf8642bdea80e96df64c3-dirty"
iss
parents:
diff changeset
599 return run_successfully, downloaded_files, sequencing_information