Mercurial > repos > iuc > data_manager_mothur_toolsuite

comparison data_manager/fetch_mothur_reference_data.py @ 2:0e532fc0a0a6 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_mothur_toolsuite/ commit b824ec6d0faa5353c5893b5a2a540b8251cae309"
author iuc
date Thu, 17 Sep 2020 09:37:41 +0000
parents aec831b54a5b
children 9d09724f2bf1
comparison
equal deleted inserted replaced
1:aec831b54a5b 2:0e532fc0a0a6
8 import sys 8 import sys
9 import tarfile 9 import tarfile
10 import tempfile 10 import tempfile
11 import urllib2 11 import urllib2
12 import zipfile 12 import zipfile
13 from functools import reduce
13 14
14 # When extracting files from archives, skip names that 15 # When extracting files from archives, skip names that
15 # start with the following strings 16 # start with the following strings
16 IGNORE_PATHS = ('.', '__MACOSX/', '__') 17 IGNORE_PATHS = ('.', '__MACOSX/', '__')
17 18
25 # Reference data URLs 26 # Reference data URLs
26 MOTHUR_REFERENCE_DATA = { 27 MOTHUR_REFERENCE_DATA = {
27 # Look up data 28 # Look up data
28 # http://www.mothur.org/wiki/Lookup_files 29 # http://www.mothur.org/wiki/Lookup_files
29 "lookup_titanium": { 30 "lookup_titanium": {
30 "GS FLX Titanium": ["http://www.mothur.org/w/images/9/96/LookUp_Titanium.zip", ] 31 "GS FLX Titanium": ["https://mothur.s3.us-east-2.amazonaws.com/wiki/lookup_titanium.zip", ]
31 }, 32 },
32 "lookup_gsflx": { 33 "lookup_gsflx": {
33 "GSFLX": ["http://www.mothur.org/w/images/8/84/LookUp_GSFLX.zip", ] 34 "GSFLX": ["https://mothur.s3.us-east-2.amazonaws.com/wiki/lookup_gsflx.zip", ]
34 }, 35 },
35 "lookup_gs20": { 36 "lookup_gs20": {
36 "GS20": ["http://www.mothur.org/w/images/7/7b/LookUp_GS20.zip", ] 37 "GS20": ["https://mothur.s3.us-east-2.amazonaws.com/wiki/lookup_gs20.zip", ]
37 }, 38 },
38 # RDP reference files 39 # RDP reference files
39 # http://www.mothur.org/wiki/RDP_reference_files 40 # http://www.mothur.org/wiki/RDP_reference_files
40 "RDP_v16": { 41 "RDP_v16": {
41 "16S rRNA RDP training set 16": 42 "16S rRNA RDP training set 16":
42 ["https://mothur.org/w/images/d/dc/Trainset16_022016.rdp.tgz", ], 43 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset16_022016.rdp.tgz", ],
43 "16S rRNA PDS training set 16": 44 "16S rRNA PDS training set 16":
44 ["https://mothur.org/w/images/c/c3/Trainset16_022016.pds.tgz", ], 45 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset16_022016.pds.tgz", ],
45 }, 46 },
46 "RDP_v14": { 47 "RDP_v14": {
47 "16S rRNA RDP training set 14": 48 "16S rRNA RDP training set 14":
48 ["https://mothur.org/w/images/6/6c/Trainset14_032015.rdp.tgz", ], 49 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset14_032015.rdp.tgz", ],
49 "16S rRNA PDS training set 14": 50 "16S rRNA PDS training set 14":
50 ["https://mothur.org/w/images/8/88/Trainset14_032015.pds.tgz", ], 51 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset14_032015.pds.tgz", ],
51 }, 52 },
52 "RDP_v10": { 53 "RDP_v10": {
53 "16S rRNA RDP training set 10": 54 "16S rRNA RDP training set 10":
54 ["http://www.mothur.org/w/images/b/b5/Trainset10_082014.rdp.tgz", ], 55 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset10_082014.rdp.tgz", ],
55 "16S rRNA PDS training set 10": 56 "16S rRNA PDS training set 10":
56 ["http://www.mothur.org/w/images/2/24/Trainset10_082014.pds.tgz", ], 57 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset10_082014.pds.tgz", ],
57 }, 58 },
58 "RDP_v9": { 59 "RDP_v9": {
59 "16S rRNA RDP training set 9": 60 "16S rRNA RDP training set 9":
60 ["http://www.mothur.org/w/images/7/72/Trainset9_032012.rdp.zip", ], 61 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset9_032012.rdp.zip", ],
61 "16S rRNA PDS training set 9": 62 "16S rRNA PDS training set 9":
62 ["http://www.mothur.org/w/images/5/59/Trainset9_032012.pds.zip", ], 63 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset9_032012.pds.zip", ],
63 }, 64 },
64 "RDP_v7": { 65 "RDP_v7": {
65 "16S rRNA RDP training set 7": 66 "16S rRNA RDP training set 7":
66 ["http://www.mothur.org/w/images/2/29/Trainset7_112011.rdp.zip", ], 67 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset7_112011.rdp.zip", ],
67 "16S rRNA PDS training set 7": 68 "16S rRNA PDS training set 7":
68 ["http://www.mothur.org/w/images/4/4a/Trainset7_112011.pds.zip", ], 69 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/trainset7_112011.pds.zip", ],
69 "8S rRNA Fungi training set 7": 70 "8S rRNA Fungi training set 7":
70 ["http://www.mothur.org/w/images/3/36/FungiLSU_train_v7.zip", ], 71 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/fungilsu_train_v7.zip", ],
71 }, 72 },
72 "RDP_v6": { 73 "RDP_v6": {
73 "RDP training set 6": 74 "RDP training set 6":
74 ["http://www.mothur.org/w/images/4/49/RDPTrainingSet.zip", ], 75 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/rdptrainingset.zip", ],
75 }, 76 },
76 # Silva reference files 77 # Silva reference files
77 # http://www.mothur.org/wiki/Silva_reference_files 78 # http://www.mothur.org/wiki/Silva_reference_files
78 "silva_release_128": { 79 "silva_release_128": {
79 "SILVA release 128": 80 "SILVA release 128":
80 ["https://mothur.org/w/images/b/b4/Silva.nr_v128.tgz", 81 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.nr_v128.tgz",
81 "https://mothur.org/w/images/a/a4/Silva.seed_v128.tgz", ], 82 "https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.seed_v128.tgz", ],
82 }, 83 },
83 "silva_release_123": { 84 "silva_release_123": {
84 "SILVA release 123": 85 "SILVA release 123":
85 ["https://mothur.org/w/images/b/be/Silva.nr_v123.tgz", 86 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.nr_v123.tgz",
86 "https://mothur.org/w/images/1/15/Silva.seed_v123.tgz", ], 87 "https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.seed_v123.tgz", ],
87 }, 88 },
88 "silva_release_119": { 89 "silva_release_119": {
89 "SILVA release 119": 90 "SILVA release 119":
90 ["http://www.mothur.org/w/images/2/27/Silva.nr_v119.tgz", 91 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.nr_v119.tgz",
91 "http://www.mothur.org/w/images/5/56/Silva.seed_v119.tgz", ], 92 "https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.seed_v119.tgz", ],
92 }, 93 },
93 "silva_release_102": { 94 "silva_release_102": {
94 "SILVA release 102": 95 "SILVA release 102":
95 ["http://www.mothur.org/w/images/9/98/Silva.bacteria.zip", 96 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.bacteria.zip",
96 "http://www.mothur.org/w/images/3/3c/Silva.archaea.zip", 97 "https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.archaea.zip",
97 "http://www.mothur.org/w/images/1/1a/Silva.eukarya.zip", ], 98 "https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.eukarya.zip", ],
98 }, 99 },
99 "silva_gold_bacteria": { 100 "silva_gold_bacteria": {
100 "SILVA gold": 101 "SILVA gold":
101 ["http://www.mothur.org/w/images/f/f1/Silva.gold.bacteria.zip", ], 102 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva.gold.bacteria.zip", ],
102 }, 103 },
103 # Greengenes 104 # Greengenes
104 # http://www.mothur.org/wiki/Greengenes-formatted_databases 105 # http://www.mothur.org/wiki/Greengenes-formatted_databases
105 "greengenes_August2013": { 106 "greengenes_August2013": {
106 "Greengenes August 2013": 107 "Greengenes August 2013":
107 ["http://www.mothur.org/w/images/1/19/Gg_13_8_99.refalign.tgz", 108 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/gg_13_8_99.refalign.tgz",
108 "http://www.mothur.org/w/images/6/68/Gg_13_8_99.taxonomy.tgz", ], 109 "https://mothur.s3.us-east-2.amazonaws.com/wiki/gg_13_8_99.taxonomy.tgz", ],
109 }, 110 },
110 "greengenes_May2013": { 111 "greengenes_May2013": {
111 "Greengenes May 2013": 112 "Greengenes May 2013":
112 ["http://www.mothur.org/w/images/c/cd/Gg_13_5_99.refalign.tgz", 113 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/gg_13_5_99.refalign.tgz",
113 "http://www.mothur.org/w/images/9/9d/Gg_13_5_99.taxonomy.tgz", ], 114 "https://mothur.s3.us-east-2.amazonaws.com/wiki/gg_13_5_99.taxonomy.tgz", ],
114 }, 115 },
115 "greengenes_old": { 116 "greengenes_old": {
116 "Greengenes pre-May 2013": 117 "Greengenes pre-May 2013":
117 ["http://www.mothur.org/w/images/7/72/Greengenes.alignment.zip", 118 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/greengenes.alignment.zip",
118 "http://www.mothur.org/w/images/1/16/Greengenes.tax.tgz", ], 119 "https://mothur.s3.us-east-2.amazonaws.com/wiki/greengenes.tax.tgz", ],
119 }, 120 },
120 "greengenes_gold_alignment": { 121 "greengenes_gold_alignment": {
121 "Greengenes gold alignment": 122 "Greengenes gold alignment":
122 ["http://www.mothur.org/w/images/2/21/Greengenes.gold.alignment.zip", ], 123 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/greengenes.gold.alignment.zip", ],
123 }, 124 },
124 # Secondary structure maps 125 # Secondary structure maps
125 # http://www.mothur.org/wiki/Secondary_structure_map 126 # http://www.mothur.org/wiki/Secondary_structure_map
126 "secondary_structure_maps_silva": { 127 "secondary_structure_maps_silva": {
127 "SILVA": 128 "SILVA":
128 ["http://www.mothur.org/w/images/6/6d/Silva_ss_map.zip", ], 129 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/silva_ss_map.zip", ],
129 }, 130 },
130 "secondary_structure_maps_greengenes": { 131 "secondary_structure_maps_greengenes": {
131 "Greengenes": 132 "Greengenes":
132 ["http://www.mothur.org/w/images/4/4b/Gg_ss_map.zip", ], 133 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/gg_ss_map.zip", ],
133 }, 134 },
134 # Lane masks: not used here? 135 # Lane masks: not used here?
135 "lane_masks": { 136 "lane_masks": {
136 "Greengenes-compatible": 137 "Greengenes-compatible":
137 ["http://www.mothur.org/w/images/2/2a/Lane1241.gg.filter", 138 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/Lane1241.gg.filter",
138 "http://www.mothur.org/w/images/a/a0/Lane1287.gg.filter", 139 "https://mothur.s3.us-east-2.amazonaws.com/wiki/lane1287.gg.filter",
139 "http://www.mothur.org/w/images/3/3d/Lane1349.gg.filter", ], 140 "https://mothur.s3.us-east-2.amazonaws.com/wiki/lane1349.gg.filter", ],
140 "SILVA-compatible": 141 "SILVA-compatible":
141 ["http://www.mothur.org/w/images/6/6d/Lane1349.silva.filter", ] 142 ["https://mothur.s3.us-east-2.amazonaws.com/wiki/lane1349.silva.filter", ]
142 }, 143 },
143 } 144 }
144 145
145 146
146 # Utility functions for interacting with Galaxy JSON 147 # Utility functions for interacting with Galaxy JSON