Mercurial > repos > iuc > data_manager_pangolearn

annotate data_manager/pangolearn_dm.py @ 4:6e24e79d3d69 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
author iuc
date Tue, 05 Apr 2022 18:40:07 +0000
parents df30a2f1db55
children 29c738066906
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
1 #!/usr/bin/env python
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
2
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
3 import argparse
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
4 import datetime
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
5 import json
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
6 import operator
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
7 import os
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
8 import shutil
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
9 import sys
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
10 import tarfile
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
11
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
12 import requests
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
13
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
14
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
15 def get_model_list(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
16 existing_release_tags,
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
17 url="https://api.github.com/repos/cov-lineages/pangoLEARN/releases"
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
18 ):
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
19 page_num = 0
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
20 while True:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
21 page_num += 1
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
22 response = requests.get(url + f'?page={page_num}')
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
23 if response.status_code == 200:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
24 release_list_chunk = json.loads(response.text)
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
25 if not release_list_chunk:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
26 # past the last page of results
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
27 return
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
28 for e in release_list_chunk:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
29 if e["tag_name"] in existing_release_tags:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
30 continue
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
31 if e["prerelease"]:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
32 continue
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
33 yield dict(
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
34 tag_name=e["tag_name"],
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
35 name=e["name"],
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
36 date=parse_date(e["tag_name"]),
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
37 tarball_url=e["tarball_url"],
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
38 )
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
39 else:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
40 response.raise_for_status()
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
41
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
42
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
43 def filter_by_date(existing_release_tags, start_date=None, end_date=None):
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
44 ret = []
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
45 for release in get_model_list(existing_release_tags):
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
46 if start_date and release["date"] < start_date:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
47 break
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
48 if not end_date or release["date"] <= end_date:
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
49 ret.append(release)
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
50
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
51 return ret
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
52
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
53
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
54 def download_and_unpack(url, output_directory):
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
55 response = requests.get(url)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
56 if response.status_code == 200:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
57 tmp_filename = url.split("/")[-1]
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
58 tmpfile = open(tmp_filename, "wb")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
59 tmpfile.write(response.content)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
60 tmpfile.close()
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
61 shutil.copy(tmp_filename, "/tmp")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
62 tf = tarfile.open(tmp_filename)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
63 pl_path = tf.next().name
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
64 tf.extractall(output_directory)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
65 os.unlink(tmp_filename)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
66 os.rename(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
67 output_directory + "/" + pl_path + "/" + "pangoLEARN",
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
68 output_directory + "/" + tmp_filename,
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
69 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
70 shutil.rmtree(output_directory + "/" + pl_path)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
71 return tmp_filename
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
72 else:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
73 response.raise_for_status()
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
74
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
75
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
76 def parse_date(d):
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
77 # Tries to parse the first 10 chars of d as a date, which currently
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
78 # succeeds for all pangolearn model releases.
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
79 return datetime.datetime.strptime(d[:10], "%Y-%m-%d")
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
80
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
81
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
82 if __name__ == "__main__":
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
83
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
84 parser = argparse.ArgumentParser()
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
85 parser.add_argument("--testmode", default=False, action="store_true")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
86 parser.add_argument("--latest", default=False, action="store_true")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
87 parser.add_argument("--start_date", type=parse_date)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
88 parser.add_argument("--end_date", type=parse_date)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
89 parser.add_argument("--overwrite", default=False, action="store_true")
1
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
90 parser.add_argument('--pangolearn_format_version')
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
91 parser.add_argument("datatable_name")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
92 parser.add_argument("galaxy_datamanager_filename")
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
93 args = parser.parse_args()
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
94
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
95 if args.testmode:
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
96 releases = filter_by_date([], start_date=args.start_date, end_date=args.end_date)
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
97 for release in releases:
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
98 print(release["tag_name"], release["tarball_url"].split("/")[-1], release["date"])
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
99 sys.exit(0)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
100
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
101 with open(args.galaxy_datamanager_filename) as fh:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
102 config = json.load(fh)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
103
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
104 output_directory = config.get("output_data", [{}])[0].get("extra_files_path", None)
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
105 data_manager_dict = {}
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
106 data_manager_dict["data_tables"] = config.get("data_tables", {})
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
107 data_manager_dict["data_tables"][args.datatable_name] = data_manager_dict[
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
108 "data_tables"
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
109 ].get(args.datatable_name, [])
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
110
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
111 # NOTE: the data_manager_dict["data_tables"][args.datatable_name] is not actually populated with the
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
112 # contents of the existing data table, so the "no-overwrite" logic and the
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
113 # only-download-what-we-don't-have logic does not in fact work. It is left but unused for now.
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
114 if not args.overwrite:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
115 existing_release_tags = set(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
116 [
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
117 el["value"]
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
118 for el in data_manager_dict["data_tables"][args.datatable_name]
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
119 ]
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
120 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
121 else:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
122 existing_release_tags = set()
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
123 if args.latest:
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
124 releases = [next(get_model_list(existing_release_tags))]
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
125 else:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
126 releases = filter_by_date(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
127 existing_release_tags, start_date=args.start_date, end_date=args.end_date
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
128 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
129 releases_to_download = [
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
130 release
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
131 for release in releases
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
132 if release["tag_name"] not in existing_release_tags
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
133 ]
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
134 for release in releases_to_download:
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
135 fname = download_and_unpack(release["tarball_url"], output_directory)
1
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
136 if args.pangolearn_format_version is not None:
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
137 version = args.pangolearn_format_version
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
138 else:
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
139 # 2021-05-27 was the first release of pangoLEARN for pangolin 3, which changed DB format
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
140 if release["date"] >= datetime.datetime(2021, 5, 27):
1
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
141 version = '3.0'
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
142 else:
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
143 version = '1.0'
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
144 data_manager_dict["data_tables"][args.datatable_name].append(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
145 dict(
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
146 value=release["tag_name"],
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
147 description=release["name"],
1
112bb7a9da3c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
148 format_version=version,
4
6e24e79d3d69 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
149 path=output_directory + "/" + fname,
0
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
150 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
151 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
152 data_manager_dict["data_tables"][args.datatable_name].sort(
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
153 key=operator.itemgetter("value"), reverse=True
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
154 )
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
155 with open(args.galaxy_datamanager_filename, "w") as fh:
3eac657893fe "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
156 json.dump(data_manager_dict, fh, indent=2, sort_keys=True)