|
0
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 ## **@AUTHOR**: Lain Pavot - lain.pavot@inrae.fr
|
|
|
4 ## **@DATE**: 22/06/2022
|
|
|
5
|
|
|
6
|
|
|
7 import json
|
|
|
8 import os
|
|
|
9 import sys
|
|
|
10
|
|
|
11
|
|
1
|
12 STATIC = os.path.join(sys.path[0], "static")
|
|
|
13 VENDOR = os.path.join(sys.path[0], "vendor")
|
|
|
14
|
|
|
15 with open(os.path.join(STATIC, "app.css")) as css:
|
|
0
|
16 CSS_STYLES = css.read()
|
|
|
17
|
|
1
|
18 with open(os.path.join(VENDOR, "bootstrap.min.css")) as bootstrap:
|
|
0
|
19 CSS_STYLES = f"{CSS_STYLES}\n{bootstrap.read()}"
|
|
|
20
|
|
1
|
21 with open(os.path.join(STATIC, "app.js")) as js:
|
|
0
|
22 JAVASCRIPT = js.read()
|
|
|
23
|
|
1
|
24 with open(os.path.join(STATIC, "app.template.html")) as template:
|
|
0
|
25 PAGE_TEMPLATE = template.read()
|
|
|
26
|
|
1
|
27 with open(os.path.join(STATIC, "title.template.html")) as template:
|
|
0
|
28 TITLE_TEMPLATE = template.read()
|
|
|
29
|
|
1
|
30 with open(os.path.join(STATIC, "table.template.html")) as template:
|
|
0
|
31 TABLE_TEMPLATE = template.read()
|
|
|
32
|
|
1
|
33 with open(os.path.join(STATIC, "header_list.template.html")) as template:
|
|
0
|
34 HEADER_LIST_TEMPLATE = template.read()
|
|
|
35
|
|
|
36 HEADER_LIST_TEMPLATE = '\n'.join((
|
|
|
37 "<thead>",
|
|
|
38 " <tr>",
|
|
|
39 "{header_list}",
|
|
|
40 " </tr>",
|
|
|
41 "</thead>",
|
|
|
42 ))
|
|
|
43
|
|
|
44 HEADER_TEMPLATE = "<th scope=\"col\">{}</th>"
|
|
|
45 COLUMN_TEMPLATE = "<th scope=\"row\">{}</th>"
|
|
|
46
|
|
|
47 TABLE_LINE_LIST_TEMPLATE = '\n'.join((
|
|
|
48 "<tr class=\"{classes}\">",
|
|
|
49 "{table_lines}",
|
|
|
50 "</tr>",
|
|
|
51 ))
|
|
|
52 TABLE_LINE_TEMPLATE = "<td>{}</td>"
|
|
|
53
|
|
|
54 INDENT = " "
|
|
|
55
|
|
|
56
|
|
|
57 HISTORY_CACHE = {}
|
|
|
58
|
|
|
59 def indent(text):
|
|
|
60 if text.startswith("\n"):
|
|
|
61 return text.replace("\n", f"\n{INDENT}")
|
|
|
62 else:
|
|
|
63 return INDENT+text.replace("\n", f"\n{INDENT}")
|
|
|
64
|
|
|
65 def noempty(ls, as_list=True):
|
|
|
66 if as_list:
|
|
|
67 return [x for x in ls if x]
|
|
|
68 return (x for x in ls if x)
|
|
|
69
|
|
|
70 def join_noempty(ls, sep=';'):
|
|
|
71 return sep.join(noempty(ls, as_list=False))
|
|
|
72
|
|
|
73 def extract_dataset_attributes(dataset_attrs):
|
|
|
74 for dataset_attr in dataset_attrs:
|
|
|
75 HISTORY_CACHE[dataset_attr["encoded_id"]] = dataset_attr
|
|
|
76 HISTORY_CACHE["dataset_attrs"] = dataset_attrs
|
|
|
77
|
|
|
78 def convert_to_html(jobs_attrs, dataset_attrs=None):
|
|
|
79 if dataset_attrs:
|
|
|
80 extract_dataset_attributes(dataset_attrs)
|
|
|
81 return PAGE_TEMPLATE.format(
|
|
|
82 styles=CSS_STYLES.replace("\n<", "\n <"),
|
|
|
83 javascript=JAVASCRIPT,
|
|
|
84 title=indent(indent(get_title(jobs_attrs))),
|
|
|
85 table_list=indent(indent(get_table_list(jobs_attrs)))
|
|
|
86 )
|
|
|
87
|
|
|
88 def get_title(jobs_attrs):
|
|
|
89 galaxy_version = jobs_attrs[0]["galaxy_version"] or "Unknown version"
|
|
|
90 return TITLE_TEMPLATE.format(galaxy_version=galaxy_version)
|
|
|
91
|
|
|
92 def get_table_list(jobs_attrs):
|
|
|
93 return '\n'.join((
|
|
|
94 convert_item_to_table(job_attr, dataset_id)
|
|
|
95 for job_attr in jobs_attrs
|
|
|
96 for dataset_id_set in sorted(list((
|
|
|
97 job_attr["output_dataset_mapping"]
|
|
|
98 or {1:"unknown"}
|
|
|
99 ).values()))
|
|
|
100 for dataset_id in sorted(dataset_id_set)
|
|
|
101 ))
|
|
|
102
|
|
|
103 def convert_item_to_table(job_attr, dataset_id):
|
|
|
104 encoded_jid = job_attr.get("encoded_id")
|
|
|
105 if HISTORY_CACHE:
|
|
|
106 history = HISTORY_CACHE.get(dataset_id, {})
|
|
|
107 hid = history.get("hid", "DELETED")
|
|
|
108 else:
|
|
|
109 hid = "?"
|
|
|
110 exit_code = job_attr.get("exit_code")
|
|
|
111 if job_attr["exit_code"] == 0:
|
|
|
112 status = f"Ok ({exit_code})"
|
|
|
113 classes = "alert alert-success"
|
|
|
114 else:
|
|
|
115 status = f"Failed ({exit_code})"
|
|
|
116 classes = "alert alert-danger"
|
|
|
117 if hid == "DELETED":
|
|
|
118 classes += " history_metadata_extractor_deleted"
|
|
|
119 tool_name = job_attr["tool_id"] or "unknown"
|
|
|
120 if tool_name.count("/") >= 4:
|
|
|
121 tool_name = job_attr["tool_id"].split("/")[-2]
|
|
|
122 tool_name = tool_name + " - " + job_attr["tool_version"]
|
|
|
123 tool_name = f"[{hid}] - {tool_name}"
|
|
|
124 return TABLE_TEMPLATE.format(
|
|
|
125 classes=classes,
|
|
|
126 tool_name=tool_name,
|
|
|
127 tool_status=status,
|
|
|
128 table=convert_parameters_to_html(job_attr)
|
|
|
129 )
|
|
|
130
|
|
|
131 def convert_parameters_to_html(job_attr):
|
|
|
132 params = job_attr["params"]
|
|
|
133 params_enrichment(job_attr, params)
|
|
|
134 keys = [
|
|
|
135 key for key in iter_parameter_keys(params)
|
|
|
136 if key not in ("dbkey", "chromInfo", "__input_ext", "request_json")
|
|
|
137 ]
|
|
|
138 return '\n'.join((
|
|
|
139 indent(get_table_header(params, ["value", "name", "extension", "hid"])),
|
|
|
140 indent(get_table_lines(params, keys)),
|
|
|
141 ))
|
|
|
142
|
|
|
143 def params_enrichment(job_attr, params):
|
|
|
144 if (
|
|
|
145 all(map(params.__contains__, ("request_json", "files")))
|
|
|
146 and "encoded_id" in job_attr
|
|
|
147 ):
|
|
|
148 params.update(json.loads(params.pop("request_json")))
|
|
|
149 for i, target in enumerate(params.pop("targets")):
|
|
|
150 files = target["elements"]
|
|
|
151 params["files"][i]["hid"] = join_noempty(
|
|
|
152 str(file["object_id"])
|
|
|
153 for file in files
|
|
|
154 )
|
|
|
155 params["files"][i]["name"] = join_noempty(
|
|
|
156 str(file["name"])
|
|
|
157 for file in files
|
|
|
158 )
|
|
|
159 params["files"][i]["extension"] = join_noempty(
|
|
|
160 str(file["ext"])
|
|
|
161 for file in files
|
|
|
162 )
|
|
|
163
|
|
|
164 def iter_parameter_keys(params):
|
|
|
165 for key in params:
|
|
|
166 param = params[key]
|
|
|
167 if param_is_file(param):
|
|
|
168 yield key
|
|
|
169 elif isinstance(param, dict):
|
|
|
170 for subkey in iter_parameter_keys(param):
|
|
|
171 if subkey not in ("__current_case__", ):
|
|
|
172 yield f"{key}.{subkey}"
|
|
|
173 else:
|
|
|
174 yield key
|
|
|
175
|
|
|
176 def param_is_file(param):
|
|
|
177 return is_file_v1(param) or is_file_v2(param)
|
|
|
178
|
|
|
179 def is_file_v1(param):
|
|
|
180 return (
|
|
|
181 isinstance(param, dict)
|
|
|
182 and all(map(
|
|
|
183 param.__contains__,
|
|
|
184 ("info", "peek", "name", "extension")
|
|
|
185 ))
|
|
|
186 )
|
|
|
187
|
|
|
188 def is_file_v2(param):
|
|
|
189 return (
|
|
|
190 isinstance(param, dict)
|
|
|
191 and "values" in param
|
|
|
192 and isinstance(param["values"], list)
|
|
|
193 and isinstance(param["values"][0], dict)
|
|
|
194 and all(map(param["values"][0].__contains__, ("id", "src")))
|
|
|
195 )
|
|
|
196
|
|
|
197 def get_table_header(params, keys):
|
|
|
198 return HEADER_LIST_TEMPLATE.format(
|
|
|
199 header_list=indent(indent('\n'.join(map(HEADER_TEMPLATE.format, [""]+keys))))
|
|
|
200 )
|
|
|
201
|
|
|
202 def get_table_lines(params, keys):
|
|
|
203 return ''.join(table_lines_iterator(params, keys))
|
|
|
204
|
|
|
205 def table_lines_iterator(params, param_names):
|
|
|
206 keys = ("value", "name", "extension", "hid",)
|
|
|
207 for param_name in param_names:
|
|
|
208 classes = ""
|
|
|
209 table_lines = []
|
|
|
210 subparam = params
|
|
|
211 while '.' in param_name:
|
|
|
212 subkey, param_name = param_name.split('.', 1)
|
|
|
213 subparam = subparam[subkey]
|
|
|
214 for key in keys:
|
|
|
215 param = extract_param_info(key, subparam[param_name])
|
|
|
216 table_lines.append(param)
|
|
|
217 yield TABLE_LINE_LIST_TEMPLATE.format(
|
|
|
218 classes=classes,
|
|
|
219 table_lines=(
|
|
|
220 indent(COLUMN_TEMPLATE.format(param_name) + '\n')
|
|
|
221 + indent('\n'.join(map(
|
|
|
222 TABLE_LINE_TEMPLATE.format,
|
|
|
223 table_lines
|
|
|
224 )))
|
|
|
225 )
|
|
|
226 )
|
|
|
227
|
|
|
228 def extract_param_info(key, param):
|
|
|
229 if key == "value":
|
|
|
230 return extract_param_value(param)
|
|
|
231 if isinstance(param, dict) and key in param:
|
|
|
232 return str(param[key])
|
|
|
233 if isinstance(param, list):
|
|
|
234 return join_noempty(extract_param_info(key, p) for p in param)
|
|
|
235 return ""
|
|
|
236
|
|
|
237 def extract_param_value(param):
|
|
|
238 if isinstance(param, dict):
|
|
|
239 if "__current_case__" in param:
|
|
|
240 return join_dict_key_values(param, ignore=("__current_case__", ))
|
|
|
241 for acceptable_value in ("file_data", "file_name"):
|
|
|
242 if acceptable_value in param:
|
|
|
243 return f"{acceptable_value}: {param[acceptable_value]}"
|
|
|
244 if "values" in param:
|
|
|
245 ids = []
|
|
|
246 for file_id in param["values"]:
|
|
|
247 file_id = file_id["id"]
|
|
|
248 if file_id in HISTORY_CACHE:
|
|
|
249 file_info = HISTORY_CACHE[file_id]
|
|
|
250 param["name"] = file_info["name"]
|
|
|
251 param["hid"] = file_info["hid"]
|
|
|
252 param["extension"] = file_info["extension"]
|
|
|
253 ids.append(file_id)
|
|
|
254 return join_noempty(ids)
|
|
|
255 if isinstance(param, (str, int, float)):
|
|
|
256 return str(param)
|
|
|
257 if isinstance(param, (list, tuple)):
|
|
|
258 return join_noempty(map(extract_param_value, param))
|
|
|
259 return str(param)
|
|
|
260
|
|
|
261 def join_dict_key_values(dico, ignore=()):
|
|
|
262 return join_noempty(
|
|
|
263 f"{name}: {dico[name]}"
|
|
|
264 for name in dico
|
|
|
265 if name not in ignore
|
|
|
266 )
|
|
|
267
|
|
|
268 if __name__ == "__main__":
|
|
|
269 import optparse
|
|
|
270 parser = optparse.OptionParser()
|
|
|
271 parser.add_option(
|
|
|
272 "-j", "--jobs-attrs",
|
|
|
273 dest="jobs_attrs",
|
|
|
274 help="write report of FILE",
|
|
|
275 metavar="FILE",
|
|
|
276 default="jobs_attrs.txt"
|
|
|
277 )
|
|
|
278 parser.add_option(
|
|
|
279 "-d", "--dataset-attrs",
|
|
|
280 dest="dataset_attrs",
|
|
|
281 help="extract additional info from this file",
|
|
|
282 metavar="FILE",
|
|
|
283 default=None,
|
|
|
284 )
|
|
|
285 parser.add_option(
|
|
|
286 "-o", "--output",
|
|
|
287 dest="output",
|
|
|
288 help="write report to FILE",
|
|
|
289 metavar="FILE",
|
|
|
290 default="out.html"
|
|
|
291 )
|
|
|
292 parser.add_option(
|
|
|
293 "-v", "--version",
|
|
|
294 action="store_true",
|
|
|
295 help="Show this script's version and exits",
|
|
|
296 )
|
|
|
297
|
|
|
298 (options, args) = parser.parse_args()
|
|
|
299
|
|
|
300 if options.version:
|
|
|
301
|
|
|
302 import re
|
|
|
303
|
|
|
304 with open(os.path.join(sys.path[0], "README.md")) as readme:
|
|
|
305 for line in readme.readlines():
|
|
|
306 if "**@VERSION**" in line:
|
|
|
307 print(re.search(r"\d+\.\d+\.\d+", line)[0])
|
|
|
308 sys.exit(0)
|
|
|
309
|
|
|
310 with open(options.jobs_attrs) as j:
|
|
|
311 jobs_attrs = json.load(j)
|
|
|
312
|
|
|
313 if options.dataset_attrs is not None:
|
|
|
314 with open(options.dataset_attrs) as ds:
|
|
|
315 dataset_attrs = json.load(ds)
|
|
|
316 else:
|
|
|
317 dataset_attrs = {}
|
|
|
318
|
|
|
319 jobs_attrs = [{
|
|
|
320 key: jobs_attr.get(key)
|
|
|
321 for key in (
|
|
|
322 "galaxy_version",
|
|
|
323 "tool_id",
|
|
|
324 "tool_version",
|
|
|
325 "encoded_id",
|
|
|
326 "params",
|
|
|
327 "output_datasets",
|
|
|
328 "exit_code",
|
|
|
329 "output_dataset_mapping",
|
|
|
330 )
|
|
|
331 } for jobs_attr in jobs_attrs]
|
|
|
332 if jobs_attrs and jobs_attrs[0].get("output_datasets"):
|
|
|
333 jobs_attrs = sorted(
|
|
|
334 jobs_attrs,
|
|
|
335 key=lambda x:x["output_datasets"][0]
|
|
|
336 )
|
|
|
337
|
|
|
338 with open(options.output, "w") as o:
|
|
|
339 o.write(convert_to_html(jobs_attrs, dataset_attrs=dataset_attrs))
|
