Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/planemo/shed_lint.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 """Logic related to linting shed repositories."""
2 from __future__ import absolute_import
3
4 import os
5 import xml.etree.ElementTree as ET
6
7 import yaml
8 from galaxy.tool_util.lint import lint_tool_source_with
9 from galaxy.tool_util.linters.help import rst_invalid
10 from galaxy.util import unicodify
11
12 from planemo.io import info
13 from planemo.lint import (
14 handle_lint_complete,
15 lint_urls,
16 lint_xsd,
17 setup_lint,
18 )
19 from planemo.shed import (
20 CURRENT_CATEGORIES,
21 REPO_TYPE_SUITE,
22 REPO_TYPE_TOOL_DEP,
23 REPO_TYPE_UNRESTRICTED,
24 validate_repo_name,
25 validate_repo_owner,
26 )
27 from planemo.shed2tap import base
28 from planemo.tool_lint import (
29 handle_tool_load_error,
30 )
31 from planemo.tools import yield_tool_sources
32 from planemo.xml import XSDS_PATH
33
34
35 TOOL_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "tool_dependencies.xsd")
36 REPO_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "repository_dependencies.xsd")
37
38
39 VALID_REPOSITORY_TYPES = [
40 REPO_TYPE_UNRESTRICTED,
41 REPO_TYPE_TOOL_DEP,
42 REPO_TYPE_SUITE,
43 ]
44
45 SHED_METADATA = [
46 "description",
47 "long_description",
48 "remote_repository_url",
49 "homepage_url",
50 "categories",
51 ]
52
53
54 def lint_repository(ctx, realized_repository, **kwds):
55 """Lint a realized shed repository.
56
57 See :mod:`planemo.shed` for details on constructing a realized
58 repository data structure.
59 """
60 failed = False
61 path = realized_repository.real_path
62 info("Linting repository %s" % path)
63 lint_args, lint_ctx = setup_lint(ctx, **kwds)
64 lint_ctx.lint(
65 "lint_expansion",
66 lint_expansion,
67 realized_repository,
68 )
69 lint_ctx.lint(
70 "lint_expected_files",
71 lint_expected_files,
72 realized_repository,
73 )
74 lint_ctx.lint(
75 "lint_tool_dependencies_xsd",
76 lint_tool_dependencies_xsd,
77 realized_repository,
78 )
79 lint_ctx.lint(
80 "lint_tool_dependencies_sha256sum",
81 lint_tool_dependencies_sha256sum,
82 realized_repository,
83 )
84 lint_ctx.lint(
85 "lint_tool_dependencies_actions",
86 lint_tool_dependencies_actions,
87 realized_repository,
88 )
89 lint_ctx.lint(
90 "lint_repository_dependencies",
91 lint_repository_dependencies,
92 realized_repository,
93 )
94 lint_ctx.lint(
95 "lint_shed_yaml",
96 lint_shed_yaml,
97 realized_repository,
98 )
99 lint_ctx.lint(
100 "lint_readme",
101 lint_readme,
102 realized_repository,
103 )
104 if kwds["urls"]:
105 lint_ctx.lint(
106 "lint_urls",
107 lint_tool_dependencies_urls,
108 realized_repository,
109 )
110 if kwds["tools"]:
111 tools_failed = lint_repository_tools(ctx, realized_repository, lint_ctx, lint_args)
112 failed = failed or tools_failed
113 if kwds["ensure_metadata"]:
114 lint_ctx.lint(
115 "lint_shed_metadata",
116 lint_shed_metadata,
117 realized_repository,
118 )
119 return handle_lint_complete(lint_ctx, lint_args, failed=failed)
120
121
122 def lint_repository_tools(ctx, realized_repository, lint_ctx, lint_args):
123 path = realized_repository.path
124 for (tool_path, tool_source) in yield_tool_sources(ctx, path,
125 recursive=True):
126 original_path = tool_path.replace(path, realized_repository.real_path)
127 info("+Linting tool %s" % original_path)
128 if handle_tool_load_error(tool_path, tool_source):
129 return True
130 lint_tool_source_with(
131 lint_ctx,
132 tool_source,
133 extra_modules=lint_args["extra_modules"]
134 )
135
136
137 def lint_expansion(realized_repository, lint_ctx):
138 missing = realized_repository.missing
139 if missing:
140 msg = "Failed to expand inclusions %s" % missing
141 lint_ctx.warn(msg)
142 else:
143 lint_ctx.info("Included files all found.")
144
145
146 def lint_shed_metadata(realized_repository, lint_ctx):
147 found_all = True
148 for key in SHED_METADATA:
149 if key not in realized_repository.config:
150 found_all = False
151 lint_ctx.warn(
152 "Missing shed metadata field [%s] for repository" % key
153 )
154 if found_all:
155 lint_ctx.info(
156 "Found all shed metadata fields required for automated repository "
157 "creation and/or updates."
158 )
159
160
161 def lint_readme(realized_repository, lint_ctx):
162 path = realized_repository.real_path
163 readme_rst = os.path.join(path, "README.rst")
164 readme = os.path.join(path, "README")
165 readme_txt = os.path.join(path, "README.txt")
166
167 readme_found = False
168 for readme in [readme_rst, readme, readme_txt]:
169 if os.path.exists(readme):
170 readme_found = readme
171
172 readme_md = os.path.join(path, "README.md")
173 if not readme_found and os.path.exists(readme_md):
174 lint_ctx.warn("Tool Shed doesn't render markdown, "
175 "README.md is invalid readme.")
176 return
177
178 if not readme_found:
179 # TODO: filter on TYPE and make this a warning if
180 # unrestricted repository - need to update iuc standards
181 # first though.
182 lint_ctx.info("No README found skipping.")
183 return
184
185 if readme_found.endswith(".rst"):
186 with open(readme_found, "r") as fh:
187 readme_text = fh.read()
188 invalid_rst = rst_invalid(readme_text)
189 if invalid_rst:
190 template = "Invalid restructured text found in README [%s]."
191 msg = template % invalid_rst
192 lint_ctx.warn(msg)
193 return
194 lint_ctx.info("README found containing valid reStructuredText.")
195 else:
196 lint_ctx.info("README found containing plain text.")
197
198
199 def lint_tool_dependencies_urls(realized_repository, lint_ctx):
200 path = realized_repository.real_path
201 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
202 if not os.path.exists(tool_dependencies):
203 lint_ctx.info("No tool_dependencies.xml, skipping.")
204 return
205
206 root = ET.parse(tool_dependencies).getroot()
207 lint_urls(root, lint_ctx)
208
209
210 def lint_tool_dependencies_sha256sum(realized_repository, lint_ctx):
211 tool_dependencies = os.path.join(realized_repository.real_path, "tool_dependencies.xml")
212 if not os.path.exists(tool_dependencies):
213 lint_ctx.info("No tool_dependencies.xml, skipping.")
214 return
215
216 root = ET.parse(tool_dependencies).getroot()
217
218 count = 0
219 for action in root.findall(".//action"):
220 assert action.tag == "action"
221 if action.attrib.get('type', '') not in ['download_by_url', 'download_file']:
222 continue
223 url = action.text.strip()
224 checksum = action.attrib.get('sha256sum', '')
225 if not checksum:
226 lint_ctx.warn("Missing checksum for %s" % url)
227 elif len(checksum) != 64 or not set("0123456789abcdef").issuperset(checksum.lower()):
228 lint_ctx.error("Invalid checksum %r for %s" % (checksum, url))
229 else:
230 # TODO - See planned --verify option to check it matches
231 # lint_ctx.info("SHA256 checkum listed for %s" % url)
232 count += 1
233 if count:
234 lint_ctx.info("Found %i download action(s) with SHA256 checksums" % count)
235
236
237 def lint_tool_dependencies_xsd(realized_repository, lint_ctx):
238 path = realized_repository.real_path
239 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
240 if not os.path.exists(tool_dependencies):
241 lint_ctx.info("No tool_dependencies.xml, skipping.")
242 return
243 lint_xsd(lint_ctx, TOOL_DEPENDENCIES_XSD, tool_dependencies)
244
245
246 def lint_tool_dependencies_actions(realized_repository, lint_ctx):
247 path = realized_repository.real_path
248 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
249 if not os.path.exists(tool_dependencies):
250 lint_ctx.info("No tool_dependencies.xml, skipping.")
251 return
252 try:
253 base.Dependencies(tool_dependencies)
254 lint_ctx.info("Parsed tool dependencies.")
255 except Exception as e:
256 import sys
257 import traceback
258 exc_type, exc_value, exc_traceback = sys.exc_info()
259 traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
260 traceback.print_exc()
261 template = "Problem parsing tool_dependenies.xml [%s]"
262 msg = template % unicodify(e)
263 lint_ctx.warn(msg)
264 return
265
266
267 def lint_expected_files(realized_repository, lint_ctx):
268 if realized_repository.is_package:
269 if not os.path.exists(realized_repository.tool_dependencies_path):
270 lint_ctx.warn("Package repository does not contain a "
271 "tool_dependencies.xml file.")
272
273 if realized_repository.is_suite:
274 if not os.path.exists(realized_repository.repo_dependencies_path):
275 lint_ctx.warn("Suite repository does not contain a "
276 "repository_dependencies.xml file.")
277
278
279 def lint_repository_dependencies(realized_repository, lint_ctx):
280 path = realized_repository.real_path
281 repo_dependencies = os.path.join(path, "repository_dependencies.xml")
282 if not os.path.exists(repo_dependencies):
283 lint_ctx.info("No repository_dependencies.xml, skipping.")
284 return
285 lint_xsd(lint_ctx, REPO_DEPENDENCIES_XSD, repo_dependencies)
286
287
288 def lint_shed_yaml(realized_repository, lint_ctx):
289 path = realized_repository.real_path
290 shed_yaml = os.path.join(path, ".shed.yml")
291 if not os.path.exists(shed_yaml):
292 lint_ctx.info("No .shed.yml file found, skipping.")
293 return
294 try:
295 with open(shed_yaml, "r") as fh:
296 yaml.safe_load(fh)
297 except Exception as e:
298 lint_ctx.warn("Failed to parse .shed.yml file [%s]" % unicodify(e))
299 return
300 lint_ctx.info(".shed.yml found and appears to be valid YAML.")
301 _lint_shed_contents(lint_ctx, realized_repository)
302
303
304 def _lint_shed_contents(lint_ctx, realized_repository):
305 config = realized_repository.config
306
307 def _lint_if_present(key, func, *args):
308 value = config.get(key, None)
309 if value is not None:
310 msg = func(value, *args)
311 if msg:
312 lint_ctx.warn(msg)
313
314 _lint_if_present("owner", validate_repo_owner)
315 _lint_if_present("name", validate_repo_name)
316 _lint_if_present("type", _validate_repo_type, config["name"])
317 _lint_if_present("categories", _validate_categories, realized_repository)
318
319
320 def _validate_repo_type(repo_type, name):
321 if repo_type not in VALID_REPOSITORY_TYPES:
322 return "Invalid repository type specified [%s]" % repo_type
323
324 is_dep = repo_type == "tool_dependency_definition"
325 is_suite = repo_type == "repository_suite_definition"
326 if is_dep and not name.startswith("package_"):
327 return ("Tool dependency definition repositories should have names "
328 "starting with package_")
329 if is_suite and not name.startswith("suite_"):
330 return ("Repository suite definition repositories should have names "
331 "starting with suite_")
332 if name.startswith("package_") or name.startswith("suite_"):
333 if repo_type == "unrestricted":
334 return ("Repository name indicated specialized repository type "
335 "but repository is listed as unrestricted.")
336
337
338 def _validate_categories(categories, realized_repository):
339 msg = None
340 if len(categories) == 0:
341 msg = "Repository should specify one or more categories."
342 else:
343 for category in categories:
344 unknown_categories = []
345 if category not in CURRENT_CATEGORIES:
346 unknown_categories.append(category)
347 if unknown_categories:
348 msg = "Categories [%s] unknown." % unknown_categories
349 if realized_repository.is_package:
350 if "Tool Dependency Packages" not in categories:
351 msg = ("Packages should be placed and should only be placed "
352 "in the category 'Tool Dependency Packages'.")
353
354 return msg
355
356
357 __all__ = (
358 "lint_repository",
359 )