Mercurial > repos > shellac > sam_consensus_v3

comparison env/lib/python3.9/site-packages/planemo/shed_lint.py @ 0:4f3585e2f14b draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author shellac
date Mon, 22 Mar 2021 18:12:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4f3585e2f14b
1 """Logic related to linting shed repositories."""
2 from __future__ import absolute_import
3
4 import os
5 import xml.etree.ElementTree as ET
6
7 import yaml
8 from galaxy.tool_util.lint import lint_tool_source_with
9 from galaxy.tool_util.linters.help import rst_invalid
10 from galaxy.util import unicodify
11
12 from planemo.io import info
13 from planemo.lint import (
14 handle_lint_complete,
15 lint_urls,
16 lint_xsd,
17 setup_lint,
18 )
19 from planemo.shed import (
20 CURRENT_CATEGORIES,
21 REPO_TYPE_SUITE,
22 REPO_TYPE_TOOL_DEP,
23 REPO_TYPE_UNRESTRICTED,
24 validate_repo_name,
25 validate_repo_owner,
26 )
27 from planemo.shed2tap import base
28 from planemo.tool_lint import (
29 build_tool_lint_args,
30 handle_tool_load_error,
31 )
32 from planemo.tools import yield_tool_sources
33 from planemo.xml import XSDS_PATH
34
35
36 TOOL_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "tool_dependencies.xsd")
37 REPO_DEPENDENCIES_XSD = os.path.join(XSDS_PATH, "repository_dependencies.xsd")
38
39
40 VALID_REPOSITORY_TYPES = [
41 REPO_TYPE_UNRESTRICTED,
42 REPO_TYPE_TOOL_DEP,
43 REPO_TYPE_SUITE,
44 ]
45
46 SHED_METADATA = [
47 "description",
48 "long_description",
49 "remote_repository_url",
50 "homepage_url",
51 "categories",
52 ]
53
54
55 def lint_repository(ctx, realized_repository, **kwds):
56 """Lint a realized shed repository.
57
58 See :mod:`planemo.shed` for details on constructing a realized
59 repository data structure.
60 """
61 failed = False
62 path = realized_repository.real_path
63 info("Linting repository %s" % path)
64 lint_args = build_tool_lint_args(ctx, **kwds)
65 lint_args, lint_ctx = setup_lint(ctx, lint_args=lint_args, **kwds)
66 lint_ctx.lint(
67 "lint_expansion",
68 lint_expansion,
69 realized_repository,
70 )
71 lint_ctx.lint(
72 "lint_expected_files",
73 lint_expected_files,
74 realized_repository,
75 )
76 lint_ctx.lint(
77 "lint_tool_dependencies_xsd",
78 lint_tool_dependencies_xsd,
79 realized_repository,
80 )
81 lint_ctx.lint(
82 "lint_tool_dependencies_sha256sum",
83 lint_tool_dependencies_sha256sum,
84 realized_repository,
85 )
86 lint_ctx.lint(
87 "lint_tool_dependencies_actions",
88 lint_tool_dependencies_actions,
89 realized_repository,
90 )
91 lint_ctx.lint(
92 "lint_repository_dependencies",
93 lint_repository_dependencies,
94 realized_repository,
95 )
96 lint_ctx.lint(
97 "lint_shed_yaml",
98 lint_shed_yaml,
99 realized_repository,
100 )
101 lint_ctx.lint(
102 "lint_readme",
103 lint_readme,
104 realized_repository,
105 )
106 if kwds["urls"]:
107 lint_ctx.lint(
108 "lint_urls",
109 lint_tool_dependencies_urls,
110 realized_repository,
111 )
112 if kwds["tools"]:
113 tools_failed = lint_repository_tools(ctx, realized_repository, lint_ctx, lint_args)
114 failed = failed or tools_failed
115 if kwds["ensure_metadata"]:
116 lint_ctx.lint(
117 "lint_shed_metadata",
118 lint_shed_metadata,
119 realized_repository,
120 )
121 return handle_lint_complete(lint_ctx, lint_args, failed=failed)
122
123
124 def lint_repository_tools(ctx, realized_repository, lint_ctx, lint_args):
125 path = realized_repository.path
126 for (tool_path, tool_source) in yield_tool_sources(ctx, path,
127 recursive=True):
128 original_path = tool_path.replace(path, realized_repository.real_path)
129 info("+Linting tool %s" % original_path)
130 if handle_tool_load_error(tool_path, tool_source):
131 return True
132 lint_tool_source_with(
133 lint_ctx,
134 tool_source,
135 extra_modules=lint_args["extra_modules"]
136 )
137
138
139 def lint_expansion(realized_repository, lint_ctx):
140 missing = realized_repository.missing
141 if missing:
142 msg = "Failed to expand inclusions %s" % missing
143 lint_ctx.warn(msg)
144 else:
145 lint_ctx.info("Included files all found.")
146
147
148 def lint_shed_metadata(realized_repository, lint_ctx):
149 found_all = True
150 for key in SHED_METADATA:
151 if key not in realized_repository.config:
152 found_all = False
153 lint_ctx.warn(
154 "Missing shed metadata field [%s] for repository" % key
155 )
156 if found_all:
157 lint_ctx.info(
158 "Found all shed metadata fields required for automated repository "
159 "creation and/or updates."
160 )
161
162
163 def lint_readme(realized_repository, lint_ctx):
164 path = realized_repository.real_path
165 readme_rst = os.path.join(path, "README.rst")
166 readme = os.path.join(path, "README")
167 readme_txt = os.path.join(path, "README.txt")
168
169 readme_found = False
170 for readme in [readme_rst, readme, readme_txt]:
171 if os.path.exists(readme):
172 readme_found = readme
173
174 readme_md = os.path.join(path, "README.md")
175 if not readme_found and os.path.exists(readme_md):
176 lint_ctx.warn("Tool Shed doesn't render markdown, "
177 "README.md is invalid readme.")
178 return
179
180 if not readme_found:
181 # TODO: filter on TYPE and make this a warning if
182 # unrestricted repository - need to update iuc standards
183 # first though.
184 lint_ctx.info("No README found skipping.")
185 return
186
187 if readme_found.endswith(".rst"):
188 with open(readme_found, "r") as fh:
189 readme_text = fh.read()
190 invalid_rst = rst_invalid(readme_text)
191 if invalid_rst:
192 template = "Invalid restructured text found in README [%s]."
193 msg = template % invalid_rst
194 lint_ctx.warn(msg)
195 return
196 lint_ctx.info("README found containing valid reStructuredText.")
197 else:
198 lint_ctx.info("README found containing plain text.")
199
200
201 def lint_tool_dependencies_urls(realized_repository, lint_ctx):
202 path = realized_repository.real_path
203 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
204 if not os.path.exists(tool_dependencies):
205 lint_ctx.info("No tool_dependencies.xml, skipping.")
206 return
207
208 root = ET.parse(tool_dependencies).getroot()
209 lint_urls(root, lint_ctx)
210
211
212 def lint_tool_dependencies_sha256sum(realized_repository, lint_ctx):
213 tool_dependencies = os.path.join(realized_repository.real_path, "tool_dependencies.xml")
214 if not os.path.exists(tool_dependencies):
215 lint_ctx.info("No tool_dependencies.xml, skipping.")
216 return
217
218 root = ET.parse(tool_dependencies).getroot()
219
220 count = 0
221 for action in root.findall(".//action"):
222 assert action.tag == "action"
223 if action.attrib.get('type', '') not in ['download_by_url', 'download_file']:
224 continue
225 url = action.text.strip()
226 checksum = action.attrib.get('sha256sum', '')
227 if not checksum:
228 lint_ctx.warn("Missing checksum for %s" % url)
229 elif len(checksum) != 64 or not set("0123456789abcdef").issuperset(checksum.lower()):
230 lint_ctx.error("Invalid checksum %r for %s" % (checksum, url))
231 else:
232 # TODO - See planned --verify option to check it matches
233 # lint_ctx.info("SHA256 checkum listed for %s" % url)
234 count += 1
235 if count:
236 lint_ctx.info("Found %i download action(s) with SHA256 checksums" % count)
237
238
239 def lint_tool_dependencies_xsd(realized_repository, lint_ctx):
240 path = realized_repository.real_path
241 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
242 if not os.path.exists(tool_dependencies):
243 lint_ctx.info("No tool_dependencies.xml, skipping.")
244 return
245 lint_xsd(lint_ctx, TOOL_DEPENDENCIES_XSD, tool_dependencies)
246
247
248 def lint_tool_dependencies_actions(realized_repository, lint_ctx):
249 path = realized_repository.real_path
250 tool_dependencies = os.path.join(path, "tool_dependencies.xml")
251 if not os.path.exists(tool_dependencies):
252 lint_ctx.info("No tool_dependencies.xml, skipping.")
253 return
254 try:
255 base.Dependencies(tool_dependencies)
256 lint_ctx.info("Parsed tool dependencies.")
257 except Exception as e:
258 import sys
259 import traceback
260 exc_type, exc_value, exc_traceback = sys.exc_info()
261 traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
262 traceback.print_exc()
263 template = "Problem parsing tool_dependenies.xml [%s]"
264 msg = template % unicodify(e)
265 lint_ctx.warn(msg)
266 return
267
268
269 def lint_expected_files(realized_repository, lint_ctx):
270 if realized_repository.is_package:
271 if not os.path.exists(realized_repository.tool_dependencies_path):
272 lint_ctx.warn("Package repository does not contain a "
273 "tool_dependencies.xml file.")
274
275 if realized_repository.is_suite:
276 if not os.path.exists(realized_repository.repo_dependencies_path):
277 lint_ctx.warn("Suite repository does not contain a "
278 "repository_dependencies.xml file.")
279
280
281 def lint_repository_dependencies(realized_repository, lint_ctx):
282 path = realized_repository.real_path
283 repo_dependencies = os.path.join(path, "repository_dependencies.xml")
284 if not os.path.exists(repo_dependencies):
285 lint_ctx.info("No repository_dependencies.xml, skipping.")
286 return
287 lint_xsd(lint_ctx, REPO_DEPENDENCIES_XSD, repo_dependencies)
288
289
290 def lint_shed_yaml(realized_repository, lint_ctx):
291 path = realized_repository.real_path
292 shed_yaml = os.path.join(path, ".shed.yml")
293 if not os.path.exists(shed_yaml):
294 lint_ctx.info("No .shed.yml file found, skipping.")
295 return
296 try:
297 with open(shed_yaml, "r") as fh:
298 yaml.safe_load(fh)
299 except Exception as e:
300 lint_ctx.warn("Failed to parse .shed.yml file [%s]" % unicodify(e))
301 return
302 lint_ctx.info(".shed.yml found and appears to be valid YAML.")
303 _lint_shed_contents(lint_ctx, realized_repository)
304
305
306 def _lint_shed_contents(lint_ctx, realized_repository):
307 config = realized_repository.config
308
309 def _lint_if_present(key, func, *args):
310 value = config.get(key, None)
311 if value is not None:
312 msg = func(value, *args)
313 if msg:
314 lint_ctx.warn(msg)
315
316 _lint_if_present("owner", validate_repo_owner)
317 _lint_if_present("name", validate_repo_name)
318 _lint_if_present("type", _validate_repo_type, config["name"])
319 _lint_if_present("categories", _validate_categories, realized_repository)
320
321
322 def _validate_repo_type(repo_type, name):
323 if repo_type not in VALID_REPOSITORY_TYPES:
324 return "Invalid repository type specified [%s]" % repo_type
325
326 is_dep = repo_type == "tool_dependency_definition"
327 is_suite = repo_type == "repository_suite_definition"
328 if is_dep and not name.startswith("package_"):
329 return ("Tool dependency definition repositories should have names "
330 "starting with package_")
331 if is_suite and not name.startswith("suite_"):
332 return ("Repository suite definition repositories should have names "
333 "starting with suite_")
334 if name.startswith("package_") or name.startswith("suite_"):
335 if repo_type == "unrestricted":
336 return ("Repository name indicated specialized repository type "
337 "but repository is listed as unrestricted.")
338
339
340 def _validate_categories(categories, realized_repository):
341 msg = None
342 if len(categories) == 0:
343 msg = "Repository should specify one or more categories."
344 else:
345 for category in categories:
346 unknown_categories = []
347 if category not in CURRENT_CATEGORIES:
348 unknown_categories.append(category)
349 if unknown_categories:
350 msg = "Categories [%s] unknown." % unknown_categories
351 if realized_repository.is_package:
352 if "Tool Dependency Packages" not in categories:
353 msg = ("Packages should be placed and should only be placed "
354 "in the category 'Tool Dependency Packages'.")
355
356 return msg
357
358
359 __all__ = (
360 "lint_repository",
361 )