Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/planemo/conda_verify/recipe.py @ 0:26e78fe6e8c4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit c699937486c35866861690329de38ec1a5d9f783"
author shellac
date Sat, 02 May 2020 07:14:21 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:26e78fe6e8c4
1 from __future__ import (
2 absolute_import,
3 division,
4 print_function,
5 )
6
7 import os
8 import re
9 from os.path import (
10 basename,
11 getsize,
12 isfile,
13 join,
14 )
15
16 import yaml
17
18 from planemo.conda_verify.const import (
19 FIELDS,
20 LICENSE_FAMILIES,
21 )
22 from planemo.conda_verify.utils import (
23 all_ascii,
24 get_bad_seq,
25 memoized,
26 )
27
28 PEDANTIC = True
29 sel_pat = re.compile(r'(.+?)\s*\[(.+)\]$')
30 name_pat = re.compile(r'[a-z0-9_][a-z0-9_\-\.]*$')
31 version_pat = re.compile(r'[\w\.]+$')
32 url_pat = re.compile(r'(ftp|http(s)?)://')
33
34
35 class RecipeError(Exception):
36 pass
37
38
39 def ns_cfg(cfg):
40 plat = cfg['plat']
41 py = cfg['PY']
42 np = cfg['NPY']
43 for x in py, np:
44 assert isinstance(x, int), x
45 return dict(
46 nomkl=False,
47 debug=False,
48 linux=plat.startswith('linux-'),
49 linux32=bool(plat == 'linux-32'),
50 linux64=bool(plat == 'linux-64'),
51 armv7l=False,
52 arm=False,
53 ppc64le=False,
54 osx=plat.startswith('osx-'),
55 unix=plat.startswith(('linux-', 'osx-')),
56 win=plat.startswith('win-'),
57 win32=bool(plat == 'win-32'),
58 win64=bool(plat == 'win-64'),
59 x86=plat.endswith(('-32', '-64')),
60 x86_64=plat.endswith('-64'),
61 py=py,
62 py3k=bool(30 <= py < 40),
63 py2k=bool(20 <= py < 30),
64 py26=bool(py == 26),
65 py27=bool(py == 27),
66 py33=bool(py == 33),
67 py34=bool(py == 34),
68 py35=bool(py == 35),
69 np=np,
70 )
71
72
73 def select_lines(data, namespace):
74 lines = []
75 for line in data.splitlines():
76 line = line.rstrip()
77 m = sel_pat.match(line)
78 if m:
79 if PEDANTIC:
80 x = m.group(1).strip()
81 # error on comment, unless the whole line is a comment
82 if '#' in x and not x.startswith('#'):
83 raise RecipeError("found commented selector: %s" % line)
84 cond = m.group(2)
85 if eval(cond, namespace, {}):
86 lines.append(m.group(1))
87 continue
88 lines.append(line)
89 return '\n'.join(lines) + '\n'
90
91
92 @memoized
93 def yamlize(data):
94 res = yaml.safe_load(data)
95 # ensure the result is a dict
96 if res is None:
97 res = {}
98 return res
99
100
101 def parse(data, cfg):
102 if cfg is not None:
103 data = select_lines(data, ns_cfg(cfg))
104 # ensure we create new object, because yamlize is memoized
105 return dict(yamlize(data))
106
107
108 def get_field(meta, field, default=None):
109 section, key = field.split('/')
110 submeta = meta.get(section)
111 if submeta is None:
112 submeta = {}
113 res = submeta.get(key)
114 if res is None:
115 res = default
116 return res
117
118
119 def check_name(name):
120 if name:
121 name = str(name)
122 else:
123 raise RecipeError("package name missing")
124 if not name_pat.match(name) or name.endswith(('.', '-', '_')):
125 raise RecipeError("invalid package name '%s'" % name)
126 seq = get_bad_seq(name)
127 if seq:
128 raise RecipeError("'%s' is not allowed in "
129 "package name: '%s'" % (seq, name))
130
131
132 def check_version(ver):
133 if ver:
134 ver = str(ver)
135 else:
136 raise RecipeError("package version missing")
137 if not version_pat.match(ver):
138 raise RecipeError("invalid version '%s'" % ver)
139 if ver.startswith(('_', '.')) or ver.endswith(('_', '.')):
140 raise RecipeError("version cannot start or end with '_' or '.': %s" %
141 ver)
142 seq = get_bad_seq(ver)
143 if seq:
144 raise RecipeError("'%s' not allowed in version '%s'" % (seq, ver))
145
146
147 def check_build_number(bn):
148 if not (isinstance(bn, int) and bn >= 0):
149 raise RecipeError("build/number '%s' (not a positive interger)" % bn)
150
151
152 def check_requirements(meta):
153 for req in get_field(meta, 'requirements/run', []):
154 name = req.split()[0]
155 if not name_pat.match(name):
156 raise RecipeError("invalid run requirement name '%s'" % name)
157
158
159 def check_license_family(meta):
160 if not PEDANTIC:
161 return
162 lf = get_field(meta, 'about/license_family',
163 get_field(meta, 'about/license'))
164 if lf not in LICENSE_FAMILIES:
165 print("""\
166 Error: license_family is invalid: %s
167 Note that about/license_family falls back to about/license.
168 Allowed license families are:""" % lf)
169 for x in LICENSE_FAMILIES:
170 print(" - %s" % x)
171 raise RecipeError("wrong license family")
172
173
174 def check_url(url):
175 if not url_pat.match(url):
176 raise RecipeError("not a valid URL: %s" % url)
177
178
179 def check_about(meta):
180 summary = get_field(meta, 'about/summary')
181 if summary and len(summary) > 80:
182 msg = "summary exceeds 80 characters"
183 if PEDANTIC:
184 raise RecipeError(msg)
185 else:
186 print("Warning: %s" % msg)
187
188 for field in ('about/home', 'about/dev_url', 'about/doc_url',
189 'about/license_url'):
190 url = get_field(meta, field)
191 if url:
192 check_url(url)
193
194 check_license_family(meta)
195
196
197 hash_pat = {'md5': re.compile(r'[a-f0-9]{32}$'),
198 'sha1': re.compile(r'[a-f0-9]{40}$'),
199 'sha256': re.compile(r'[a-f0-9]{64}$')}
200
201
202 def check_source(meta):
203 src = meta.get('source')
204 if not src:
205 return
206 fn = src.get('fn')
207 if fn:
208 for ht in 'md5', 'sha1', 'sha256':
209 hexgigest = src.get(ht)
210 if hexgigest and not hash_pat[ht].match(hexgigest):
211 raise RecipeError("invalid hash: %s" % hexgigest)
212 url = src.get('url')
213 if url:
214 check_url(url)
215
216 git_url = src.get('git_url')
217 if git_url and (src.get('git_tag') and src.get('git_branch')):
218 raise RecipeError("cannot specify both git_branch and git_tag")
219
220
221 def validate_meta(meta):
222 for section in meta:
223 if PEDANTIC and section not in FIELDS:
224 raise RecipeError("Unknown section: %s" % section)
225 submeta = meta.get(section)
226 if submeta is None:
227 submeta = {}
228 for key in submeta:
229 if PEDANTIC and key not in FIELDS[section]:
230 raise RecipeError("in section %r: unknown key %r" %
231 (section, key))
232
233 check_name(get_field(meta, 'package/name'))
234 check_version(get_field(meta, 'package/version'))
235 check_build_number(get_field(meta, 'build/number', 0))
236 check_requirements(meta)
237 check_about(meta)
238 check_source(meta)
239
240
241 def validate_files(recipe_dir, meta):
242 for field in 'test/files', 'source/patches':
243 flst = get_field(meta, field)
244 if not flst:
245 continue
246 for fn in flst:
247 if PEDANTIC and fn.startswith('..'):
248 raise RecipeError("path outsite recipe: %s" % fn)
249 path = join(recipe_dir, fn)
250 if isfile(path):
251 continue
252 raise RecipeError("no such file '%s'" % path)
253
254
255 def iter_cfgs():
256 for py in 27, 34, 35:
257 for plat in 'linux-64', 'linux-32', 'osx-64', 'win-32', 'win-64':
258 yield dict(plat=plat, PY=py, NPY=111)
259
260
261 def dir_size(dir_path):
262 return sum(sum(getsize(join(root, fn)) for fn in files)
263 for root, unused_dirs, files in os.walk(dir_path))
264
265
266 def check_dir_content(recipe_dir):
267 disallowed_extensions = (
268 '.tar', '.tar.gz', '.tar.bz2', '.tar.xz',
269 '.so', '.dylib', '.la', '.a', '.dll', '.pyd',
270 )
271 for root, unused_dirs, files in os.walk(recipe_dir):
272 for fn in files:
273 fn_lower = fn.lower()
274 if fn_lower.endswith(disallowed_extensions):
275 if PEDANTIC:
276 raise RecipeError("found: %s" % fn)
277 else:
278 print("Warning: found: %s" % fn)
279 path = join(root, fn)
280 # only allow small archives for testing
281 if (PEDANTIC and fn_lower.endswith(('.bz2', '.gz')) and getsize(path) > 512):
282 raise RecipeError("found: %s (too large)" % fn)
283
284 if basename(recipe_dir) == 'icu':
285 return
286
287 # check total size od recipe directory (recursively)
288 kb_size = dir_size(recipe_dir) / 1024
289 kb_limit = 512
290 if PEDANTIC and kb_size > kb_limit:
291 raise RecipeError("recipe too large: %d KB (limit %d KB)" %
292 (kb_size, kb_limit))
293
294 if PEDANTIC:
295 try:
296 with open(join(recipe_dir, 'build.sh'), 'rb') as fi:
297 data = fi.read()
298 if data and not data.decode('utf-8').startswith(('#!/bin/bash\n',
299 '#!/bin/sh\n')):
300 raise RecipeError("not a bash script: build.sh")
301 except IOError:
302 pass
303
304
305 def render_jinja2(recipe_dir):
306 import jinja2
307
308 loaders = [jinja2.FileSystemLoader(recipe_dir)]
309 env = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders))
310 template = env.get_or_select_template('meta.yaml')
311 return template.render(environment=env)
312
313
314 def validate_recipe(recipe_dir, pedantic=True):
315 global PEDANTIC
316 PEDANTIC = bool(pedantic)
317
318 meta_path = join(recipe_dir, 'meta.yaml')
319 with open(meta_path, 'rb') as fi:
320 data = fi.read()
321 if PEDANTIC and not all_ascii(data):
322 raise RecipeError("non-ASCII in: %s" % meta_path)
323 if b'{{' in data:
324 if PEDANTIC:
325 raise RecipeError("found {{ in %s (Jinja templating not allowed)" %
326 meta_path)
327 else:
328 data = render_jinja2(recipe_dir)
329 else:
330 data = data.decode('utf-8')
331
332 check_dir_content(recipe_dir)
333
334 for cfg in iter_cfgs():
335 meta = parse(data, cfg)
336 validate_meta(meta)
337 validate_files(recipe_dir, meta)