Mercurial > repos > chrisb > gap_datatypes

comparison datatypes/glycan.py @ 0:0e941a69a6fa draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author chrisb
date Wed, 23 Mar 2016 14:34:50 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:0e941a69a6fa
1 __license__ = "MIT"
2
3 import logging
4 from galaxy.datatypes import metadata
5 import mimetypes
6 import os
7 import shutil
8 import sys
9 import traceback
10 import tempfile
11 import zipfile
12 from cgi import escape
13 from inspect import isclass
14 import galaxy.util as util
15 from galaxy.datatypes import data
16 from galaxy.datatypes.metadata import \
17 MetadataElement # import directly to maintain ease of use in Datatype class definitions
18 from galaxy.util import inflector
19 from galaxy.util.bunch import Bunch
20 from galaxy.util.odict import odict
21 from galaxy.util.sanitize_html import sanitize_html
22
23 from galaxy.datatypes import dataproviders
24
25 from galaxy import eggs
26
27 eggs.require("Paste")
28 import paste
29
30
31 class kcf(data.Data):
32 file_ext = 'kcf'
33 line_class = 'line'
34
35 """Add metadata elements"""
36 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
37 visible=False, no_value=0)
38
39 def write_from_stream(self, dataset, stream):
40 """Writes data from a stream"""
41 # write it twice for now
42 fd, temp_name = tempfile.mkstemp()
43 while 1:
44 chunk = stream.read(1048576)
45 if not chunk:
46 break
47 os.write(fd, chunk)
48 os.close(fd)
49 # rewrite the file with unix newlines
50 fp = open(dataset.file_name, 'wt')
51 for line in file(temp_name, "U"):
52 line = line.strip() + '\n'
53 fp.write(line)
54 fp.close()
55
56 def set_raw_data(self, dataset, data):
57 """Saves the data on the disc"""
58 fd, temp_name = tempfile.mkstemp()
59 os.write(fd, data)
60 os.close(fd)
61 # rewrite the file with unix newlines
62 fp = open(dataset.file_name, 'wt')
63 for line in file(temp_name, "U"):
64 line = line.strip() + '\n'
65 fp.write(line)
66 fp.close()
67 os.remove(temp_name)
68
69 def get_mime(self):
70 """Returns the mime type of the datatype"""
71 return 'text/plain'
72
73 def set_meta(self, dataset, **kwd):
74 """
75 Set the number of lines of data in dataset.
76 """
77 dataset.metadata.data_lines = self.count_data_lines(dataset)
78
79 def estimate_file_lines(self, dataset):
80 """
81 Perform a rough estimate by extrapolating number of lines from a small read.
82 """
83 sample_size = 1048576
84 dataset_fh = open(dataset.file_name)
85 dataset_read = dataset_fh.read(sample_size)
86 dataset_fh.close()
87 sample_lines = dataset_read.count('\n')
88 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
89 return est_lines
90
91 def count_data_lines(self, dataset):
92 """
93 Count the number of lines of data in dataset,
94 skipping all blank lines and comments.
95 """
96 data_lines = 0
97 for line in file(dataset.file_name):
98 line = line.strip()
99 if line and not line.startswith('#'):
100 data_lines += 1
101 return data_lines
102
103 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
104 """
105 Set the peek. This method is used by various subclasses of Text.
106 """
107 if not dataset.dataset.purged:
108 # The file must exist on disk for the get_file_peek() method
109 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
110 skipchars=skipchars)
111 if line_count is None:
112 # See if line_count is stored in the metadata
113 if dataset.metadata.data_lines:
114 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
115 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
116 else:
117 # Number of lines is not known ( this should not happen ), and auto-detect is
118 # needed to set metadata
119 # This can happen when the file is larger than max_optional_metadata_filesize.
120 if int(dataset.get_size()) <= 1048576:
121 # Small dataset, recount all lines and reset peek afterward.
122 lc = self.count_data_lines(dataset)
123 dataset.metadata.data_lines = lc
124 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
125 else:
126 est_lines = self.estimate_file_lines(dataset)
127 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
128 inflector.cond_plural(est_lines, self.line_class) )
129 else:
130 dataset.blurb = "%s %s" % (
131 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
132 else:
133 dataset.peek = 'file does not exist'
134 dataset.blurb = 'file purged from disk'
135
136 def sniff(self, filename):
137 """All KCF Files simply put a 'ENTRY' in its first line.
138 This applies to all possible kcfs. In this case check
139 for 'Glycan' to confirm it's a glycan """
140 try:
141 from suds.client import Client
142
143 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
144 client = Client(url)
145 kcfresponse = client.service.DeterminingForm(file(filename, 'r').read())
146 if kcfresponse.array[0] == "KCF":
147 return True
148 else:
149 return False
150 except ImportError:
151 # cannot use import suds so use simple checker
152 print "using KCF simple checker"
153 f = open(filename, "r")
154 firstline = f.readline().upper() # note we are uppercasing here to avoid CasE SenSitIVity
155 f.close()
156
157 if "ENTRY" in firstline and "GLYCAN" in firstline:
158 return True
159 else:
160 return False
161 except Exception, e:
162 # note I am not raising an error rather return False and let another sniffer try to type this data
163 traceback.print_exc(file=sys.stdout)
164 return False
165
166 def split(cls, input_datasets, subdir_generator_function, split_params):
167 """
168 Split the input files by line.
169 """
170 if split_params is None:
171 return
172
173 if len(input_datasets) > 1:
174 raise Exception("Text file splitting does not support multiple files")
175 input_files = [ds.file_name for ds in input_datasets]
176
177 lines_per_file = None
178 chunk_size = None
179 if split_params['split_mode'] == 'number_of_parts':
180 lines_per_file = []
181 # Computing the length is expensive!
182 def _file_len(fname):
183 i = 0
184 f = open(fname)
185 for i, l in enumerate(f):
186 pass
187 f.close()
188 return i + 1
189
190 length = _file_len(input_files[0])
191 parts = int(split_params['split_size'])
192 if length < parts:
193 parts = length
194 len_each, remainder = divmod(length, parts)
195 while length > 0:
196 chunk = len_each
197 if remainder > 0:
198 chunk += 1
199 lines_per_file.append(chunk)
200 remainder = - 1
201 length -= chunk
202 elif split_params['split_mode'] == 'to_size':
203 chunk_size = int(split_params['split_size'])
204 else:
205 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
206
207 f = open(input_files[0], 'rt')
208 try:
209 chunk_idx = 0
210 file_done = False
211 part_file = None
212 while not file_done:
213 if lines_per_file is None:
214 this_chunk_size = chunk_size
215 elif chunk_idx < len(lines_per_file):
216 this_chunk_size = lines_per_file[chunk_idx]
217 chunk_idx += 1
218 lines_remaining = this_chunk_size
219 part_file = None
220 while lines_remaining > 0:
221 a_line = f.readline()
222 if a_line == '':
223 file_done = True
224 break
225 if part_file is None:
226 part_dir = subdir_generator_function()
227 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
228 part_file = open(part_path, 'w')
229 part_file.write(a_line)
230 lines_remaining -= 1
231 if part_file is not None:
232 part_file.close()
233 except Exception, e:
234 log.error('Unable to split files: %s' % str(e))
235 f.close()
236 if part_file is not None:
237 part_file.close()
238 raise
239 f.close()
240
241 split = classmethod(split)
242
243
244 class glycoct(data.Data):
245 file_ext = 'glycoct'
246 line_class = 'line'
247
248 """Add metadata elements"""
249 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
250 visible=False, no_value=0)
251
252 def write_from_stream(self, dataset, stream):
253 """Writes data from a stream"""
254 # write it twice for now
255 fd, temp_name = tempfile.mkstemp()
256 while 1:
257 chunk = stream.read(1048576)
258 if not chunk:
259 break
260 os.write(fd, chunk)
261 os.close(fd)
262 # rewrite the file with unix newlines
263 fp = open(dataset.file_name, 'wt')
264 for line in file(temp_name, "U"):
265 line = line.strip() + '\n'
266 fp.write(line)
267 fp.close()
268
269 def set_raw_data(self, dataset, data):
270 """Saves the data on the disc"""
271 fd, temp_name = tempfile.mkstemp()
272 os.write(fd, data)
273 os.close(fd)
274 # rewrite the file with unix newlines
275 fp = open(dataset.file_name, 'wt')
276 for line in file(temp_name, "U"):
277 line = line.strip() + '\n'
278 fp.write(line)
279 fp.close()
280 os.remove(temp_name)
281
282 def get_mime(self):
283 """Returns the mime type of the datatype"""
284 return 'text/plain'
285
286 def set_meta(self, dataset, **kwd):
287 """
288 Set the number of lines of data in dataset.
289 """
290 dataset.metadata.data_lines = self.count_data_lines(dataset)
291
292 def estimate_file_lines(self, dataset):
293 """
294 Perform a rough estimate by extrapolating number of lines from a small read.
295 """
296 sample_size = 1048576
297 dataset_fh = open(dataset.file_name)
298 dataset_read = dataset_fh.read(sample_size)
299 dataset_fh.close()
300 sample_lines = dataset_read.count('\n')
301 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
302 return est_lines
303
304 def count_data_lines(self, dataset):
305 """
306 Count the number of lines of data in dataset,
307 skipping all blank lines and comments.
308 """
309 data_lines = 0
310 for line in file(dataset.file_name):
311 line = line.strip()
312 if line and not line.startswith('#'):
313 data_lines += 1
314 return data_lines
315
316 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
317 """
318 Set the peek. This method is used by various subclasses of Text.
319 """
320 if not dataset.dataset.purged:
321 # The file must exist on disk for the get_file_peek() method
322 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
323 skipchars=skipchars)
324 if line_count is None:
325 # See if line_count is stored in the metadata
326 if dataset.metadata.data_lines:
327 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
328 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
329 else:
330 # Number of lines is not known ( this should not happen ), and auto-detect is
331 # needed to set metadata
332 # This can happen when the file is larger than max_optional_metadata_filesize.
333 if int(dataset.get_size()) <= 1048576:
334 # Small dataset, recount all lines and reset peek afterward.
335 lc = self.count_data_lines(dataset)
336 dataset.metadata.data_lines = lc
337 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
338 else:
339 est_lines = self.estimate_file_lines(dataset)
340 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
341 inflector.cond_plural(est_lines, self.line_class) )
342 else:
343 dataset.blurb = "%s %s" % (
344 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
345 else:
346 dataset.peek = 'file does not exist'
347 dataset.blurb = 'file purged from disk'
348
349 def sniff(self, filename):
350 """All glycoct_condensed files simply put a 'RES' in its first line and a LIN later. """
351 try:
352 f = open(filename, "r")
353 firstline = f.readline().upper() # note we are uppercasing here to avoid CasE SenSitIVity
354 lines = f.read()
355 f.close()
356
357 # if "RES" in firstline and "LIN" in lines:
358 if "RES" in firstline and "LIN" in lines:
359 return True
360 else:
361 return False
362 except Exception, e:
363 # note I am not raising an error rather return False and let another sniffer try to type this data
364 traceback.print_exc(file=sys.stdout)
365 return False
366
367 def split(cls, input_datasets, subdir_generator_function, split_params):
368 """
369 Split the input files by line.
370 """
371 if split_params is None:
372 return
373
374 if len(input_datasets) > 1:
375 raise Exception("Text file splitting does not support multiple files")
376 input_files = [ds.file_name for ds in input_datasets]
377
378 lines_per_file = None
379 chunk_size = None
380 if split_params['split_mode'] == 'number_of_parts':
381 lines_per_file = []
382 # Computing the length is expensive!
383 def _file_len(fname):
384 i = 0
385 f = open(fname)
386 for i, l in enumerate(f):
387 pass
388 f.close()
389 return i + 1
390
391 length = _file_len(input_files[0])
392 parts = int(split_params['split_size'])
393 if length < parts:
394 parts = length
395 len_each, remainder = divmod(length, parts)
396 while length > 0:
397 chunk = len_each
398 if remainder > 0:
399 chunk += 1
400 lines_per_file.append(chunk)
401 remainder = - 1
402 length -= chunk
403 elif split_params['split_mode'] == 'to_size':
404 chunk_size = int(split_params['split_size'])
405 else:
406 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
407
408 f = open(input_files[0], 'rt')
409 try:
410 chunk_idx = 0
411 file_done = False
412 part_file = None
413 while not file_done:
414 if lines_per_file is None:
415 this_chunk_size = chunk_size
416 elif chunk_idx < len(lines_per_file):
417 this_chunk_size = lines_per_file[chunk_idx]
418 chunk_idx += 1
419 lines_remaining = this_chunk_size
420 part_file = None
421 while lines_remaining > 0:
422 a_line = f.readline()
423 if a_line == '':
424 file_done = True
425 break
426 if part_file is None:
427 part_dir = subdir_generator_function()
428 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
429 part_file = open(part_path, 'w')
430 part_file.write(a_line)
431 lines_remaining -= 1
432 if part_file is not None:
433 part_file.close()
434 except Exception, e:
435 log.error('Unable to split files: %s' % str(e))
436 f.close()
437 if part_file is not None:
438 part_file.close()
439 raise
440 f.close()
441
442 split = classmethod(split)
443
444 # ------------- Utility methods --------------
445
446 # nice_size used to be here, but to resolve cyclical dependencies it's been
447 # moved to galaxy.util. It belongs there anyway since it's used outside
448 # datatypes.
449 nice_size = util.nice_size
450
451
452 def get_test_fname(fname):
453 """Returns test data filename"""
454 path, name = os.path.split(__file__)
455 full_path = os.path.join(path, 'test', fname)
456 return full_path
457
458
459 def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=[]):
460 """
461 Returns the first LINE_COUNT lines wrapped to WIDTH
462
463 ## >>> fname = get_test_fname('4.bed')
464 ## >>> get_file_peek(fname)
465 ## 'chr22 30128507 31828507 uc003bnx.1_cds_2_0_chr22_29227_f 0 +\n'
466
467 """
468 # Set size for file.readline() to a negative number to force it to
469 # read until either a newline or EOF. Needed for datasets with very
470 # long lines.
471 if WIDTH == 'unlimited':
472 WIDTH = -1
473 lines = []
474 count = 0
475 file_type = None
476 data_checked = False
477 temp = open(file_name, "U")
478 while count <= LINE_COUNT:
479 line = temp.readline(WIDTH)
480 if line and not is_multi_byte and not data_checked:
481 # See if we have a compressed or binary file
482 if line[0:2] == util.gzip_magic:
483 file_type = 'gzipped'
484 break
485 else:
486 for char in line:
487 if ord(char) > 128:
488 file_type = 'binary'
489 break
490 data_checked = True
491 if file_type in ['gzipped', 'binary']:
492 break
493 skip_line = False
494 for skipchar in skipchars:
495 if line.startswith(skipchar):
496 skip_line = True
497 break
498 if not skip_line:
499 lines.append(line)
500 count += 1
501 temp.close()
502 if file_type in ['gzipped', 'binary']:
503 text = "%s file" % file_type
504 else:
505 try:
506 text = unicode('\n'.join(lines), 'utf-8')
507 except UnicodeDecodeError:
508 text = "binary/unknown file"
509 return text
510
511
512 class glycoct_xml(data.Data):
513 file_ext = 'glycoct_xml'
514 line_class = 'line'
515
516 """Add metadata elements"""
517 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
518 visible=False, no_value=0)
519
520 def write_from_stream(self, dataset, stream):
521 """Writes data from a stream"""
522 # write it twice for now
523 fd, temp_name = tempfile.mkstemp()
524 while 1:
525 chunk = stream.read(1048576)
526 if not chunk:
527 break
528 os.write(fd, chunk)
529 os.close(fd)
530 # rewrite the file with unix newlines
531 fp = open(dataset.file_name, 'wt')
532 for line in file(temp_name, "U"):
533 line = line.strip() + '\n'
534 fp.write(line)
535 fp.close()
536
537 def set_raw_data(self, dataset, data):
538 """Saves the data on the disc"""
539 fd, temp_name = tempfile.mkstemp()
540 os.write(fd, data)
541 os.close(fd)
542 # rewrite the file with unix newlines
543 fp = open(dataset.file_name, 'wt')
544 for line in file(temp_name, "U"):
545 line = line.strip() + '\n'
546 fp.write(line)
547 fp.close()
548 os.remove(temp_name)
549
550 def get_mime(self):
551 """Returns the mime type of the datatype"""
552 return 'text/xml'
553
554 def set_meta(self, dataset, **kwd):
555 """
556 Set the number of lines of data in dataset.
557 """
558 dataset.metadata.data_lines = self.count_data_lines(dataset)
559
560 def estimate_file_lines(self, dataset):
561 """
562 Perform a rough estimate by extrapolating number of lines from a small read.
563 """
564 sample_size = 1048576
565 dataset_fh = open(dataset.file_name)
566 dataset_read = dataset_fh.read(sample_size)
567 dataset_fh.close()
568 sample_lines = dataset_read.count('\n')
569 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
570 return est_lines
571
572 def count_data_lines(self, dataset):
573 """
574 Count the number of lines of data in dataset,
575 skipping all blank lines and comments.
576 """
577 data_lines = 0
578 for line in file(dataset.file_name):
579 line = line.strip()
580 if line and not line.startswith('#'):
581 data_lines += 1
582 return data_lines
583
584 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
585 """
586 Set the peek. This method is used by various subclasses of Text.
587 """
588 if not dataset.dataset.purged:
589 # The file must exist on disk for the get_file_peek() method
590 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
591 skipchars=skipchars)
592 if line_count is None:
593 # See if line_count is stored in the metadata
594 if dataset.metadata.data_lines:
595 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
596 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
597 else:
598 # Number of lines is not known ( this should not happen ), and auto-detect is
599 # needed to set metadata
600 # This can happen when the file is larger than max_optional_metadata_filesize.
601 if int(dataset.get_size()) <= 1048576:
602 # Small dataset, recount all lines and reset peek afterward.
603 lc = self.count_data_lines(dataset)
604 dataset.metadata.data_lines = lc
605 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
606 else:
607 est_lines = self.estimate_file_lines(dataset)
608 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
609 inflector.cond_plural(est_lines, self.line_class) )
610 else:
611 dataset.blurb = "%s %s" % (
612 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
613 else:
614 dataset.peek = 'file does not exist'
615 dataset.blurb = 'file purged from disk'
616
617 def sniff(self, filename):
618 """All glycoct XML files should use the rings form determination script """
619 try:
620 from suds.client import Client
621
622 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
623 client = Client(url)
624 response = client.service.DeterminingForm(file(filename, 'r').read())
625 if response.array[0] == "GlycoCT":
626 return True
627 else:
628 return False
629 except ImportError:
630 # cannot use import suds so use simple checker
631 print "using glycoct XML simple checker"
632 import xml.etree.cElementTree as ET
633
634 tree = ET.parse(filename)
635 root = tree.getroot()
636 if root.tag == 'sugar':
637 print root.tag, root.attrib
638 return True
639 else:
640 return False
641 except Exception, e:
642 # note I am not raising an error rather return False and let another sniffer try to type this data
643 traceback.print_exc(file=sys.stdout)
644 return False
645
646 def split(cls, input_datasets, subdir_generator_function, split_params):
647 """
648 Split the input files by line.
649 """
650 if split_params is None:
651 return
652
653 if len(input_datasets) > 1:
654 raise Exception("Text file splitting does not support multiple files")
655 input_files = [ds.file_name for ds in input_datasets]
656
657 lines_per_file = None
658 chunk_size = None
659 if split_params['split_mode'] == 'number_of_parts':
660 lines_per_file = []
661 # Computing the length is expensive!
662 def _file_len(fname):
663 i = 0
664 f = open(fname)
665 for i, l in enumerate(f):
666 pass
667 f.close()
668 return i + 1
669
670 length = _file_len(input_files[0])
671 parts = int(split_params['split_size'])
672 if length < parts:
673 parts = length
674 len_each, remainder = divmod(length, parts)
675 while length > 0:
676 chunk = len_each
677 if remainder > 0:
678 chunk += 1
679 lines_per_file.append(chunk)
680 remainder = - 1
681 length -= chunk
682 elif split_params['split_mode'] == 'to_size':
683 chunk_size = int(split_params['split_size'])
684 else:
685 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
686
687 f = open(input_files[0], 'rt')
688 try:
689 chunk_idx = 0
690 file_done = False
691 part_file = None
692 while not file_done:
693 if lines_per_file is None:
694 this_chunk_size = chunk_size
695 elif chunk_idx < len(lines_per_file):
696 this_chunk_size = lines_per_file[chunk_idx]
697 chunk_idx += 1
698 lines_remaining = this_chunk_size
699 part_file = None
700 while lines_remaining > 0:
701 a_line = f.readline()
702 if a_line == '':
703 file_done = True
704 break
705 if part_file is None:
706 part_dir = subdir_generator_function()
707 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
708 part_file = open(part_path, 'w')
709 part_file.write(a_line)
710 lines_remaining -= 1
711 if part_file is not None:
712 part_file.close()
713 except Exception, e:
714 log.error('Unable to split files: %s' % str(e))
715 f.close()
716 if part_file is not None:
717 part_file.close()
718 raise
719 f.close()
720
721 split = classmethod(split)
722
723
724 class glydeii(data.Data):
725 file_ext = 'glydeii'
726 line_class = 'line'
727
728 """Add metadata elements"""
729 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
730 visible=False, no_value=0)
731
732 def write_from_stream(self, dataset, stream):
733 """Writes data from a stream"""
734 # write it twice for now
735 fd, temp_name = tempfile.mkstemp()
736 while 1:
737 chunk = stream.read(1048576)
738 if not chunk:
739 break
740 os.write(fd, chunk)
741 os.close(fd)
742 # rewrite the file with unix newlines
743 fp = open(dataset.file_name, 'wt')
744 for line in file(temp_name, "U"):
745 line = line.strip() + '\n'
746 fp.write(line)
747 fp.close()
748
749 def set_raw_data(self, dataset, data):
750 """Saves the data on the disc"""
751 fd, temp_name = tempfile.mkstemp()
752 os.write(fd, data)
753 os.close(fd)
754 # rewrite the file with unix newlines
755 fp = open(dataset.file_name, 'wt')
756 for line in file(temp_name, "U"):
757 line = line.strip() + '\n'
758 fp.write(line)
759 fp.close()
760 os.remove(temp_name)
761
762 def get_mime(self):
763 """Returns the mime type of the datatype"""
764 return 'text/xml'
765
766 def set_meta(self, dataset, **kwd):
767 """
768 Set the number of lines of data in dataset.
769 """
770 dataset.metadata.data_lines = self.count_data_lines(dataset)
771
772 def estimate_file_lines(self, dataset):
773 """
774 Perform a rough estimate by extrapolating number of lines from a small read.
775 """
776 sample_size = 1048576
777 dataset_fh = open(dataset.file_name)
778 dataset_read = dataset_fh.read(sample_size)
779 dataset_fh.close()
780 sample_lines = dataset_read.count('\n')
781 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
782 return est_lines
783
784 def count_data_lines(self, dataset):
785 """
786 Count the number of lines of data in dataset,
787 skipping all blank lines and comments.
788 """
789 data_lines = 0
790 for line in file(dataset.file_name):
791 line = line.strip()
792 if line and not line.startswith('#'):
793 data_lines += 1
794 return data_lines
795
796 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
797 """
798 Set the peek. This method is used by various subclasses of Text.
799 """
800 if not dataset.dataset.purged:
801 # The file must exist on disk for the get_file_peek() method
802 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
803 skipchars=skipchars)
804 if line_count is None:
805 # See if line_count is stored in the metadata
806 if dataset.metadata.data_lines:
807 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
808 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
809 else:
810 # Number of lines is not known ( this should not happen ), and auto-detect is
811 # needed to set metadata
812 # This can happen when the file is larger than max_optional_metadata_filesize.
813 if int(dataset.get_size()) <= 1048576:
814 # Small dataset, recount all lines and reset peek afterward.
815 lc = self.count_data_lines(dataset)
816 dataset.metadata.data_lines = lc
817 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
818 else:
819 est_lines = self.estimate_file_lines(dataset)
820 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
821 inflector.cond_plural(est_lines, self.line_class) )
822 else:
823 dataset.blurb = "%s %s" % (
824 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
825 else:
826 dataset.peek = 'file does not exist'
827 dataset.blurb = 'file purged from disk'
828
829 def sniff(self, filename):
830 """All GlydeII XML files should use the rings form determination script """
831 try:
832 from suds.client import Client
833
834 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
835 client = Client(url)
836 response = client.service.DeterminingForm(file(filename, 'r').read())
837 if response.array[0] == "GLYDEII":
838 return True
839 else:
840 return False
841 except ImportError:
842 # cannot use import suds so use simple checker
843 print "using GlydeII simple checker"
844 import xml.etree.cElementTree as ET
845
846 tree = ET.parse(filename)
847 root = tree.getroot()
848 if root.tag == 'GlydeII':
849 print root.tag
850 return True
851 else:
852 return False
853 except Exception, e:
854 # note I am not raising an error rather return False and let another sniffer try to type this data
855 traceback.print_exc(file=sys.stdout)
856 return False
857
858 def split(cls, input_datasets, subdir_generator_function, split_params):
859 """
860 Split the input files by line.
861 """
862 if split_params is None:
863 return
864
865 if len(input_datasets) > 1:
866 raise Exception("Text file splitting does not support multiple files")
867 input_files = [ds.file_name for ds in input_datasets]
868
869 lines_per_file = None
870 chunk_size = None
871 if split_params['split_mode'] == 'number_of_parts':
872 lines_per_file = []
873 # Computing the length is expensive!
874 def _file_len(fname):
875 i = 0
876 f = open(fname)
877 for i, l in enumerate(f):
878 pass
879 f.close()
880 return i + 1
881
882 length = _file_len(input_files[0])
883 parts = int(split_params['split_size'])
884 if length < parts:
885 parts = length
886 len_each, remainder = divmod(length, parts)
887 while length > 0:
888 chunk = len_each
889 if remainder > 0:
890 chunk += 1
891 lines_per_file.append(chunk)
892 remainder = - 1
893 length -= chunk
894 elif split_params['split_mode'] == 'to_size':
895 chunk_size = int(split_params['split_size'])
896 else:
897 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
898
899 f = open(input_files[0], 'rt')
900 try:
901 chunk_idx = 0
902 file_done = False
903 part_file = None
904 while not file_done:
905 if lines_per_file is None:
906 this_chunk_size = chunk_size
907 elif chunk_idx < len(lines_per_file):
908 this_chunk_size = lines_per_file[chunk_idx]
909 chunk_idx += 1
910 lines_remaining = this_chunk_size
911 part_file = None
912 while lines_remaining > 0:
913 a_line = f.readline()
914 if a_line == '':
915 file_done = True
916 break
917 if part_file is None:
918 part_dir = subdir_generator_function()
919 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
920 part_file = open(part_path, 'w')
921 part_file.write(a_line)
922 lines_remaining -= 1
923 if part_file is not None:
924 part_file.close()
925 except Exception, e:
926 log.error('Unable to split files: %s' % str(e))
927 f.close()
928 if part_file is not None:
929 part_file.close()
930 raise
931 f.close()
932
933 split = classmethod(split)
934
935
936 class linucs(data.Data):
937 file_ext = 'linucs'
938 line_class = 'line'
939
940 """Add metadata elements"""
941 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
942 visible=False, no_value=0)
943
944 def write_from_stream(self, dataset, stream):
945 """Writes data from a stream"""
946 # write it twice for now
947 fd, temp_name = tempfile.mkstemp()
948 while 1:
949 chunk = stream.read(1048576)
950 if not chunk:
951 break
952 os.write(fd, chunk)
953 os.close(fd)
954 # rewrite the file with unix newlines
955 fp = open(dataset.file_name, 'wt')
956 for line in file(temp_name, "U"):
957 line = line.strip() + '\n'
958 fp.write(line)
959 fp.close()
960
961 def set_raw_data(self, dataset, data):
962 """Saves the data on the disc"""
963 fd, temp_name = tempfile.mkstemp()
964 os.write(fd, data)
965 os.close(fd)
966 # rewrite the file with unix newlines
967 fp = open(dataset.file_name, 'wt')
968 for line in file(temp_name, "U"):
969 line = line.strip() + '\n'
970 fp.write(line)
971 fp.close()
972 os.remove(temp_name)
973
974 def get_mime(self):
975 """Returns the mime type of the datatype"""
976 return 'text/plain'
977
978 def set_meta(self, dataset, **kwd):
979 """
980 Set the number of lines of data in dataset.
981 """
982 dataset.metadata.data_lines = self.count_data_lines(dataset)
983
984 def estimate_file_lines(self, dataset):
985 """
986 Perform a rough estimate by extrapolating number of lines from a small read.
987 """
988 sample_size = 1048576
989 dataset_fh = open(dataset.file_name)
990 dataset_read = dataset_fh.read(sample_size)
991 dataset_fh.close()
992 sample_lines = dataset_read.count('\n')
993 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
994 return est_lines
995
996 def count_data_lines(self, dataset):
997 """
998 Count the number of lines of data in dataset,
999 skipping all blank lines and comments.
1000 """
1001 data_lines = 0
1002 for line in file(dataset.file_name):
1003 line = line.strip()
1004 if line and not line.startswith('#'):
1005 data_lines += 1
1006 return data_lines
1007
1008 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
1009 """
1010 Set the peek. This method is used by various subclasses of Text.
1011 """
1012 if not dataset.dataset.purged:
1013 # The file must exist on disk for the get_file_peek() method
1014 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
1015 skipchars=skipchars)
1016 if line_count is None:
1017 # See if line_count is stored in the metadata
1018 if dataset.metadata.data_lines:
1019 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
1020 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
1021 else:
1022 # Number of lines is not known ( this should not happen ), and auto-detect is
1023 # needed to set metadata
1024 # This can happen when the file is larger than max_optional_metadata_filesize.
1025 if int(dataset.get_size()) <= 1048576:
1026 # Small dataset, recount all lines and reset peek afterward.
1027 lc = self.count_data_lines(dataset)
1028 dataset.metadata.data_lines = lc
1029 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
1030 else:
1031 est_lines = self.estimate_file_lines(dataset)
1032 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
1033 inflector.cond_plural(est_lines, self.line_class) )
1034 else:
1035 dataset.blurb = "%s %s" % (
1036 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
1037 else:
1038 dataset.peek = 'file does not exist'
1039 dataset.blurb = 'file purged from disk'
1040
1041 def sniff(self, filename):
1042 """All LINUCS files should use the rings form determination script """
1043 try:
1044 from suds.client import Client
1045
1046 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
1047 client = Client(url)
1048 response = client.service.DeterminingForm(file(filename, 'r').read())
1049 if response.array[0] == "LINUCS":
1050 return True
1051 else:
1052 return False
1053 except ImportError:
1054 # cannot use import suds so use simple checker
1055 print "using LINUCS simple checker"
1056
1057 f = open(filename, "r")
1058 firstline = f.readline()
1059 f.close()
1060
1061 if "[" in firstline and "]" in firstline and "{" in firstline and "}" in firstline:
1062 return True
1063 else:
1064 return False
1065 except Exception, e:
1066 # note I am not raising an error rather return False and let another sniffer try to type this data
1067 traceback.print_exc(file=sys.stdout)
1068 return False
1069
1070 def split(cls, input_datasets, subdir_generator_function, split_params):
1071 """
1072 Split the input files by line.
1073 """
1074 if split_params is None:
1075 return
1076
1077 if len(input_datasets) > 1:
1078 raise Exception("Text file splitting does not support multiple files")
1079 input_files = [ds.file_name for ds in input_datasets]
1080
1081 lines_per_file = None
1082 chunk_size = None
1083 if split_params['split_mode'] == 'number_of_parts':
1084 lines_per_file = []
1085 # Computing the length is expensive!
1086 def _file_len(fname):
1087 i = 0
1088 f = open(fname)
1089 for i, l in enumerate(f):
1090 pass
1091 f.close()
1092 return i + 1
1093
1094 length = _file_len(input_files[0])
1095 parts = int(split_params['split_size'])
1096 if length < parts:
1097 parts = length
1098 len_each, remainder = divmod(length, parts)
1099 while length > 0:
1100 chunk = len_each
1101 if remainder > 0:
1102 chunk += 1
1103 lines_per_file.append(chunk)
1104 remainder = - 1
1105 length -= chunk
1106 elif split_params['split_mode'] == 'to_size':
1107 chunk_size = int(split_params['split_size'])
1108 else:
1109 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
1110
1111 f = open(input_files[0], 'rt')
1112 try:
1113 chunk_idx = 0
1114 file_done = False
1115 part_file = None
1116 while not file_done:
1117 if lines_per_file is None:
1118 this_chunk_size = chunk_size
1119 elif chunk_idx < len(lines_per_file):
1120 this_chunk_size = lines_per_file[chunk_idx]
1121 chunk_idx += 1
1122 lines_remaining = this_chunk_size
1123 part_file = None
1124 while lines_remaining > 0:
1125 a_line = f.readline()
1126 if a_line == '':
1127 file_done = True
1128 break
1129 if part_file is None:
1130 part_dir = subdir_generator_function()
1131 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
1132 part_file = open(part_path, 'w')
1133 part_file.write(a_line)
1134 lines_remaining -= 1
1135 if part_file is not None:
1136 part_file.close()
1137 except Exception, e:
1138 log.error('Unable to split files: %s' % str(e))
1139 f.close()
1140 if part_file is not None:
1141 part_file.close()
1142 raise
1143 f.close()
1144
1145 split = classmethod(split)
1146
1147
1148 class iupac(data.Data):
1149 file_ext = 'iupac'
1150 line_class = 'line'
1151
1152 """Add metadata elements"""
1153 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
1154 visible=False, no_value=0)
1155
1156 def write_from_stream(self, dataset, stream):
1157 """Writes data from a stream"""
1158 # write it twice for now
1159 fd, temp_name = tempfile.mkstemp()
1160 while 1:
1161 chunk = stream.read(1048576)
1162 if not chunk:
1163 break
1164 os.write(fd, chunk)
1165 os.close(fd)
1166 # rewrite the file with unix newlines
1167 fp = open(dataset.file_name, 'wt')
1168 for line in file(temp_name, "U"):
1169 line = line.strip() + '\n'
1170 fp.write(line)
1171 fp.close()
1172
1173 def set_raw_data(self, dataset, data):
1174 """Saves the data on the disc"""
1175 fd, temp_name = tempfile.mkstemp()
1176 os.write(fd, data)
1177 os.close(fd)
1178 # rewrite the file with unix newlines
1179 fp = open(dataset.file_name, 'wt')
1180 for line in file(temp_name, "U"):
1181 line = line.strip() + '\n'
1182 fp.write(line)
1183 fp.close()
1184 os.remove(temp_name)
1185
1186 def get_mime(self):
1187 """Returns the mime type of the datatype"""
1188 return 'text/plain'
1189
1190 def set_meta(self, dataset, **kwd):
1191 """
1192 Set the number of lines of data in dataset.
1193 """
1194 dataset.metadata.data_lines = self.count_data_lines(dataset)
1195
1196 def estimate_file_lines(self, dataset):
1197 """
1198 Perform a rough estimate by extrapolating number of lines from a small read.
1199 """
1200 sample_size = 1048576
1201 dataset_fh = open(dataset.file_name)
1202 dataset_read = dataset_fh.read(sample_size)
1203 dataset_fh.close()
1204 sample_lines = dataset_read.count('\n')
1205 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
1206 return est_lines
1207
1208 def count_data_lines(self, dataset):
1209 """
1210 Count the number of lines of data in dataset,
1211 skipping all blank lines and comments.
1212 """
1213 data_lines = 0
1214 for line in file(dataset.file_name):
1215 line = line.strip()
1216 if line and not line.startswith('#'):
1217 data_lines += 1
1218 return data_lines
1219
1220 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
1221 """
1222 Set the peek. This method is used by various subclasses of Text.
1223 """
1224 if not dataset.dataset.purged:
1225 # The file must exist on disk for the get_file_peek() method
1226 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
1227 skipchars=skipchars)
1228 if line_count is None:
1229 # See if line_count is stored in the metadata
1230 if dataset.metadata.data_lines:
1231 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
1232 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
1233 else:
1234 # Number of lines is not known ( this should not happen ), and auto-detect is
1235 # needed to set metadata
1236 # This can happen when the file is larger than max_optional_metadata_filesize.
1237 if int(dataset.get_size()) <= 1048576:
1238 # Small dataset, recount all lines and reset peek afterward.
1239 lc = self.count_data_lines(dataset)
1240 dataset.metadata.data_lines = lc
1241 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
1242 else:
1243 est_lines = self.estimate_file_lines(dataset)
1244 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
1245 inflector.cond_plural(est_lines, self.line_class) )
1246 else:
1247 dataset.blurb = "%s %s" % (
1248 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
1249 else:
1250 dataset.peek = 'file does not exist'
1251 dataset.blurb = 'file purged from disk'
1252
1253 def sniff(self, filename):
1254 """All IUPAC files should use the rings form determination script """
1255 try:
1256 from suds.client import Client
1257
1258 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
1259 client = Client(url)
1260 response = client.service.DeterminingForm(file(filename, 'r').read())
1261 if response.array[0] == "IUPAC":
1262 return True
1263 else:
1264 return False
1265 except ImportError:
1266 # cannot use import suds so use simple checker
1267 print "using IUPAC simple checker"
1268 f = open(filename, "r")
1269 firstline = f.readline()
1270 f.close()
1271
1272 if "[" in firstline or "]" in firstline or "(" in firstline or ")" in firstline:
1273 if "{" in firstline or "}" in firstline:
1274 return False
1275 else:
1276 return True
1277 else:
1278 return False
1279 except Exception, e:
1280 # note I am not raising an error rather return False and let another sniffer try to type this data
1281 traceback.print_exc(file=sys.stdout)
1282 return False
1283
1284 def split(cls, input_datasets, subdir_generator_function, split_params):
1285 """
1286 Split the input files by line.
1287 """
1288 if split_params is None:
1289 return
1290
1291 if len(input_datasets) > 1:
1292 raise Exception("Text file splitting does not support multiple files")
1293 input_files = [ds.file_name for ds in input_datasets]
1294
1295 lines_per_file = None
1296 chunk_size = None
1297 if split_params['split_mode'] == 'number_of_parts':
1298 lines_per_file = []
1299 # Computing the length is expensive!
1300 def _file_len(fname):
1301 i = 0
1302 f = open(fname)
1303 for i, l in enumerate(f):
1304 pass
1305 f.close()
1306 return i + 1
1307
1308 length = _file_len(input_files[0])
1309 parts = int(split_params['split_size'])
1310 if length < parts:
1311 parts = length
1312 len_each, remainder = divmod(length, parts)
1313 while length > 0:
1314 chunk = len_each
1315 if remainder > 0:
1316 chunk += 1
1317 lines_per_file.append(chunk)
1318 remainder = - 1
1319 length -= chunk
1320 elif split_params['split_mode'] == 'to_size':
1321 chunk_size = int(split_params['split_size'])
1322 else:
1323 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
1324
1325 f = open(input_files[0], 'rt')
1326 try:
1327 chunk_idx = 0
1328 file_done = False
1329 part_file = None
1330 while not file_done:
1331 if lines_per_file is None:
1332 this_chunk_size = chunk_size
1333 elif chunk_idx < len(lines_per_file):
1334 this_chunk_size = lines_per_file[chunk_idx]
1335 chunk_idx += 1
1336 lines_remaining = this_chunk_size
1337 part_file = None
1338 while lines_remaining > 0:
1339 a_line = f.readline()
1340 if a_line == '':
1341 file_done = True
1342 break
1343 if part_file is None:
1344 part_dir = subdir_generator_function()
1345 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
1346 part_file = open(part_path, 'w')
1347 part_file.write(a_line)
1348 lines_remaining -= 1
1349 if part_file is not None:
1350 part_file.close()
1351 except Exception, e:
1352 log.error('Unable to split files: %s' % str(e))
1353 f.close()
1354 if part_file is not None:
1355 part_file.close()
1356 raise
1357 f.close()
1358
1359 split = classmethod(split)
1360
1361
1362 class linearcode(data.Data):
1363 file_ext = 'linearcode'
1364 line_class = 'line'
1365
1366 """Add metadata elements"""
1367 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
1368 visible=False, no_value=0)
1369
1370 def write_from_stream(self, dataset, stream):
1371 """Writes data from a stream"""
1372 # write it twice for now
1373 fd, temp_name = tempfile.mkstemp()
1374 while 1:
1375 chunk = stream.read(1048576)
1376 if not chunk:
1377 break
1378 os.write(fd, chunk)
1379 os.close(fd)
1380 # rewrite the file with unix newlines
1381 fp = open(dataset.file_name, 'wt')
1382 for line in file(temp_name, "U"):
1383 line = line.strip() + '\n'
1384 fp.write(line)
1385 fp.close()
1386
1387 def set_raw_data(self, dataset, data):
1388 """Saves the data on the disc"""
1389 fd, temp_name = tempfile.mkstemp()
1390 os.write(fd, data)
1391 os.close(fd)
1392 # rewrite the file with unix newlines
1393 fp = open(dataset.file_name, 'wt')
1394 for line in file(temp_name, "U"):
1395 line = line.strip() + '\n'
1396 fp.write(line)
1397 fp.close()
1398 os.remove(temp_name)
1399
1400 def get_mime(self):
1401 """Returns the mime type of the datatype"""
1402 return 'text/plain'
1403
1404 def set_meta(self, dataset, **kwd):
1405 """
1406 Set the number of lines of data in dataset.
1407 """
1408 dataset.metadata.data_lines = self.count_data_lines(dataset)
1409
1410 def estimate_file_lines(self, dataset):
1411 """
1412 Perform a rough estimate by extrapolating number of lines from a small read.
1413 """
1414 sample_size = 1048576
1415 dataset_fh = open(dataset.file_name)
1416 dataset_read = dataset_fh.read(sample_size)
1417 dataset_fh.close()
1418 sample_lines = dataset_read.count('\n')
1419 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
1420 return est_lines
1421
1422 def count_data_lines(self, dataset):
1423 """
1424 Count the number of lines of data in dataset,
1425 skipping all blank lines and comments.
1426 """
1427 data_lines = 0
1428 for line in file(dataset.file_name):
1429 line = line.strip()
1430 if line and not line.startswith('#'):
1431 data_lines += 1
1432 return data_lines
1433
1434 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
1435 """
1436 Set the peek. This method is used by various subclasses of Text.
1437 """
1438 if not dataset.dataset.purged:
1439 # The file must exist on disk for the get_file_peek() method
1440 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
1441 skipchars=skipchars)
1442 if line_count is None:
1443 # See if line_count is stored in the metadata
1444 if dataset.metadata.data_lines:
1445 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
1446 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
1447 else:
1448 # Number of lines is not known ( this should not happen ), and auto-detect is
1449 # needed to set metadata
1450 # This can happen when the file is larger than max_optional_metadata_filesize.
1451 if int(dataset.get_size()) <= 1048576:
1452 # Small dataset, recount all lines and reset peek afterward.
1453 lc = self.count_data_lines(dataset)
1454 dataset.metadata.data_lines = lc
1455 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
1456 else:
1457 est_lines = self.estimate_file_lines(dataset)
1458 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
1459 inflector.cond_plural(est_lines, self.line_class) )
1460 else:
1461 dataset.blurb = "%s %s" % (
1462 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
1463 else:
1464 dataset.peek = 'file does not exist'
1465 dataset.blurb = 'file purged from disk'
1466
1467 def sniff(self, filename):
1468 """All linear code files should use the rings form determination script """
1469 try:
1470 from suds.client import Client
1471
1472 url = 'http://rings.t.soka.ac.jp/axis2/services/Utilities?wsdl'
1473 client = Client(url)
1474 lcresponse = client.service.DeterminingForm(file(filename, 'r').read())
1475 if lcresponse.array[0] == "LinearCode":
1476 print "LinearCode"
1477 return True
1478 else:
1479 print "Unable to guess format"
1480 return False
1481 except ImportError:
1482 # cannot use import suds so use simple checker
1483 print "using LinearCode simple checker - nope it does not exist yet"
1484 return False
1485 except Exception, e:
1486 # note I am not raising an error rather return False and let another sniffer try to type this data
1487 traceback.print_exc(file=sys.stdout)
1488 return False
1489
1490 def split(cls, input_datasets, subdir_generator_function, split_params):
1491 """
1492 Split the input files by line.
1493 """
1494 if split_params is None:
1495 return
1496
1497 if len(input_datasets) > 1:
1498 raise Exception("Text file splitting does not support multiple files")
1499 input_files = [ds.file_name for ds in input_datasets]
1500
1501 lines_per_file = None
1502 chunk_size = None
1503 if split_params['split_mode'] == 'number_of_parts':
1504 lines_per_file = []
1505 # Computing the length is expensive!
1506 def _file_len(fname):
1507 i = 0
1508 f = open(fname)
1509 for i, l in enumerate(f):
1510 pass
1511 f.close()
1512 return i + 1
1513
1514 length = _file_len(input_files[0])
1515 parts = int(split_params['split_size'])
1516 if length < parts:
1517 parts = length
1518 len_each, remainder = divmod(length, parts)
1519 while length > 0:
1520 chunk = len_each
1521 if remainder > 0:
1522 chunk += 1
1523 lines_per_file.append(chunk)
1524 remainder = - 1
1525 length -= chunk
1526 elif split_params['split_mode'] == 'to_size':
1527 chunk_size = int(split_params['split_size'])
1528 else:
1529 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
1530
1531 f = open(input_files[0], 'rt')
1532 try:
1533 chunk_idx = 0
1534 file_done = False
1535 part_file = None
1536 while not file_done:
1537 if lines_per_file is None:
1538 this_chunk_size = chunk_size
1539 elif chunk_idx < len(lines_per_file):
1540 this_chunk_size = lines_per_file[chunk_idx]
1541 chunk_idx += 1
1542 lines_remaining = this_chunk_size
1543 part_file = None
1544 while lines_remaining > 0:
1545 a_line = f.readline()
1546 if a_line == '':
1547 file_done = True
1548 break
1549 if part_file is None:
1550 part_dir = subdir_generator_function()
1551 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
1552 part_file = open(part_path, 'w')
1553 part_file.write(a_line)
1554 lines_remaining -= 1
1555 if part_file is not None:
1556 part_file.close()
1557 except Exception, e:
1558 log.error('Unable to split files: %s' % str(e))
1559 f.close()
1560 if part_file is not None:
1561 part_file.close()
1562 raise
1563 f.close()
1564
1565 split = classmethod(split)
1566
1567
1568 class msa(data.Data):
1569 file_ext = 'msa'
1570 line_class = 'line'
1571
1572 """Add metadata elements"""
1573 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
1574 visible=False, no_value=0)
1575
1576 def write_from_stream(self, dataset, stream):
1577 """Writes data from a stream"""
1578 # write it twice for now
1579 fd, temp_name = tempfile.mkstemp()
1580 while 1:
1581 chunk = stream.read(1048576)
1582 if not chunk:
1583 break
1584 os.write(fd, chunk)
1585 os.close(fd)
1586 # rewrite the file with unix newlines
1587 fp = open(dataset.file_name, 'wt')
1588 for line in file(temp_name, "U"):
1589 line = line.strip() + '\n'
1590 fp.write(line)
1591 fp.close()
1592
1593 def set_raw_data(self, dataset, data):
1594 """Saves the data on the disc"""
1595 fd, temp_name = tempfile.mkstemp()
1596 os.write(fd, data)
1597 os.close(fd)
1598 # rewrite the file with unix newlines
1599 fp = open(dataset.file_name, 'wt')
1600 for line in file(temp_name, "U"):
1601 line = line.strip() + '\n'
1602 fp.write(line)
1603 fp.close()
1604 os.remove(temp_name)
1605
1606 def get_mime(self):
1607 """Returns the mime type of the datatype"""
1608 return 'text/plain'
1609
1610 def set_meta(self, dataset, **kwd):
1611 """
1612 Set the number of lines of data in dataset.
1613 """
1614 dataset.metadata.data_lines = self.count_data_lines(dataset)
1615
1616 def estimate_file_lines(self, dataset):
1617 """
1618 Perform a rough estimate by extrapolating number of lines from a small read.
1619 """
1620 sample_size = 1048576
1621 dataset_fh = open(dataset.file_name)
1622 dataset_read = dataset_fh.read(sample_size)
1623 dataset_fh.close()
1624 sample_lines = dataset_read.count('\n')
1625 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
1626 return est_lines
1627
1628 def count_data_lines(self, dataset):
1629 """
1630 Count the number of lines of data in dataset,
1631 skipping all blank lines and comments.
1632 """
1633 data_lines = 0
1634 for line in file(dataset.file_name):
1635 line = line.strip()
1636 if line and not line.startswith('#'):
1637 data_lines += 1
1638 return data_lines
1639
1640 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
1641 """
1642 Set the peek. This method is used by various subclasses of Text.
1643 """
1644 if not dataset.dataset.purged:
1645 # The file must exist on disk for the get_file_peek() method
1646 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
1647 skipchars=skipchars)
1648 if line_count is None:
1649 # See if line_count is stored in the metadata
1650 if dataset.metadata.data_lines:
1651 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
1652 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
1653 else:
1654 # Number of lines is not known ( this should not happen ), and auto-detect is
1655 # needed to set metadata
1656 # This can happen when the file is larger than max_optional_metadata_filesize.
1657 if int(dataset.get_size()) <= 1048576:
1658 # Small dataset, recount all lines and reset peek afterward.
1659 lc = self.count_data_lines(dataset)
1660 dataset.metadata.data_lines = lc
1661 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
1662 else:
1663 est_lines = self.estimate_file_lines(dataset)
1664 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
1665 inflector.cond_plural(est_lines, self.line_class) )
1666 else:
1667 dataset.blurb = "%s %s" % (
1668 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
1669 else:
1670 dataset.peek = 'file does not exist'
1671 dataset.blurb = 'file purged from disk'
1672
1673 def sniff(self, filename):
1674 """All msa Files simply put a '# .msa' in the first line. """
1675 try:
1676 f = open(filename, "r")
1677 firstline = f.readline().upper() # note we are uppercasing here to avoid CasE SenSitIVity
1678 f.close()
1679
1680 if "# .MSA" in firstline:
1681 return True
1682 else:
1683 return False
1684 except:
1685 traceback.print_exc(file=sys.stdout)
1686 return False
1687
1688 def split(cls, input_datasets, subdir_generator_function, split_params):
1689 """
1690 Split the input files by line.
1691 """
1692 if split_params is None:
1693 return
1694
1695 if len(input_datasets) > 1:
1696 raise Exception("Text file splitting does not support multiple files")
1697 input_files = [ds.file_name for ds in input_datasets]
1698
1699 lines_per_file = None
1700 chunk_size = None
1701 if split_params['split_mode'] == 'number_of_parts':
1702 lines_per_file = []
1703 # Computing the length is expensive!
1704 def _file_len(fname):
1705 i = 0
1706 f = open(fname)
1707 for i, l in enumerate(f):
1708 pass
1709 f.close()
1710 return i + 1
1711
1712 length = _file_len(input_files[0])
1713 parts = int(split_params['split_size'])
1714 if length < parts:
1715 parts = length
1716 len_each, remainder = divmod(length, parts)
1717 while length > 0:
1718 chunk = len_each
1719 if remainder > 0:
1720 chunk += 1
1721 lines_per_file.append(chunk)
1722 remainder = - 1
1723 length -= chunk
1724 elif split_params['split_mode'] == 'to_size':
1725 chunk_size = int(split_params['split_size'])
1726 else:
1727 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
1728
1729 f = open(input_files[0], 'rt')
1730 try:
1731 chunk_idx = 0
1732 file_done = False
1733 part_file = None
1734 while not file_done:
1735 if lines_per_file is None:
1736 this_chunk_size = chunk_size
1737 elif chunk_idx < len(lines_per_file):
1738 this_chunk_size = lines_per_file[chunk_idx]
1739 chunk_idx += 1
1740 lines_remaining = this_chunk_size
1741 part_file = None
1742 while lines_remaining > 0:
1743 a_line = f.readline()
1744 if a_line == '':
1745 file_done = True
1746 break
1747 if part_file is None:
1748 part_dir = subdir_generator_function()
1749 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
1750 part_file = open(part_path, 'w')
1751 part_file.write(a_line)
1752 lines_remaining -= 1
1753 if part_file is not None:
1754 part_file.close()
1755 except Exception, e:
1756 log.error('Unable to split files: %s' % str(e))
1757 f.close()
1758 if part_file is not None:
1759 part_file.close()
1760 raise
1761 f.close()
1762
1763 split = classmethod(split)
1764
1765
1766 class wurcs(data.Data):
1767 file_ext = 'wurcs'
1768 line_class = 'line'
1769
1770 """Add metadata elements"""
1771 MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True,
1772 visible=False, no_value=0)
1773
1774 def write_from_stream(self, dataset, stream):
1775 """Writes data from a stream"""
1776 # write it twice for now
1777 fd, temp_name = tempfile.mkstemp()
1778 while 1:
1779 chunk = stream.read(1048576)
1780 if not chunk:
1781 break
1782 os.write(fd, chunk)
1783 os.close(fd)
1784 # rewrite the file with unix newlines
1785 fp = open(dataset.file_name, 'wt')
1786 for line in file(temp_name, "U"):
1787 line = line.strip() + '\n'
1788 fp.write(line)
1789 fp.close()
1790
1791 def set_raw_data(self, dataset, data):
1792 """Saves the data on the disc"""
1793 fd, temp_name = tempfile.mkstemp()
1794 os.write(fd, data)
1795 os.close(fd)
1796 # rewrite the file with unix newlines
1797 fp = open(dataset.file_name, 'wt')
1798 for line in file(temp_name, "U"):
1799 line = line.strip() + '\n'
1800 fp.write(line)
1801 fp.close()
1802 os.remove(temp_name)
1803
1804 def get_mime(self):
1805 """Returns the mime type of the datatype"""
1806 return 'text/plain'
1807
1808 def set_meta(self, dataset, **kwd):
1809 """
1810 Set the number of lines of data in dataset.
1811 """
1812 dataset.metadata.data_lines = self.count_data_lines(dataset)
1813
1814 def estimate_file_lines(self, dataset):
1815 """
1816 Perform a rough estimate by extrapolating number of lines from a small read.
1817 """
1818 sample_size = 1048576
1819 dataset_fh = open(dataset.file_name)
1820 dataset_read = dataset_fh.read(sample_size)
1821 dataset_fh.close()
1822 sample_lines = dataset_read.count('\n')
1823 est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
1824 return est_lines
1825
1826 def count_data_lines(self, dataset):
1827 """
1828 Count the number of lines of data in dataset,
1829 skipping all blank lines and comments.
1830 """
1831 data_lines = 0
1832 for line in file(dataset.file_name):
1833 line = line.strip()
1834 if line and not line.startswith('#'):
1835 data_lines += 1
1836 return data_lines
1837
1838 def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=[]):
1839 """
1840 Set the peek. This method is used by various subclasses of Text.
1841 """
1842 if not dataset.dataset.purged:
1843 # The file must exist on disk for the get_file_peek() method
1844 dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH,
1845 skipchars=skipchars)
1846 if line_count is None:
1847 # See if line_count is stored in the metadata
1848 if dataset.metadata.data_lines:
1849 dataset.blurb = "%s %s" % (util.commaify(str(dataset.metadata.data_lines)),
1850 inflector.cond_plural(dataset.metadata.data_lines, self.line_class) )
1851 else:
1852 # Number of lines is not known ( this should not happen ), and auto-detect is
1853 # needed to set metadata
1854 # This can happen when the file is larger than max_optional_metadata_filesize.
1855 if int(dataset.get_size()) <= 1048576:
1856 # Small dataset, recount all lines and reset peek afterward.
1857 lc = self.count_data_lines(dataset)
1858 dataset.metadata.data_lines = lc
1859 dataset.blurb = "%s %s" % ( util.commaify(str(lc)), inflector.cond_plural(lc, self.line_class) )
1860 else:
1861 est_lines = self.estimate_file_lines(dataset)
1862 dataset.blurb = "~%s %s" % (util.commaify(util.roundify(str(est_lines))),
1863 inflector.cond_plural(est_lines, self.line_class) )
1864 else:
1865 dataset.blurb = "%s %s" % (
1866 util.commaify(str(line_count)), inflector.cond_plural(line_count, self.line_class) )
1867 else:
1868 dataset.peek = 'file does not exist'
1869 dataset.blurb = 'file purged from disk'
1870
1871 def sniff(self, filename):
1872 """All WURCS Files start with WURCS= then the version number. see http://www.wurcs-wg.org/definition.php and http://rings.t.soka.ac.jp/
1873 WURCS=2.0/4,3/[x2112h+1:x|1,5|2*NCC/3=O|4*OSO/3=O/3=O][12122a+1:b|1,5][12112h+1:b|1,5|2*NCC/3=O|6*OSO/3=O/3=O][12122a+1:b|1,5]1+3,2+1|2+4,3+1|3+3,4+1"""
1874 try:
1875 f = open(filename, "r")
1876 firstline = f.readline().upper() # note we are uppercasing here to avoid CasE SenSitIVity
1877 f.close()
1878 if "WURCS" in firstline:
1879 return True
1880 else:
1881 return False
1882 except:
1883 traceback.print_exc(file=sys.stdout)
1884 return False
1885
1886
1887 def split(cls, input_datasets, subdir_generator_function, split_params):
1888 """
1889 Split the input files by line.
1890 """
1891 if split_params is None:
1892 return
1893
1894 if len(input_datasets) > 1:
1895 raise Exception("Text file splitting does not support multiple files")
1896 input_files = [ds.file_name for ds in input_datasets]
1897
1898 lines_per_file = None
1899 chunk_size = None
1900 if split_params['split_mode'] == 'number_of_parts':
1901 lines_per_file = []
1902 # Computing the length is expensive!
1903 def _file_len(fname):
1904 i = 0
1905 f = open(fname)
1906 for i, l in enumerate(f):
1907 pass
1908 f.close()
1909 return i + 1
1910
1911 length = _file_len(input_files[0])
1912 parts = int(split_params['split_size'])
1913 if length < parts:
1914 parts = length
1915 len_each, remainder = divmod(length, parts)
1916 while length > 0:
1917 chunk = len_each
1918 if remainder > 0:
1919 chunk += 1
1920 lines_per_file.append(chunk)
1921 remainder = - 1
1922 length -= chunk
1923 elif split_params['split_mode'] == 'to_size':
1924 chunk_size = int(split_params['split_size'])
1925 else:
1926 raise Exception('Unsupported split mode %s' % split_params['split_mode'])
1927
1928 f = open(input_files[0], 'rt')
1929 try:
1930 chunk_idx = 0
1931 file_done = False
1932 part_file = None
1933 while not file_done:
1934 if lines_per_file is None:
1935 this_chunk_size = chunk_size
1936 elif chunk_idx < len(lines_per_file):
1937 this_chunk_size = lines_per_file[chunk_idx]
1938 chunk_idx += 1
1939 lines_remaining = this_chunk_size
1940 part_file = None
1941 while lines_remaining > 0:
1942 a_line = f.readline()
1943 if a_line == '':
1944 file_done = True
1945 break
1946 if part_file is None:
1947 part_dir = subdir_generator_function()
1948 part_path = os.path.join(part_dir, os.path.basename(input_files[0]))
1949 part_file = open(part_path, 'w')
1950 part_file.write(a_line)
1951 lines_remaining -= 1
1952 if part_file is not None:
1953 part_file.close()
1954 except Exception, e:
1955 log.error('Unable to split files: %s' % str(e))
1956 f.close()
1957 if part_file is not None:
1958 part_file.close()
1959 raise
1960 f.close()
1961
1962 split = classmethod(split)
1963
1964