Mercurial > repos > iuc > sqlite_to_tabular

annotate load_db.py @ 0:859064f07be4 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
author iuc
date Tue, 18 Jul 2017 09:07:26 -0400
parents
children c1b700bc0150
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
1 #!/usr/bin/env python
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
2
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
3 from __future__ import print_function
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
4
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
5 import sys
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
6
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
7 from filters import TabularReader
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
8
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
9
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
10 def getValueType(val):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
11 if val or 0. == val:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
12 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
13 int(val)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
14 return 'INTEGER'
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
15 except:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
16 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
17 float(val)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
18 return 'REAL'
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
19 except:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
20 return 'TEXT'
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
21 return None
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
22
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
23
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
24 def get_column_def(file_path, table_name, skip=0, comment_char='#',
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
25 column_names=None, max_lines=100, load_named_columns=False,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
26 filters=None):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
27 col_pref = ['TEXT', 'REAL', 'INTEGER', None]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
28 col_types = []
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
29 col_idx = None
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
30 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
31 tr = TabularReader(file_path, skip=skip, comment_char=comment_char,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
32 col_idx=None, filters=filters)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
33 for linenum, fields in enumerate(tr):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
34 if linenum > max_lines:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
35 break
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
36 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
37 while len(col_types) < len(fields):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
38 col_types.append(None)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
39 for i, val in enumerate(fields):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
40 colType = getValueType(val)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
41 if col_pref.index(colType) < col_pref.index(col_types[i]):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
42 col_types[i] = colType
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
43 except Exception as e:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
44 print('Failed at line: %d err: %s' % (linenum, e),
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
45 file=sys.stderr)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
46 except Exception as e:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
47 print('Failed: %s' % (e), file=sys.stderr)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
48 for i, col_type in enumerate(col_types):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
49 if not col_type:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
50 col_types[i] = 'TEXT'
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
51 if column_names:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
52 col_names = []
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
53 if load_named_columns:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
54 col_idx = []
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
55 for i, cname in enumerate(
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
56 [cn.strip() for cn in column_names.split(',')]):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
57 if cname != '':
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
58 col_idx.append(i)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
59 col_names.append(cname)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
60 col_types = [col_types[i] for i in col_idx]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
61 else:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
62 col_names = ['c%d' % i for i in range(1, len(col_types) + 1)]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
63 for i, cname in enumerate(
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
64 [cn.strip() for cn in column_names.split(',')]):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
65 if cname and i < len(col_names):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
66 col_names[i] = cname
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
67 else:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
68 col_names = ['c%d' % i for i in range(1, len(col_types) + 1)]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
69 col_def = []
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
70 for i, col_name in enumerate(col_names):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
71 col_def.append('%s %s' % (col_names[i], col_types[i]))
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
72 return col_names, col_types, col_def, col_idx
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
73
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
74
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
75 def create_table(conn, file_path, table_name, skip=0, comment_char='#',
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
76 pkey_autoincr=None, column_names=None,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
77 load_named_columns=False, filters=None,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
78 unique_indexes=[], indexes=[]):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
79 col_names, col_types, col_def, col_idx = \
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
80 get_column_def(file_path, table_name, skip=skip,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
81 comment_char=comment_char, column_names=column_names,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
82 load_named_columns=load_named_columns, filters=filters)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
83 col_func = [float if t == 'REAL' else int
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
84 if t == 'INTEGER' else str for t in col_types]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
85 table_def = 'CREATE TABLE %s (\n %s%s\n);' % (
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
86 table_name,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
87 '%s INTEGER PRIMARY KEY AUTOINCREMENT,' %
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
88 pkey_autoincr if pkey_autoincr else '',
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
89 ', \n '.join(col_def))
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
90 # print >> sys.stdout, table_def
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
91 insert_stmt = 'INSERT INTO %s(%s) VALUES(%s)' % (
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
92 table_name, ','.join(col_names),
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
93 ','.join(["?" for x in col_names]))
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
94 # print >> sys.stdout, insert_stmt
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
95 data_lines = 0
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
96 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
97 c = conn.cursor()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
98 c.execute(table_def)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
99 conn.commit()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
100 c.close()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
101 for i, index in enumerate(unique_indexes):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
102 index_name = 'idx_uniq_%s_%d' % (table_name, i)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
103 index_columns = index.split(',')
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
104 create_index(conn, table_name, index_name, index_columns,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
105 unique=True)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
106 for i, index in enumerate(indexes):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
107 index_name = 'idx_%s_%d' % (table_name, i)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
108 index_columns = index.split(',')
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
109 create_index(conn, table_name, index_name, index_columns)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
110 c = conn.cursor()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
111 tr = TabularReader(file_path, skip=skip, comment_char=comment_char,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
112 col_idx=col_idx, filters=filters)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
113 for linenum, fields in enumerate(tr):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
114 data_lines += 1
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
115 try:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
116 vals = [col_func[i](x)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
117 if x else None for i, x in enumerate(fields)]
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
118 c.execute(insert_stmt, vals)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
119 except Exception as e:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
120 print('Failed at line: %d err: %s' % (linenum, e),
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
121 file=sys.stderr)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
122 conn.commit()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
123 c.close()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
124 except Exception as e:
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
125 exit('Error: %s' % (e))
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
126
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
127
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
128 def create_index(conn, table_name, index_name, index_columns, unique=False):
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
129 index_def = "CREATE %s INDEX %s on %s(%s)" % (
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
130 'UNIQUE' if unique else '', index_name,
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
131 table_name, ','.join(index_columns))
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
132 c = conn.cursor()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
133 c.execute(index_def)
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
134 conn.commit()
859064f07be4 planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/query_tabular commit 74915fc9cee746bbce1c4b507e13231259de177d
iuc
parents:
diff changeset
135 c.close()