Mercurial > repos > jjohnson > query_tabular
comparison query_tabular.xml @ 6:03842a4f71c6 draft
Uploaded
author | jjohnson |
---|---|
date | Fri, 17 Feb 2017 15:20:24 -0500 |
parents | 19ae309ec53c |
children | ae27dab228b8 |
comparison
equal
deleted
inserted
replaced
5:19ae309ec53c | 6:03842a4f71c6 |
---|---|
1 <tool id="query_tabular" name="Query Tabular" version="0.1.3"> | 1 <tool id="query_tabular" name="Query Tabular" version="2.0.0"> |
2 <description>using sqlite sql</description> | 2 <description>using sqlite sql</description> |
3 | 3 |
4 <requirements> | 4 <requirements> |
5 </requirements> | 5 </requirements> |
6 <stdio> | 6 <stdio> |
7 <exit_code range="1:" /> | 7 <exit_code range="1:" /> |
8 </stdio> | 8 </stdio> |
9 <command interpreter="python"><![CDATA[ | 9 <command><![CDATA[ |
10 query_tabular.py | 10 #if $add_to_database.withdb: |
11 #if $save_db: | |
12 cp "$add_to_database.withdb" "$save_db" && | |
13 #else: | |
14 cp "$add_to_database.withdb" "$workdb" && | |
15 #end if | |
16 #end if | |
17 python $__tool_directory__/query_tabular.py | |
11 #if $save_db | 18 #if $save_db |
12 -s $sqlitedb | 19 -s $sqlitedb |
13 #else | 20 #else |
14 -s $workdb | 21 -s $workdb |
15 #end if | 22 #end if |
17 #if $sqlquery: | 24 #if $sqlquery: |
18 -Q "$query_file" | 25 -Q "$query_file" |
19 $no_header | 26 $no_header |
20 -o $output | 27 -o $output |
21 #end if | 28 #end if |
29 && cat $query_file | |
30 && cat $table_json | |
22 ]]></command> | 31 ]]></command> |
23 <configfiles> | 32 <configfiles> |
24 <configfile name="query_file"> | 33 <configfile name="query_file"> |
25 $sqlquery | 34 $sqlquery |
26 </configfile> | 35 </configfile> |
51 #end if | 60 #end if |
52 #else | 61 #else |
53 #set $col_names = '' | 62 #set $col_names = '' |
54 #end if | 63 #end if |
55 #set $jtbl['column_names'] = $col_names | 64 #set $jtbl['column_names'] = $col_names |
56 #if str($tbl.tbl_opts.skip_lines) != '': | |
57 #set $jtbl['comment_lines'] = int($tbl.tbl_opts.skip_lines) | |
58 #elif $tbl.table.metadata.comment_lines and $tbl.table.metadata.comment_lines > 0: | |
59 #set $jtbl['comment_lines'] = int($tbl.table.metadata.comment_lines) | |
60 #end if | |
61 #set $idx_unique = [] | 65 #set $idx_unique = [] |
62 #set $idx_non = [] | 66 #set $idx_non = [] |
63 #for $idx in $tbl.tbl_opts.indexes: | 67 #for $idx in $tbl.tbl_opts.indexes: |
64 #if $idx.unique: | 68 #if $idx.unique: |
65 #silent $idx_unique.append(str($idx.index_columns)) | 69 #silent $idx_unique.append(str($idx.index_columns)) |
71 #set $jtbl['unique'] = $idx_unique | 75 #set $jtbl['unique'] = $idx_unique |
72 #end if | 76 #end if |
73 #if len($idx_non) > 0: | 77 #if len($idx_non) > 0: |
74 #set $jtbl['index'] = $idx_non | 78 #set $jtbl['index'] = $idx_non |
75 #end if | 79 #end if |
80 #set $input_filters = [] | |
81 #for $fi in $tbl.input_opts.linefilters: | |
82 #if $fi.filter.filter_type == 'skip': | |
83 #if str($tbl.tbl_opts.skip_lines) != '': | |
84 #set $jtbl['comment_lines'] = int($fi.filter.skip_lines) | |
85 #elif $tbl.table.metadata.comment_lines and $tbl.table.metadata.comment_lines > 0: | |
86 #set $jtbl['comment_lines'] = int($tbl.table.metadata.comment_lines) | |
87 #end if | |
88 #elif $fi.filter.filter_type == 'comment': | |
89 #set $jtbl['comment_char'] = str($fi.filter.comment_char) | |
90 #elif $fi.filter.filter_type == 'regex': | |
91 #set $filter_dict = dict() | |
92 #set $filter_dict['filter'] = str($fi.filter.filter_type) | |
93 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern) | |
94 #set $filter_dict['action'] = str($fi.filter.regex_action) | |
95 #silent $input_filters.append($filter_dict) | |
96 #elif $fi.filter.filter_type == 'replace': | |
97 #set $filter_dict = dict() | |
98 #set $filter_dict['filter'] = str($fi.filter.filter_type) | |
99 #set $filter_dict['column'] = int(str($fi.filter.column)) | |
100 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern) | |
101 #set $filter_dict['replace'] = str($fi.filter.regex_replace) | |
102 #silent $input_filters.append($filter_dict) | |
103 ## #elif $fi.filter.filter_type == 'normalize': | |
104 ## #set $filter_dict = dict() | |
105 ## #set $filter_dict['filter'] = str($fi.filter.filter_type) | |
106 ## #set $filter_dict['columns'] = [int(str($ci)) for $ci in str($fi.filter.columns).split(',')] | |
107 ## #set $filter_dict['separator'] = str($fi.filter.separator) | |
108 ## #silent $input_filters.append($filter_dict) | |
109 #end if | |
110 #end for | |
111 #if $input_filters: | |
112 #set $jtbl['filters'] = $input_filters | |
113 #end if | |
76 #set $jtbls += [$jtbl] | 114 #set $jtbls += [$jtbl] |
77 #end for | 115 #end for |
78 #echo $json.dumps($jtbldef) | 116 #echo $json.dumps($jtbldef) |
79 </configfile> | 117 </configfile> |
80 </configfiles> | 118 </configfiles> |
81 <inputs> | 119 <inputs> |
82 <param name="workdb" type="hidden" value="workdb.sqlite" label=""/> | 120 <param name="workdb" type="hidden" value="workdb.sqlite" label=""/> |
121 <section name="add_to_database" expanded="false" title="Add tables to an existing database"> | |
122 <param name="withdb" type="data" format="sqlite" optional="true" label="Add tables to this Database" | |
123 help="Make sure your added table names are not already in this database"/> | |
124 </section> | |
83 <repeat name="tables" title="Database Table" min="1"> | 125 <repeat name="tables" title="Database Table" min="1"> |
84 <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/> | 126 <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/> |
127 <section name="input_opts" expanded="false" title="Filter Dataset Input"> | |
128 <repeat name="linefilters" title="Filter Tabular Input Lines"> | |
129 <conditional name="filter"> | |
130 <param name="filter_type" type="select" label="Filter By"> | |
131 <option value="skip">skip leading lines</option> | |
132 <option value="comment">comment char</option> | |
133 <option value="regex">by regex expression matching</option> | |
134 <option value="replace">regex replace value in column</option> | |
135 <!-- | |
136 <option value="normalize">normalize list columns, replicates row for each item in list</option> | |
137 --> | |
138 </param> | |
139 <when value="skip"> | |
140 <param name="skip_lines" type="integer" value="" min="0" label="Skip lines" | |
141 help="Leave blank to use the comment lines metadata for this dataset" /> | |
142 </when> | |
143 <when value="comment"> | |
144 <param name="comment_char" type="text" value="#" label="Comment line starting text"> | |
145 </param> | |
146 </when> | |
147 <when value="regex"> | |
148 <param name="regex_pattern" type="text" value="" label="regex pattern"> | |
149 <sanitizer sanitize="False"/> | |
150 </param> | |
151 <param name="regex_action" type="select" label="action for regex match"> | |
152 <option value="exclude_match">exclude line on pattern match</option> | |
153 <option value="include_match">include line on pattern match</option> | |
154 <option value="exclude_find">exclude line if pattern found</option> | |
155 <option value="include_find">include line if pattern found</option> | |
156 </param> | |
157 </when> | |
158 <when value="replace"> | |
159 <param name="column" type="data_column" data_ref="table" label="Column to replace text" | |
160 help=""/> | |
161 <param name="regex_pattern" type="text" value="" label="regex pattern"> | |
162 <sanitizer sanitize="False"/> | |
163 </param> | |
164 <param name="regex_replace" type="text" value="" label="replacement expression"> | |
165 <sanitizer sanitize="False"/> | |
166 </param> | |
167 </when> | |
168 <!-- | |
169 <when value="normalize"> | |
170 <param name="columns" type="data_column" data_ref="table" multiple="True" label="Columns to split" | |
171 help=""/> | |
172 <param name="separator" type="text" value="," label="List item delimiter in column"> | |
173 <sanitizer sanitize="False"/> | |
174 <validator type="regex" message="Anything but TAB or Newline">^[^\t\n\r\f\v]+$</validator> | |
175 </param> | |
176 </when> | |
177 --> | |
178 </conditional> | |
179 </repeat> | |
180 </section> | |
85 <section name="tbl_opts" expanded="false" title="Table Options"> | 181 <section name="tbl_opts" expanded="false" title="Table Options"> |
86 <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table"> | 182 <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table"> |
87 <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help> | 183 <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help> |
88 <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator> | 184 <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator> |
89 </param> | 185 </param> |
95 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/> | 191 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/> |
96 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name" | 192 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name" |
97 help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)"> | 193 help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)"> |
98 <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator> | 194 <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator> |
99 </param> | 195 </param> |
100 <param name="skip_lines" type="integer" value="" min="0" optional="true" label="Skip lines" help="Leave blank to use the comment lines metadata for this dataset" /> | |
101 <repeat name="indexes" title="Table Index"> | 196 <repeat name="indexes" title="Table Index"> |
102 <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/> | 197 <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/> |
103 <param name="index_columns" type="text" value="" label="Index on Columns"> | 198 <param name="index_columns" type="text" value="" label="Index on Columns"> |
104 <help>Create an index on the column names: e,g, c1 or c2,c4</help> | 199 <help>Create an index on the column names: e,g, c1 or c2,c4</help> |
105 <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator> | 200 <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator> |
106 </param> | 201 </param> |
107 </repeat> | 202 </repeat> |
108 </section> | 203 </section> |
109 </repeat> | 204 </repeat> |
110 <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"/> | 205 <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"/> |
111 <param name="sqlquery" type="text" area="true" size="10x80" value="" optional="true" label="SQL Query to generate tabular output"> | 206 <param name="sqlquery" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to generate tabular output"> |
112 <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help> | 207 <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help> |
113 <sanitizer sanitize="False"/> | 208 <sanitizer sanitize="False"/> |
114 <validator type="regex" message="">^(?ims)\s*select\s+.*\s+from\s+.*$</validator> | 209 <validator type="regex" message="">^(?ims)\s*select\s+.*\s+from\s+.*$</validator> |
115 </param> | 210 </param> |
116 <param name="no_header" type="boolean" truevalue="-n" falsevalue="" checked="False" label="Omit column headers from tabular output"/> | 211 <param name="no_header" type="boolean" truevalue="-n" falsevalue="" checked="False" label="Omit column headers from tabular output"/> |
184 | 279 |
185 **Inputs** | 280 **Inputs** |
186 | 281 |
187 Loads tabular datasets into a SQLite_ data base. | 282 Loads tabular datasets into a SQLite_ data base. |
188 | 283 |
284 An existing SQLite_ data base can be used as input, and any selected tabular datasets will be added as new tables in that data base. | |
285 | |
286 | |
189 **Outputs** | 287 **Outputs** |
190 | 288 |
191 The results of a SQL query are output to the history as a tabular file. | 289 The results of a SQL query are output to the history as a tabular file. |
192 | 290 |
193 The SQLite_ data base can also be saved and output as a dataset in the history. | 291 The SQLite_ data base can also be saved and output as a dataset in the history. |
194 | 292 |
195 *(The* **SQLite to tabular** *tool can run additional queries on this database.)* | 293 *(The* **SQLite to tabular** *tool can run additional queries on this database.)* |
196 | |
197 | 294 |
198 | 295 |
199 For help in using SQLite_ see: http://www.sqlite.org/docs.html | 296 For help in using SQLite_ see: http://www.sqlite.org/docs.html |
200 | 297 |
201 **NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30 | 298 **NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30 |