Mercurial > repos > jjohnson > query_tabular

comparison query_tabular.xml @ 6:03842a4f71c6 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author jjohnson
date Fri, 17 Feb 2017 15:20:24 -0500
parents 19ae309ec53c
children ae27dab228b8
comparison
equal deleted inserted replaced
5:19ae309ec53c 6:03842a4f71c6
1 <tool id="query_tabular" name="Query Tabular" version="0.1.3"> 1 <tool id="query_tabular" name="Query Tabular" version="2.0.0">
2 <description>using sqlite sql</description> 2 <description>using sqlite sql</description>
3 3
4 <requirements> 4 <requirements>
5 </requirements> 5 </requirements>
6 <stdio> 6 <stdio>
7 <exit_code range="1:" /> 7 <exit_code range="1:" />
8 </stdio> 8 </stdio>
9 <command interpreter="python"><![CDATA[ 9 <command><![CDATA[
10 query_tabular.py 10 #if $add_to_database.withdb:
11 #if $save_db:
12 cp "$add_to_database.withdb" "$save_db" &&
13 #else:
14 cp "$add_to_database.withdb" "$workdb" &&
15 #end if
16 #end if
17 python $__tool_directory__/query_tabular.py
11 #if $save_db 18 #if $save_db
12 -s $sqlitedb 19 -s $sqlitedb
13 #else 20 #else
14 -s $workdb 21 -s $workdb
15 #end if 22 #end if
17 #if $sqlquery: 24 #if $sqlquery:
18 -Q "$query_file" 25 -Q "$query_file"
19 $no_header 26 $no_header
20 -o $output 27 -o $output
21 #end if 28 #end if
29 && cat $query_file
30 && cat $table_json
22 ]]></command> 31 ]]></command>
23 <configfiles> 32 <configfiles>
24 <configfile name="query_file"> 33 <configfile name="query_file">
25 $sqlquery 34 $sqlquery
26 </configfile> 35 </configfile>
51 #end if 60 #end if
52 #else 61 #else
53 #set $col_names = '' 62 #set $col_names = ''
54 #end if 63 #end if
55 #set $jtbl['column_names'] = $col_names 64 #set $jtbl['column_names'] = $col_names
56 #if str($tbl.tbl_opts.skip_lines) != '':
57 #set $jtbl['comment_lines'] = int($tbl.tbl_opts.skip_lines)
58 #elif $tbl.table.metadata.comment_lines and $tbl.table.metadata.comment_lines > 0:
59 #set $jtbl['comment_lines'] = int($tbl.table.metadata.comment_lines)
60 #end if
61 #set $idx_unique = [] 65 #set $idx_unique = []
62 #set $idx_non = [] 66 #set $idx_non = []
63 #for $idx in $tbl.tbl_opts.indexes: 67 #for $idx in $tbl.tbl_opts.indexes:
64 #if $idx.unique: 68 #if $idx.unique:
65 #silent $idx_unique.append(str($idx.index_columns)) 69 #silent $idx_unique.append(str($idx.index_columns))
71 #set $jtbl['unique'] = $idx_unique 75 #set $jtbl['unique'] = $idx_unique
72 #end if 76 #end if
73 #if len($idx_non) > 0: 77 #if len($idx_non) > 0:
74 #set $jtbl['index'] = $idx_non 78 #set $jtbl['index'] = $idx_non
75 #end if 79 #end if
80 #set $input_filters = []
81 #for $fi in $tbl.input_opts.linefilters:
82 #if $fi.filter.filter_type == 'skip':
83 #if str($tbl.tbl_opts.skip_lines) != '':
84 #set $jtbl['comment_lines'] = int($fi.filter.skip_lines)
85 #elif $tbl.table.metadata.comment_lines and $tbl.table.metadata.comment_lines > 0:
86 #set $jtbl['comment_lines'] = int($tbl.table.metadata.comment_lines)
87 #end if
88 #elif $fi.filter.filter_type == 'comment':
89 #set $jtbl['comment_char'] = str($fi.filter.comment_char)
90 #elif $fi.filter.filter_type == 'regex':
91 #set $filter_dict = dict()
92 #set $filter_dict['filter'] = str($fi.filter.filter_type)
93 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
94 #set $filter_dict['action'] = str($fi.filter.regex_action)
95 #silent $input_filters.append($filter_dict)
96 #elif $fi.filter.filter_type == 'replace':
97 #set $filter_dict = dict()
98 #set $filter_dict['filter'] = str($fi.filter.filter_type)
99 #set $filter_dict['column'] = int(str($fi.filter.column))
100 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
101 #set $filter_dict['replace'] = str($fi.filter.regex_replace)
102 #silent $input_filters.append($filter_dict)
103 ## #elif $fi.filter.filter_type == 'normalize':
104 ## #set $filter_dict = dict()
105 ## #set $filter_dict['filter'] = str($fi.filter.filter_type)
106 ## #set $filter_dict['columns'] = [int(str($ci)) for $ci in str($fi.filter.columns).split(',')]
107 ## #set $filter_dict['separator'] = str($fi.filter.separator)
108 ## #silent $input_filters.append($filter_dict)
109 #end if
110 #end for
111 #if $input_filters:
112 #set $jtbl['filters'] = $input_filters
113 #end if
76 #set $jtbls += [$jtbl] 114 #set $jtbls += [$jtbl]
77 #end for 115 #end for
78 #echo $json.dumps($jtbldef) 116 #echo $json.dumps($jtbldef)
79 </configfile> 117 </configfile>
80 </configfiles> 118 </configfiles>
81 <inputs> 119 <inputs>
82 <param name="workdb" type="hidden" value="workdb.sqlite" label=""/> 120 <param name="workdb" type="hidden" value="workdb.sqlite" label=""/>
121 <section name="add_to_database" expanded="false" title="Add tables to an existing database">
122 <param name="withdb" type="data" format="sqlite" optional="true" label="Add tables to this Database"
123 help="Make sure your added table names are not already in this database"/>
124 </section>
83 <repeat name="tables" title="Database Table" min="1"> 125 <repeat name="tables" title="Database Table" min="1">
84 <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/> 126 <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/>
127 <section name="input_opts" expanded="false" title="Filter Dataset Input">
128 <repeat name="linefilters" title="Filter Tabular Input Lines">
129 <conditional name="filter">
130 <param name="filter_type" type="select" label="Filter By">
131 <option value="skip">skip leading lines</option>
132 <option value="comment">comment char</option>
133 <option value="regex">by regex expression matching</option>
134 <option value="replace">regex replace value in column</option>
135 <!--
136 <option value="normalize">normalize list columns, replicates row for each item in list</option>
137 -->
138 </param>
139 <when value="skip">
140 <param name="skip_lines" type="integer" value="" min="0" label="Skip lines"
141 help="Leave blank to use the comment lines metadata for this dataset" />
142 </when>
143 <when value="comment">
144 <param name="comment_char" type="text" value="#" label="Comment line starting text">
145 </param>
146 </when>
147 <when value="regex">
148 <param name="regex_pattern" type="text" value="" label="regex pattern">
149 <sanitizer sanitize="False"/>
150 </param>
151 <param name="regex_action" type="select" label="action for regex match">
152 <option value="exclude_match">exclude line on pattern match</option>
153 <option value="include_match">include line on pattern match</option>
154 <option value="exclude_find">exclude line if pattern found</option>
155 <option value="include_find">include line if pattern found</option>
156 </param>
157 </when>
158 <when value="replace">
159 <param name="column" type="data_column" data_ref="table" label="Column to replace text"
160 help=""/>
161 <param name="regex_pattern" type="text" value="" label="regex pattern">
162 <sanitizer sanitize="False"/>
163 </param>
164 <param name="regex_replace" type="text" value="" label="replacement expression">
165 <sanitizer sanitize="False"/>
166 </param>
167 </when>
168 <!--
169 <when value="normalize">
170 <param name="columns" type="data_column" data_ref="table" multiple="True" label="Columns to split"
171 help=""/>
172 <param name="separator" type="text" value="," label="List item delimiter in column">
173 <sanitizer sanitize="False"/>
174 <validator type="regex" message="Anything but TAB or Newline">^[^\t\n\r\f\v]+$</validator>
175 </param>
176 </when>
177 -->
178 </conditional>
179 </repeat>
180 </section>
85 <section name="tbl_opts" expanded="false" title="Table Options"> 181 <section name="tbl_opts" expanded="false" title="Table Options">
86 <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table"> 182 <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table">
87 <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help> 183 <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help>
88 <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator> 184 <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator>
89 </param> 185 </param>
95 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/> 191 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/>
96 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name" 192 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name"
97 help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)"> 193 help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)">
98 <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator> 194 <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator>
99 </param> 195 </param>
100 <param name="skip_lines" type="integer" value="" min="0" optional="true" label="Skip lines" help="Leave blank to use the comment lines metadata for this dataset" />
101 <repeat name="indexes" title="Table Index"> 196 <repeat name="indexes" title="Table Index">
102 <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/> 197 <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/>
103 <param name="index_columns" type="text" value="" label="Index on Columns"> 198 <param name="index_columns" type="text" value="" label="Index on Columns">
104 <help>Create an index on the column names: e,g, c1 or c2,c4</help> 199 <help>Create an index on the column names: e,g, c1 or c2,c4</help>
105 <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator> 200 <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])(,([A-Za-z]\w*|"\S+.*"|`\S+[^,`]*`|[[]\S+[^,"]*[]])?)*$</validator>
106 </param> 201 </param>
107 </repeat> 202 </repeat>
108 </section> 203 </section>
109 </repeat> 204 </repeat>
110 <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"/> 205 <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"/>
111 <param name="sqlquery" type="text" area="true" size="10x80" value="" optional="true" label="SQL Query to generate tabular output"> 206 <param name="sqlquery" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to generate tabular output">
112 <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help> 207 <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help>
113 <sanitizer sanitize="False"/> 208 <sanitizer sanitize="False"/>
114 <validator type="regex" message="">^(?ims)\s*select\s+.*\s+from\s+.*$</validator> 209 <validator type="regex" message="">^(?ims)\s*select\s+.*\s+from\s+.*$</validator>
115 </param> 210 </param>
116 <param name="no_header" type="boolean" truevalue="-n" falsevalue="" checked="False" label="Omit column headers from tabular output"/> 211 <param name="no_header" type="boolean" truevalue="-n" falsevalue="" checked="False" label="Omit column headers from tabular output"/>
184 279
185 **Inputs** 280 **Inputs**
186 281
187 Loads tabular datasets into a SQLite_ data base. 282 Loads tabular datasets into a SQLite_ data base.
188 283
284 An existing SQLite_ data base can be used as input, and any selected tabular datasets will be added as new tables in that data base.
285
286
189 **Outputs** 287 **Outputs**
190 288
191 The results of a SQL query are output to the history as a tabular file. 289 The results of a SQL query are output to the history as a tabular file.
192 290
193 The SQLite_ data base can also be saved and output as a dataset in the history. 291 The SQLite_ data base can also be saved and output as a dataset in the history.
194 292
195 *(The* **SQLite to tabular** *tool can run additional queries on this database.)* 293 *(The* **SQLite to tabular** *tool can run additional queries on this database.)*
196
197 294
198 295
199 For help in using SQLite_ see: http://www.sqlite.org/docs.html 296 For help in using SQLite_ see: http://www.sqlite.org/docs.html
200 297
201 **NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30 298 **NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30