Mercurial > repos > kellrott > synapse_interface
changeset 5:f798da48a30d
Uploaded
| author | kellrott |
|---|---|
| date | Mon, 22 Jul 2013 17:43:48 -0400 |
| parents | 2925d82b84fc |
| children | 131300efb435 |
| files | synapse_interface/synapse_create.xml synapse_interface/synapse_download.xml synapse_interface/synapse_galaxy_client.py synapse_interface/synapse_query.xml synapse_interface/tool_dependencies.xml |
| diffstat | 5 files changed, 288 insertions(+), 232 deletions(-) [+] |
line wrap: on
line diff
--- a/synapse_interface/synapse_create.xml Thu Oct 25 02:39:06 2012 -0400 +++ b/synapse_interface/synapse_create.xml Mon Jul 22 17:43:48 2013 -0400 @@ -1,21 +1,59 @@ -<tool id="synapse_create" name="Synapse Create" version="2.0.0"> +<tool id="synapse_upload" name="Synapse Upload" version="3.0.0"> <description>Create Synapse</description> - <command interpreter="python">$script_file</command> + <requirements> + <requirement type="package" version="0.3.0">synapseclient</requirement> + </requirements> + <command interpreter="python">synapse_galaxy_client.py upload $script_file +$etype +$properties_file +$annotations_file +$outfile + +#if str($attachment) != '' +$attachment +#else +- +#end if + +#if str($attachment_name) != '' +$attachment_name +#else +- +#end if + + +#if str($synid) != '' +$synid +#else +- +#end if +</command> + <request_param_translation> + <request_param galaxy_name="sessionToken" remote_name="sessionToken" missing="" /> + </request_param_translation> <inputs> - <param name="user" type="text" size="90" label="Username"/> - <param name="pass" type="text" size="90" label="Password"/> - <param name="name" type="text" size="90" label="Name"/> + <param name="user" type="text" size="90" label="Username"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="pass" type="text" size="90" label="Password" help="If you have a sessionToken, you don't have to enter your password"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="sessionToken" type="text" size="90" label="SessionToken" help="Log in via OpenID using the Link in the help section"/> + <param name="name" type="text" size="90" label="Entity Name"/> + <param name="synid" type="text" size="30" label="Synapse ID" help="Leave Blank to create new Entity"/> <param name="attachment" type="data" optional="true" label="Attachment"/> <param name="attachment_name" type="text" size="90" optional="true" label="Attachment Name"/> <param name="etype" type="select" label="Entity Type"> - <option value="Project">Project</option> - <option value="Study">Study</option> - <option value="Data">Data</option> + <option value="Data">Data</option> + <option value="Folder">Folder</option> </param> + <param name="parentId" type="text" size="30" label="Parent ID"/> <repeat name="properties" title="Properties"> <param name="name" type="select" size="90" label="Name"> - <option value="name">Name</option> - <option value="parentId">ParentID</option> <option value="description">Description</option> <option value="species">Species</option> <option value="numSamples">Number of Samples</option> @@ -53,86 +91,37 @@ <data format="txt" name="outfile" /> </outputs> <configfiles> - <configfile name="script_file"><![CDATA[#!/usr/bin/env python -rcode=""" -require(synapseClient) - -galaxyClean <- function(x) { -y = gsub("__gt__", ">", x); -y = gsub("__lt__", "<", y ); -y = gsub("__sq__", "'", y); -y = gsub("__dq__", "\\"",y); -y = gsub("__ob__", "[", y); -y = gsub("__cb__", "]", y); -y = gsub("__oc__", "{", y); -y = gsub("__cc__", "}", y); -y = gsub("__at__", "@", y); -y = gsub("__cn__", "\\n", y); -y = gsub("__cr__", "\\r", y); -y = gsub("__tc__", "\\t", y); -y = gsub("__pd__", "#", y); -y = gsub("__at__", "@", y); -return(y); -} - -if (nchar("$user") && nchar("$pass")) { - synapseLogin(username=galaxyClean("$user"), password=galaxyClean("$pass")) -} - - -ent <- ${etype}( list(name=galaxyClean("${name}")) ) - -#for $a in $annotations: -annotValue(ent, galaxyClean("${a.name}")) <- galaxyClean("${a.value}") -#end for - + <configfile name="script_file">#if str($sessionToken) != '' +token +$sessionToken +#else +password +$user +$pass +#end if +</configfile> + <configfile name="properties_file">parentId $parentId #for a in $properties: -propertyValue(ent, galaxyClean("${a.name}")) <- galaxyClean("${a.value}") +${a.name} ${a.value} #end for - -ent <- createEntity(ent) - -#if $attachment != 'None' -#if $attachment_name == '': -ent <- addFile(ent, galaxyClean("${attachment}")) -#else -ent <- addFile(ent, galaxyClean("${attachment}"), galaxyClean("${attachment_name}")) +#if str($name) != '' +name $name #end if -#end if - -ent <- storeEntity(ent) - -write(properties(ent)\$id, file=galaxyClean("${outfile}")); - -""" - +</configfile> + <configfile name="annotations_file">#for a in $annotations: +${a.name} ${a.value} +#end for +</configfile> -import tempfile -import os -import sys -import subprocess - -h, path = tempfile.mkstemp() -os.write(h,rcode) -os.close(h) - -proc = subprocess.Popen( ["Rscript", path] + sys.argv[1:], stderr=subprocess.PIPE, stdout=subprocess.PIPE ) -(stdoutdata, stderrdata) = proc.communicate() -if proc.returncode: - sys.stderr.write(stderrdata) -sys.stdout.write(stdoutdata) -os.unlink(path) - -]]></configfile> </configfiles> <help> -Install R SynapseClient: - source('http://depot.sagebase.org/CRAN.R') - pkgInstall("synapseClient") +Login via OpenID |location_link|. -`To setup auto login (so you don't have to type in your password) <https://sagebionetworks.jira.com/wiki/display/SYNR/How+to+configure+automatic+login>`_ +.. |location_link| raw:: html + + <a href="https://synapse.sagebase.org/Portal/openid?OPEN_ID_PROVIDER=https://www.google.com/accounts/o8/id&RETURN_TO_URL=${host_url}?tool_id=synapse_upload&" target="_blank">at Synapse</a> </help> -</tool> \ No newline at end of file +</tool>
--- a/synapse_interface/synapse_download.xml Thu Oct 25 02:39:06 2012 -0400 +++ b/synapse_interface/synapse_download.xml Mon Jul 22 17:43:48 2013 -0400 @@ -1,92 +1,48 @@ -<tool id="synapse_download" name="Synapse Download" version="2.0.0"> +<tool id="synapse_download" name="Synapse Download" version="3.0.0"> <description>Download Synapse Entity</description> - <command interpreter="python">$script_file</command> + <requirements> + <requirement type="package" version="0.3.0">synapseclient</requirement> + </requirements> + <command interpreter="python">synapse_galaxy_client.py get $script_file $synid $outfile</command> + <request_param_translation> + <request_param galaxy_name="sessionToken" remote_name="sessionToken" missing="" /> + </request_param_translation> <inputs> - <param name="user" type="text" size="90" label="Username"/> - <param name="pass" type="text" size="90" label="Password"/> - <param name="synid" type="text" label="Synapse ID"/> - <param name="data_type" type="select" label="Data Type"> - <option value="e">Affy Eset</option> - <option value="a">Attachment</option> - </param> + <param name="user" type="text" size="90" label="Username"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="pass" type="text" size="90" label="Password" help="If you have a sessionToken, you don't have to enter your password"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="sessionToken" type="text" size="90" label="SessionToken" help="Log in via OpenID using the Link in the help section"/> + + <param name="synid" type="text" size="30" label="Synapse ID"/> </inputs> <outputs> <data format="txt" name="outfile" label="${synid}" /> </outputs> <configfiles> - <configfile name="script_file"><![CDATA[#!/usr/bin/env python -rcode = """ -library(affy) -require(synapseClient) - -galaxyClean <- function(x) { -y = gsub("__gt__", ">", x); -y = gsub("__lt__", "<", y ); -y = gsub("__sq__", "'", y); -y = gsub("__dq__", "\\"",y); -y = gsub("__ob__", "[", y); -y = gsub("__cb__", "]", y); -y = gsub("__oc__", "{", y); -y = gsub("__cc__", "}", y); -y = gsub("__at__", "@", y); -y = gsub("__cn__", "\\n", y); -y = gsub("__cr__", "\\r", y); -y = gsub("__tc__", "\\t", y); -y = gsub("__pd__", "#", y); -y = gsub("__at__", "@", y); -return(y); -} - -if (nchar("$user") && nchar("$pass")) { - synapseLogin(username=galaxyClean("$user"), password=galaxyClean("$pass")) -} - - -cat("Loading...") -ent <- loadEntity("$synid"); -cat("Done\n") - -if ("e" == "$data_type") { - eset <- exprs(ent\$objects\$eset); - write(paste(c("PROBE", colnames(eset)), collapse="\\t"), file="$outfile") - write.table(eset, "$outfile", col.names=FALSE, sep="\\t", quote=FALSE, append=TRUE); -} -if ("a" == "$data_type") { - fpath <- file.path(ent\$cacheDir, ent\$files[[1]]); - file.copy(fpath, "$outfile", overwrite=T); -} - -""" - -import tempfile -import os -import sys -import subprocess - -h, path = tempfile.mkstemp() -os.write(h,rcode) -os.close(h) - -cmd_args = ["R", "CMD", "BATCH", path ] -proc = subprocess.Popen(cmd_args, stderr=subprocess.PIPE) -stdout, stderr = proc.communicate() -if proc.poll() != 0: - sys.stderr.write(stderr) -os.unlink(path) -]]></configfile> + <configfile name="script_file">#if str($sessionToken) != '' +token +$sessionToken +#else +password +$user +$pass +#end if +</configfile> </configfiles> <help> -This tool current supports two types of data from Synapse: Affy eset data and attachment files. -For Affy eset data, it will attempt to turn the eset into a table and write a tab separated file. For attachment files, it will -save the first attached file as the output. +Login via OpenID |location_link|. -Install R SynapseClient:: +.. |location_link| raw:: html - source('http://depot.sagebase.org/CRAN.R') - pkgInstall("synapseClient") - -`To setup auto login (so you don't have to type in your password) <https://sagebionetworks.jira.com/wiki/display/SYNR/How+to+configure+automatic+login>`_ + <a href="https://synapse.sagebase.org/Portal/openid?OPEN_ID_PROVIDER=https://www.google.com/accounts/o8/id&RETURN_TO_URL=${host_url}?tool_id=synapse_download&" target="_blank">at Synapse</a> </help> -</tool> \ No newline at end of file +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/synapse_interface/synapse_galaxy_client.py Mon Jul 22 17:43:48 2013 -0400 @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +import synapseclient +import sys +import json +import os +import shutil +import csv + +def dict_list_to_tsv(data, handle): + headers = {} + for row in data: + for col in row: + if col not in headers: + headers[col] = len(headers) + + head = headers.keys() + head.sort(key=lambda x : headers[x]) + + writer = csv.writer(handle, delimiter="\t", lineterminator="\n") + writer.writerow(head) + + for row in data: + out = [] + for c in head: + out.append(row.get(c, "")) + writer.writerow(out) + + +if __name__ == "__main__": + cmd = sys.argv[1] + info_file = sys.argv[2] + + handle = open(info_file) + mode = handle.readline().rstrip("\n\r") + syn = None + if mode == 'password': + username = handle.readline().rstrip("\n\r") + password = handle.readline().rstrip("\n\r") + syn = synapseclient.Synapse() + syn.login(username, password) + elif mode == 'token': + token = handle.readline().rstrip("\n\r") + syn = synapseclient.Synapse() + syn.login(sessionToken=token) + handle.close() + + if syn is None: + sys.stderr.write("No login info\n") + sys.exit(1) + + if cmd == "query": + querypath = sys.argv[3] + outpath = sys.argv[4] + handle = open(querypath) + query = handle.read() + handle.close() + ohandle = open(outpath, "w") + dict_list_to_tsv(syn.query(query)['results'], ohandle) + ohandle.close() + + + if cmd == "get": + synid = sys.argv[3] + outpath = sys.argv[4] + + ent = syn.downloadEntity(synid) + src_path = os.path.join(ent['cacheDir'], ent['files'][0]) + shutil.copy(src_path, outpath) + + if cmd == "upload": + etype = sys.argv[3] + properties_file = sys.argv[4] + annotations_file = sys.argv[5] + outfile = sys.argv[6] + attach_file = sys.argv[7] + attach_file_name = os.path.basename(sys.argv[8]) + synid = sys.argv[9] + + handle = open(properties_file) + props = {} + for line in handle: + tmp = line.rstrip("\r\n").split("\t") + props[tmp[0]] = tmp[1] + handle.close() + + handle = open(annotations_file) + annon = {} + for line in handle: + tmp = line.rstrip("\r\n").split("\t") + if tmp[0] in annon: + annon[tmp[0]].append( tmp[1] ) + else: + annon[tmp[0]] = [tmp[1]] + handle.close() + + + if synid is not None and synid.startswith("syn"): + entity = syn.getEntity(synid) + else: + entityData = { u'entityType': u'org.sagebionetworks.repo.model.Data' } + if etype == 'folder': + entityData = { u'entityType': u'org.sagebionetworks.repo.model.Folder' } + entityData['name'] = attach_file_name + entityData['parentId'] = props['parentId'] + entity = syn.createEntity(entityData) + synid = entity['id'] + + if len(props): + for p in props: + if p != 'parentId': + entity[p] = props[p] + entity = syn.updateEntity(entity) + + if len(annon): + ann = syn.getAnnotations(entity) + for a in annon: + ann[a] = annon[a] + syn.setAnnotations(entity, ann) + + if attach_file != '-' and attach_file_name != '-': + if not os.path.exists(attach_file_name): + os.symlink(attach_file, attach_file_name) + attach_file = attach_file_name + print "upload", attach_file + entity = syn.getEntity(synid) + syn.uploadFile(entity, attach_file) + + handle = open(outfile, "w") + handle.write(json.dumps(dict(entity))) + handle.close() + +
--- a/synapse_interface/synapse_query.xml Thu Oct 25 02:39:06 2012 -0400 +++ b/synapse_interface/synapse_query.xml Mon Jul 22 17:43:48 2013 -0400 @@ -1,89 +1,54 @@ -<tool id="synapse_query" name="Synapse Query" version="1.0"> +<tool id="synapse_query" name="Synapse Query" version="3.0.0"> <description>Query Synapse</description> - <command interpreter="python">$script_file</command> + <requirements> + <requirement type="package" version="0.3.0">synapseclient</requirement> + </requirements> + <command interpreter="python">synapse_galaxy_client.py query $script_file $query_file $outfile</command> + <request_param_translation> + <request_param galaxy_name="sessionToken" remote_name="sessionToken" missing="" /> + </request_param_translation> <inputs> - <param name="user" type="text" size="90" label="Username"/> - <param name="pass" type="text" size="90" label="Password"/> - <param name="query" type="text" area="True" size="5x35" label="Query"/> - - <!-- + <param name="user" type="text" size="90" label="Username"> <sanitizer> - <valid initial="string.printable"> - <remove value="""/> - </valid> - <mapping initial="none"> - <add source=""" target="'"/> - </mapping> + <valid initial="string.printable"/> </sanitizer> </param> - --> + <param name="pass" type="text" size="90" label="Password" help="If you have a sessionToken, you don't have to enter your password"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="sessionToken" type="text" size="90" label="SessionToken" help="Log in via OpenID using the Link in the help section"/> + + <param name="query" type="text" area="True" size="5x35" label="Query"> + <sanitizer> + <valid initial="string.printable"/> + </sanitizer> + </param> + <param name="stall" type="hidden" required="True"/> </inputs> <outputs> <data format="tabular" name="outfile" /> </outputs> <configfiles> - <configfile name="script_file"><![CDATA[#!/usr/bin/env python -rcode=""" -library(affy) -require(synapseClient) - -galaxyClean <- function(x) { -y = gsub("__gt__", ">", x); -y = gsub("__lt__", "<", y ); -y = gsub("__sq__", "'", y); -y = gsub("__dq__", "\\"",y); -y = gsub("__ob__", "[", y); -y = gsub("__cb__", "]", y); -y = gsub("__oc__", "{", y); -y = gsub("__cc__", "}", y); -y = gsub("__at__", "@", y); -y = gsub("__cn__", "\\n", y); -y = gsub("__cr__", "\\r", y); -y = gsub("__tc__", "\\t", y); -y = gsub("__pd__", "#", y); -y = gsub("__at__", "@", y); -return(y); -} - -if (nchar("$user") && nchar("$pass")) { - synapseLogin(username=galaxyClean("$user"), password=galaxyClean("$pass")) -} - - - -query <- galaxyClean("${query}"); -cat(query) - -query.out <- synapseQuery(query); -write.table(query.out, quote=F, file="$outfile", row.name=F, sep="\\t") - -""" -import tempfile -import os -import sys -import subprocess - -h, path = tempfile.mkstemp(dir="/tmp") -os.write(h,rcode) -os.close(h) - -proc = subprocess.Popen( ["Rscript", path], stderr=subprocess.PIPE, stdout=subprocess.PIPE ) -(stdoutdata, stderrdata) = proc.communicate() -if proc.returncode: - sys.stderr.write(stderrdata) -sys.stdout.write(stdoutdata) -os.unlink(path) - -]]></configfile> + <configfile name="script_file">#if str($sessionToken) != '' +token +$sessionToken +#else +password +$user +$pass +#end if +</configfile> + <configfile name="query_file">$query</configfile> </configfiles> <help> -Install R SynapseClient:: +Login via OpenID |location_link|. - source('http://depot.sagebase.org/CRAN.R') - pkgInstall("synapseClient") +.. |location_link| raw:: html -`To setup auto login (so you don't have to type in your password) <https://sagebionetworks.jira.com/wiki/display/SYNR/How+to+configure+automatic+login>`_ + <a href="https://synapse.sagebase.org/Portal/openid?OPEN_ID_PROVIDER=https://www.google.com/accounts/o8/id&RETURN_TO_URL=${host_url}?tool_id=synapse_query&" target="_blank">at Synapse</a> Example Query: @@ -91,4 +56,4 @@ </help> -</tool> \ No newline at end of file +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/synapse_interface/tool_dependencies.xml Mon Jul 22 17:43:48 2013 -0400 @@ -0,0 +1,13 @@ +<?xml version="1.0"?> +<tool_dependency> + <package name="synapseclient" version="0.3.0"> + <install version="1.0"> + <actions> + <action type="setup_virtualenv">synapseclient == 0.3.0</action> + </actions> + </install> + <readme> +A client for Synapse, a collaborative compute space that allows scientists to share and analyze data together. Synapse brings together scientific data, tools, and disease models into a commons that enables true collaborative research. The platform consists of a web portal, web services, and integration with data analysis tools such as R, python, Galaxy and Java. + </readme> + </package> +</tool_dependency> \ No newline at end of file