# HG changeset patch # User mir-bioinf # Date 1429134699 14400 # Node ID 46c880ae6db2ed73ec67e36c4c9db63b3434ea82 # Parent 0aa0ebcd307c27271c36e863a2759647fa01889a Uploaded diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_left/multi_join_serial.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multi_join_left/multi_join_serial.xml Wed Apr 15 17:51:39 2015 -0400 @@ -0,0 +1,84 @@ + + tab delimited files serially + + + #for $j, $s in enumerate( $Files ) + #silent $j + #end for + + #for $i, $s in enumerate( $Files ) + run-multi_join_serial.pl --join_file $s.joinMe --join_col $s.joinCol --iteration $i --totalfiles $j --with_header $headerYes --resultsfile $Joined_all + ##print "loop iteration $i.\n"; + ; + #end for + + + + + + + + + + + + + + + + + + + + + + + + + + + + +This tool performs a left-outer join on multiple (at least two) files using a perl script that Ron wrote (thanks, Ron!). The resulting joined file will have the same number of rows as the first file chosen and subsequent files' matches will be shown if present. Rows in the first file without matches in the other files will have empty cells. If none of the input files have a header present, a simple column number header will be added to the output file to denote the start of each set of matches (from each file, start denoted by "C1"). + + +.. class:: warningmark + +This tool may fail due to the system running out of memory depending on the number and size of input files and number of matching lines. The higher all of these are, the more likely the tool is to fail. A red output dataset saying "Job killed" typically means the system ran into an out of memory error and as a result the job was killed. This issue has yet to be addressed at the moment... + + +**Steps:** + 1. Click Add new File for each tab-delimited file you'd like to add and the column you want to join on. + 2. After adding all files to join, select whether the headers should all be preserved (this should be Yes if all input datasets have headers). + 3. Click Execute. + +----- + +**Example** + +Dataset1:: + + chr1 10 20 geneA + chr1 50 80 geneB + chr5 10 40 geneL + +Dataset2:: + + geneA tumor-supressor + geneB Foxp2 + geneC Gnas1 + geneE INK4a + +Joining the 4th column of Dataset1 with the 1st column of Dataset2, no header, will yield:: + + C1 C2 C3 C4 C1 C2 + chr1 10 20 geneA geneA tumor-suppressor + chr1 50 80 geneB geneB Foxp2 + chr5 10 40 geneL + + + + + + + diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_left/run-multi_join_serial.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multi_join_left/run-multi_join_serial.pl Wed Apr 15 17:51:39 2015 -0400 @@ -0,0 +1,173 @@ +#!/usr/bin/perl + + +use Getopt::Long; +use Pod::Usage; +use IO::File; +use Data::Dumper; + +#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/multi_join_shell.pl'; ##comment this line out when finished testing +#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl'; +#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/GetOptWC.pm'; + +GetOptions( + "join_file=s" => \$data_in, + "join_col=s" => \$coljoin, + "time" => \$mTime, + "q|quiet" => \$quiet, + "iteration=i" => \$I, + "totalfiles=i" => \$N, + "with_header=s" => \$header_yes, + "input_name=s" => \$in_name, + "resultsfile=s" => \$out_file, +# "h|help" => \$help +) or pod2usage( -exitval => 2, -verbose => 2 ); + + +#check parameters and options +my $debug = scalar(@ARGV); + +$coljoin--; +#pod2usage(-msg => "To troubleshoot. ARGV should be @ARGV with $debug arguments in it."); +pod2usage(-msg => "Forward probability should be in [0, 1]!", -exitval => 2, -verbose => 2) if ($probF < 0 || $probF > 1); + +$N++; + +# # +use IO::Handle; +STDOUT->fdopen( \*OUTPUT, 'a' ) or die "cant open file $!\n"; #cms changing mode from 'w' to 'a' for multiple files in one run +STDERR->fdopen( \*ERROR, 'a' ) or die "cant open file $!\n"; #cms changing mode from 'w' to 'a' for multiple files in one run +# # # + +my @options; + +my $fileno = $I + 1; + +##Keeping track of the input files (one per iteration of this script) in an external file: +open $Filenames, '>>', "temp_filenames.txt" or die "cannot open the temporary file $!\n"; +print $Filenames "$data_in\t"; +print $Filenames "$coljoin\n"; + +if (($I==$N-1)&&($N>=2)) { + ## At the end of the last iteration + close($Filenames); + + + ##Read in file temp_filenames.txt + open(my $tmpfile, "<", "temp_filenames.txt") or die "Cannot open temp file: $!"; + my @fileArray = <$tmpfile>; + #unshift @fileArray,$conditions; ##don't need to do this since conditions aren't used here + close($tmpfile) or die "what is that??!!! $!"; + + + ##Need to send output file name to shell script: + push @fileArray, $out_file; ##adds out_file to the end of fileArray + ##Also need to send yes/no for keeping header: + push @fileArray, $header_yes; + + + ##@fileArray has one file per line,output,header_yes, so $N+1 rows + my $f=0; + my @first; + my @second; + do { + + @first = split('\t',$fileArray[$f]); ##was filename\tJoinCol + + ##CMS DEALING WITH HEADER OR NOT: + if ($header_yes eq "no") { + my $fh1; + $fh1 = IO::File->new("<$first[0]"); + my $line1file1 = $fh1->getline(); + $line1file1 =~ s/\s+$//; + @cols = split "\t",$line1file1; + my $numcols1 = @cols; + my $head1; + for (my $i=1; $i<$numcols1; $i++) { + $head1.="C$i\t"; + } + $head1.="C$numcols1\n"; + open(my $fh_sub, '>', './header1.txt') or die "OOPIES: $!\n"; + print $fh_sub $head1; + close $fh_sub; + system("cat $first[0] >> ./header1.txt"); ##put header in front of file + ##now want to use ./header1.txt instead of what was in $first[0] earlier + $first[0] = "./header1.txt"; + } + + + + @second = split('\t',$fileArray[$f+1]); + + if ($header_yes eq "no") { + my $fh2; + $fh2 = IO::File->new("<$second[0]"); + my $line1file2 = $fh2->getline(); + $line1file2 =~ s/\s+$//; + @cols = split "\t",$line1file2; + my $numcols2 = @cols; + my $head2; + for (my $i=1; $i<$numcols2; $i++) { + $head2.="C$i\t"; + } + $head2.="C$numcols2\n"; + open(my $fh_sub, '>', './header2.txt') or die "ERROR: $!\n"; + print $fh_sub $head2; + close $fh_sub; + system("cat $second[0] >> ./header2.txt"); + $second[0]="./header2.txt"; + } + + system("/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl", "-File1=$first[0]", "-File2=$second[0]", "-cola1=$first[1]", "-cola2=$second[1]", "-colb1=$first[1]", "-colb2=$second[1]"); + $f+=2; + system("mv file1_file2.txt joined.txt"); + if ($header_yes eq "no") { + system("rm ./header2.txt"); + system("rm ./header1.txt"); + } + } while ($f < 2); ##FIRST TWO ONLY!!! + + for ($f; $f<$N; $f++) { + my @current = split('\t',$fileArray[$f]); ##was filename\tJoinCol + + if ($header_yes eq "no") { + my $fh; + $fh = IO::File->new("<$current[0]"); + my $line1file = $fh->getline(); + $line1file =~ s/\s+$//; + @cols = split "\t",$line1file; + my $numcols = @cols; + my $head; + for (my $i=1; $i<$numcols; $i++) { + $head.="C$i\t"; + } + $head.="C$numcols\n"; + open(my $fh_sub, '>', './header.txt') or die "OOPIES: $!\n"; + print $fh_sub $head; + close $fh_sub; + system("cat $current[0] >> ./header.txt"); + $current[0]="./header.txt"; + } + + system("/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl","-File1=joined.txt", "-File2=$current[0]", "-cola1=$first[1]", "-cola2=$current[1]", "-colb1=$first[1]", "-colb2=$current[1]"); + system("mv file1_file2.txt joined.txt"); + if ($header_yes eq "no") { + system("rm ./header.txt"); + } + } + + system("mv joined.txt $fileArray[-2]"); + + ##NOT SURE WHAT TO DO WITH THIS FOR THE MULTI-JOIN TOOL: + ##Now, make the EC files from the genes-results files (extract appropriate columns): + #my $condStr = multi_join_shell(@fileArray); ##RSEMgetTPMs needs to take care of carriage returns + ##NEED TO MODIFY RSEMTOEBSEQ_SHELL SO IT TAKES THE OUTPUT FILENAME AS WELL + + system("rm temp_filenames.txt"); + +} +elsif ($N<2) { +} + + + diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_left/test-data/multi_join_serial_in1.tab --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multi_join_left/test-data/multi_join_serial_in1.tab Wed Apr 15 17:51:39 2015 -0400 @@ -0,0 +1,7 @@ +symbol sample1 sample2 sample3 +CSNK2A1P 12345 1.2345 8.90 +APOM 0.0 0.0 0.0 +HIST1H2AJ 89.5 75.6 32.3 +ASPHD1 0.001 6.98 2.33 +STBD1 0.2545 600.7 0.05 + diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_left/test-data/multi_join_serial_in2.tab --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multi_join_left/test-data/multi_join_serial_in2.tab Wed Apr 15 17:51:39 2015 -0400 @@ -0,0 +1,7 @@ +Gene Sample_4 Sample_5 Sample_6 Sample_7 +NOP10 11.1 22.2 33.3 0.0 +COX7B 0.05 0.0 6.75 0.0 +ASPHD1 10 0.075 3.57 1.14 +TSTD1 1.14 0.788 99.06 94.57 +APOM 123 123 123 123 + diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_left/test-data/multi_join_serial_in3.tab --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/multi_join_left/test-data/multi_join_serial_in3.tab Wed Apr 15 17:51:39 2015 -0400 @@ -0,0 +1,7 @@ +Experiment GENE SampleX SampleY SampleZ +1 HIST1H2AJ 0 0 0 +1 COX7B 1.1100 59.142 16602 +1 ZYX 1.04 1.04 1.04 +1 APOM 132 85 97 +2 PFDN6 0.06 1.87 99.96 +2 NPIPL3 1234 80.14 0.075 diff -r 0aa0ebcd307c -r 46c880ae6db2 multi_join_serial.xml --- a/multi_join_serial.xml Wed Apr 15 16:31:04 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,84 +0,0 @@ - - tab delimited files serially - - - #for $j, $s in enumerate( $Files ) - #silent $j - #end for - - #for $i, $s in enumerate( $Files ) - /opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/run-multi_join_serial.pl --join_file $s.joinMe --join_col $s.joinCol --iteration $i --totalfiles $j --with_header $headerYes --resultsfile $Joined_all - ##print "loop iteration $i.\n"; - ; - #end for - - - - - - - - - - - - - - - - - - - - - - - - - - - - -This tool performs a left-outer join on multiple (at least two) files using a perl script that Ron wrote (thanks, Ron!). The resulting joined file will have the same number of rows as the first file chosen and subsequent files' matches will be shown if present. Rows in the first file without matches in the other files will have empty cells. If none of the input files have a header present, a simple column number header will be added to the output file to denote the start of each set of matches (from each file, start denoted by "C1"). - - -.. class:: warningmark - -This tool may fail due to the system running out of memory depending on the number and size of input files and number of matching lines. The higher all of these are, the more likely the tool is to fail. A red output dataset saying "Job killed" typically means the system ran into an out of memory error and as a result the job was killed. This issue has yet to be addressed at the moment... - - -**Steps:** - 1. Click Add new File for each tab-delimited file you'd like to add and the column you want to join on. - 2. After adding all files to join, select whether the headers should all be preserved (this should be Yes if all input datasets have headers). - 3. Click Execute. - ------ - -**Example** - -Dataset1:: - - chr1 10 20 geneA - chr1 50 80 geneB - chr5 10 40 geneL - -Dataset2:: - - geneA tumor-supressor - geneB Foxp2 - geneC Gnas1 - geneE INK4a - -Joining the 4th column of Dataset1 with the 1st column of Dataset2, no header, will yield:: - - C1 C2 C3 C4 C1 C2 - chr1 10 20 geneA geneA tumor-suppressor - chr1 50 80 geneB geneB Foxp2 - chr5 10 40 geneL - - - - - - - diff -r 0aa0ebcd307c -r 46c880ae6db2 run-multi_join_serial.pl --- a/run-multi_join_serial.pl Wed Apr 15 16:31:04 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,173 +0,0 @@ -#!/usr/bin/perl - - -use Getopt::Long; -use Pod::Usage; -use IO::File; -use Data::Dumper; - -#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/multi_join_shell.pl'; ##comment this line out when finished testing -#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl'; -#require '/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/GetOptWC.pm'; - -GetOptions( - "join_file=s" => \$data_in, - "join_col=s" => \$coljoin, - "time" => \$mTime, - "q|quiet" => \$quiet, - "iteration=i" => \$I, - "totalfiles=i" => \$N, - "with_header=s" => \$header_yes, - "input_name=s" => \$in_name, - "resultsfile=s" => \$out_file, -# "h|help" => \$help -) or pod2usage( -exitval => 2, -verbose => 2 ); - - -#check parameters and options -my $debug = scalar(@ARGV); - -$coljoin--; -#pod2usage(-msg => "To troubleshoot. ARGV should be @ARGV with $debug arguments in it."); -pod2usage(-msg => "Forward probability should be in [0, 1]!", -exitval => 2, -verbose => 2) if ($probF < 0 || $probF > 1); - -$N++; - -# # -use IO::Handle; -STDOUT->fdopen( \*OUTPUT, 'a' ) or die "cant open file $!\n"; #cms changing mode from 'w' to 'a' for multiple files in one run -STDERR->fdopen( \*ERROR, 'a' ) or die "cant open file $!\n"; #cms changing mode from 'w' to 'a' for multiple files in one run -# # # - -my @options; - -my $fileno = $I + 1; - -##Keeping track of the input files (one per iteration of this script) in an external file: -open $Filenames, '>>', "temp_filenames.txt" or die "cannot open the temporary file $!\n"; -print $Filenames "$data_in\t"; -print $Filenames "$coljoin\n"; - -if (($I==$N-1)&&($N>=2)) { - ## At the end of the last iteration - close($Filenames); - - - ##Read in file temp_filenames.txt - open(my $tmpfile, "<", "temp_filenames.txt") or die "Cannot open temp file: $!"; - my @fileArray = <$tmpfile>; - #unshift @fileArray,$conditions; ##don't need to do this since conditions aren't used here - close($tmpfile) or die "what is that??!!! $!"; - - - ##Need to send output file name to shell script: - push @fileArray, $out_file; ##adds out_file to the end of fileArray - ##Also need to send yes/no for keeping header: - push @fileArray, $header_yes; - - - ##@fileArray has one file per line,output,header_yes, so $N+1 rows - my $f=0; - my @first; - my @second; - do { - - @first = split('\t',$fileArray[$f]); ##was filename\tJoinCol - - ##CMS DEALING WITH HEADER OR NOT: - if ($header_yes eq "no") { - my $fh1; - $fh1 = IO::File->new("<$first[0]"); - my $line1file1 = $fh1->getline(); - $line1file1 =~ s/\s+$//; - @cols = split "\t",$line1file1; - my $numcols1 = @cols; - my $head1; - for (my $i=1; $i<$numcols1; $i++) { - $head1.="C$i\t"; - } - $head1.="C$numcols1\n"; - open(my $fh_sub, '>', './header1.txt') or die "OOPIES: $!\n"; - print $fh_sub $head1; - close $fh_sub; - system("cat $first[0] >> ./header1.txt"); ##put header in front of file - ##now want to use ./header1.txt instead of what was in $first[0] earlier - $first[0] = "./header1.txt"; - } - - - - @second = split('\t',$fileArray[$f+1]); - - if ($header_yes eq "no") { - my $fh2; - $fh2 = IO::File->new("<$second[0]"); - my $line1file2 = $fh2->getline(); - $line1file2 =~ s/\s+$//; - @cols = split "\t",$line1file2; - my $numcols2 = @cols; - my $head2; - for (my $i=1; $i<$numcols2; $i++) { - $head2.="C$i\t"; - } - $head2.="C$numcols2\n"; - open(my $fh_sub, '>', './header2.txt') or die "ERROR: $!\n"; - print $fh_sub $head2; - close $fh_sub; - system("cat $second[0] >> ./header2.txt"); - $second[0]="./header2.txt"; - } - - system("/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl", "-File1=$first[0]", "-File2=$second[0]", "-cola1=$first[1]", "-cola2=$second[1]", "-colb1=$first[1]", "-colb2=$second[1]"); - $f+=2; - system("mv file1_file2.txt joined.txt"); - if ($header_yes eq "no") { - system("rm ./header2.txt"); - system("rm ./header1.txt"); - } - } while ($f < 2); ##FIRST TWO ONLY!!! - - for ($f; $f<$N; $f++) { - my @current = split('\t',$fileArray[$f]); ##was filename\tJoinCol - - if ($header_yes eq "no") { - my $fh; - $fh = IO::File->new("<$current[0]"); - my $line1file = $fh->getline(); - $line1file =~ s/\s+$//; - @cols = split "\t",$line1file; - my $numcols = @cols; - my $head; - for (my $i=1; $i<$numcols; $i++) { - $head.="C$i\t"; - } - $head.="C$numcols\n"; - open(my $fh_sub, '>', './header.txt') or die "OOPIES: $!\n"; - print $fh_sub $head; - close $fh_sub; - system("cat $current[0] >> ./header.txt"); - $current[0]="./header.txt"; - } - - system("/opt/galaxy/galaxy-dist/tools/ngs_rna/Unreleased/addColumnsFromFile2ToFile1.pl","-File1=joined.txt", "-File2=$current[0]", "-cola1=$first[1]", "-cola2=$current[1]", "-colb1=$first[1]", "-colb2=$current[1]"); - system("mv file1_file2.txt joined.txt"); - if ($header_yes eq "no") { - system("rm ./header.txt"); - } - } - - system("mv joined.txt $fileArray[-2]"); - - ##NOT SURE WHAT TO DO WITH THIS FOR THE MULTI-JOIN TOOL: - ##Now, make the EC files from the genes-results files (extract appropriate columns): - #my $condStr = multi_join_shell(@fileArray); ##RSEMgetTPMs needs to take care of carriage returns - ##NEED TO MODIFY RSEMTOEBSEQ_SHELL SO IT TAKES THE OUTPUT FILENAME AS WELL - - system("rm temp_filenames.txt"); - -} -elsif ($N<2) { -} - - - diff -r 0aa0ebcd307c -r 46c880ae6db2 test-data/multi_join_serial_in1.tab --- a/test-data/multi_join_serial_in1.tab Wed Apr 15 16:31:04 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -symbol sample1 sample2 sample3 -CSNK2A1P 12345 1.2345 8.90 -APOM 0.0 0.0 0.0 -HIST1H2AJ 89.5 75.6 32.3 -ASPHD1 0.001 6.98 2.33 -STBD1 0.2545 600.7 0.05 - diff -r 0aa0ebcd307c -r 46c880ae6db2 test-data/multi_join_serial_in2.tab --- a/test-data/multi_join_serial_in2.tab Wed Apr 15 16:31:04 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -Gene Sample_4 Sample_5 Sample_6 Sample_7 -NOP10 11.1 22.2 33.3 0.0 -COX7B 0.05 0.0 6.75 0.0 -ASPHD1 10 0.075 3.57 1.14 -TSTD1 1.14 0.788 99.06 94.57 -APOM 123 123 123 123 - diff -r 0aa0ebcd307c -r 46c880ae6db2 test-data/multi_join_serial_in3.tab --- a/test-data/multi_join_serial_in3.tab Wed Apr 15 16:31:04 2015 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -Experiment GENE SampleX SampleY SampleZ -1 HIST1H2AJ 0 0 0 -1 COX7B 1.1100 59.142 16602 -1 ZYX 1.04 1.04 1.04 -1 APOM 132 85 97 -2 PFDN6 0.06 1.87 99.96 -2 NPIPL3 1234 80.14 0.075