# HG changeset patch # User crs4 # Date 1395420975 14400 # Node ID ebb02ba5987c0a6577e5c899beca6cad93ea812c # Parent 6820983ba5d5218f8df18105c1c9c78b44c98bb5 Rewrite of param handling. interPairEnd param moved to "paired" section. Add param for '-a' option. Remove basic parallelism tag, which does not work with multiple inputs (thanks Bjoern Gruening for the notice). Simplify Python code. diff -r 6820983ba5d5 -r ebb02ba5987c bwa_mem.py --- a/bwa_mem.py Tue Mar 18 07:49:22 2014 -0400 +++ b/bwa_mem.py Fri Mar 21 12:56:15 2014 -0400 @@ -18,25 +18,6 @@ import optparse, os, shutil, subprocess, sys, tempfile -def stop_err( msg ): - sys.stderr.write( '%s\n' % msg ) - sys.exit() - -def check_is_double_encoded( fastq ): - # check that first read is bases, not one base followed by numbers - bases = [ 'A', 'C', 'G', 'T', 'a', 'c', 'g', 't', 'N' ] - nums = [ '0', '1', '2', '3' ] - for line in file( fastq, 'rb'): - if not line.strip() or line.startswith( '@' ): - continue - if len( [ b for b in line.strip() if b in nums ] ) > 0: - return False - elif line.strip()[0] in bases and len( [ b for b in line.strip() if b in bases ] ) == len( line.strip() ): - return True - else: - raise Exception, 'First line in first read does not appear to be a valid FASTQ read in either base-space or color-space' - raise Exception, 'There is no non-comment and non-blank line in your FASTQ file' - def __main__(): descr = "bwa_mem.py: Map (long length) reads against a reference genome with BWA-MEM." parser = optparse.OptionParser(description=descr) @@ -50,20 +31,20 @@ parser.add_option( '-s', '--fileSource', help='Whether to use a previously indexed reference sequence or one form history (indexed or history)' ) parser.add_option( '-D', '--dbkey', help='Dbkey for reference genome' ) - parser.add_option( '-k', '--minEditDistSeed', default=19, type=int, help='Minimum edit distance to the seed [19]' ) - parser.add_option( '-w', '--bandWidth', default=100, type=int, help='Band width for banded alignment [100]' ) - parser.add_option( '-d', '--offDiagonal', default=100, type=int, help='off-diagonal X-dropoff [100]' ) - parser.add_option( '-r', '--internalSeeds', default=1.5, type=float, help='look for internal seeds inside a seed longer than {-k} * FLOAT [1.5]' ) - parser.add_option( '-c', '--seedsOccurrence', default=10000, type=int, help='skip seeds with more than INT occurrences [10000]' ) - parser.add_option( '-S', '--mateRescue', default=False, help='skip mate rescue' ) - parser.add_option( '-P', '--skipPairing', default=False, help='skpe pairing, mate rescue performed unless -S also in use' ) - parser.add_option( '-A', '--seqMatch', default=1, type=int, help='score of a sequence match' ) - parser.add_option( '-B', '--mismatch', default=4, type=int, help='penalty for a mismatch' ) - parser.add_option( '-O', '--gapOpen', default=6, type=int, help='gap open penalty' ) - parser.add_option( '-E', '--gapExtension', default=None, help='gap extension penalty; a gap of size k cost {-O} + {-E}*k [1]' ) - parser.add_option( '-L', '--clipping', default=5, type=int, help='penalty for clipping [5]' ) - parser.add_option( '-U', '--unpairedReadpair', default=17, type=int, help='penalty for an unpaired read pair [17]' ) - parser.add_option( '-p', '--interPairEnd', default=False, help='first query file consists of interleaved paired-end sequences' ) + parser.add_option( '-k', '--minSeedLength', type=int, help='Minimum seed length [19]' ) + parser.add_option( '-w', '--bandWidth', type=int, help='Band width for banded alignment [100]' ) + parser.add_option( '-d', '--offDiagonal', type=int, help='Off-diagonal X-dropoff [100]' ) + parser.add_option( '-r', '--internalSeeds', type=float, help='Look for internal seeds inside a seed longer than {-k} * FLOAT [1.5]' ) + parser.add_option( '-c', '--seedsOccurrence', type=int, help='Skip seeds with more than INT occurrences [10000]' ) + parser.add_option( '-S', '--mateRescue', action='store_true', help='Skip mate rescue' ) + parser.add_option( '-P', '--skipPairing', action='store_true', help='Skip pairing' ) + parser.add_option( '-A', '--seqMatch', type=int, help='Score for a sequence match [1]' ) + parser.add_option( '-B', '--mismatch', type=int, help='Penalty for a mismatch [4]' ) + parser.add_option( '-O', '--gapOpen', type=int, help='Gap open penalty [6]' ) + parser.add_option( '-E', '--gapExtension', type=int, help='Gap extension penalty; a gap of length k costs {-O} + {-E}*k [1]' ) + parser.add_option( '-L', '--clipping', help='Penalty for clipping [5]' ) + parser.add_option( '-U', '--unpairedReadpair', type=int, help='Penalty for an unpaired read pair [17]' ) + parser.add_option( '-p', '--interPairEnd', action='store_true', help='FASTQ file consists of interleaved paired-end sequences' ) parser.add_option( '--rgid', help='Read group identifier' ) parser.add_option( '--rgsm', help='Sample' ) parser.add_option( '--rgpl', choices=[ 'CAPILLARY', 'LS454', 'ILLUMINA', 'SOLID', 'HELICOS', 'IONTORRENT', 'PACBIO' ], help='Platform/technology used to produce the reads' ) @@ -76,9 +57,10 @@ parser.add_option( '--rgks', help='The array of nucleotide bases that correspond to the key sequence of each read' ) parser.add_option( '--rgpg', help='Programs used for processing the read group' ) parser.add_option( '--rgpi', help='Predicted median insert size' ) - parser.add_option( '-T', '--minScore', default=30, type=int, help='minimum score to output [30]' ) - parser.add_option( '-M', '--mark', default=False, help='mark shorter split hits as secondary (for Picard/GATK compatibility)' ) - parser.add_option( '-H', '--suppressHeader', dest='suppressHeader', help='Suppress header' ) + parser.add_option( '-T', '--minScore', type=int, help='Minimum score to output [30]' ) + parser.add_option( '-a', '--outputAll', action='store_true', help='Output all found alignments for single-end or unpaired paired-end reads' ) + parser.add_option( '-M', '--mark', action='store_true', help='Mark shorter split hits as secondary (for Picard/GATK compatibility)' ) + parser.add_option( '-H', '--suppressHeader', dest='suppressHeader', action='store_true', help='Suppress header' ) (options, args) = parser.parse_args() if len(args) > 0: parser.error('Wrong number of arguments') @@ -152,7 +134,7 @@ shutil.rmtree( tmp_index_dir ) if os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) - stop_err( 'Error indexing reference sequence. ' + str( e ) ) + raise Exception, 'Error indexing reference sequence. ' + str( e ) else: ref_file_name = options.ref # if options.illumina13qual: @@ -161,29 +143,45 @@ # illumina_quals = "" # set up aligning and generate aligning command args - start_cmds = '-t %s ' % options.threads - if options.params == 'pre_set': - # aligning_cmds = '-t %s %s' % ( options.threads, illumina_quals ) - #start_cmds = '-t %s ' % options.threads - end_cmds = ' ' - print start_cmds, end_cmds - - else: - end_cmds = '-k %s -w %s -d %s -r %s -c %s -A %s -B %s -O %s -L %s -U %s -T %s ' % (options.minEditDistSeed, options.bandWidth, options.offDiagonal, options.internalSeeds, options.seedsOccurrence, options.seqMatch, options.mismatch, options.gapOpen, options.clipping, options.unpairedReadpair, options.minScore) + start_cmds = "bwa mem -t %s" % options.threads + if options.interPairEnd: + start_cmds += ' -p' + if options.params != 'pre_set': + if options.minSeedLength is not None: + start_cmds += " -k %d" % options.minSeedLength + if options.bandWidth is not None: + start_cmds += " -w %d" % options.bandWidth + if options.offDiagonal is not None: + start_cmds += " -d %d" % options.offDiagonal + if options.internalSeeds is not None: + start_cmds += " -r %s" % options.internalSeeds + if options.seedsOccurrence is not None: + start_cmds += " -c %d" % options.seedsOccurrence if options.mateRescue: - end_cmds += '-S ' - if options.skipPairing: - end_cmds += '-P ' - else: - if options.skipPairing: - print "Option Error and will not be considered, you should also choose 'skip mate rescue -S' option! " - if options.gapExtension != None: - end_cmds += '-E %s ' % options.gapExtension - + start_cmds += ' -S' + if options.skipPairing: + start_cmds += ' -P' + if options.seqMatch is not None: + start_cmds += " -A %d" % options.seqMatch + if options.mismatch is not None: + start_cmds += " -B %d" % options.mismatch + if options.gapOpen is not None: + start_cmds += " -O %d" % options.gapOpen + if options.gapExtension is not None: + start_cmds += " -E %d" % options.gapExtension + if options.clipping: + start_cmds += " -L %s" % options.clipping + if options.unpairedReadpair is not None: + start_cmds += " -U %d" % options.unpairedReadpair + if options.minScore is not None: + start_cmds += " -T %d" % options.minScore + if options.outputAll: + start_cmds += ' -a' + if options.mark: + start_cmds += ' -M' if options.rgid: if not options.rglb or not options.rgpl or not options.rgsm or not options.rglb: - stop_err( 'If you want to specify read groups, you must include the ID, LB, PL, and SM tags.' ) - # readGroup = '@RG\tID:%s\tLB:%s\tPL:%s\tSM:%s' % ( options.rgid, options.rglb, options.rgpl, options.rgsm ) + sys.exit( 'If you want to specify read groups, you must include the ID, LB, PL, and SM tags.' ) readGroup = '@RG\tID:%s\tLB:%s\tPL:%s\tSM:%s' % ( options.rgid, options.rglb, options.rgpl, options.rgsm ) if options.rgpu: readGroup += '\tPU:%s' % options.rgpu @@ -201,67 +199,58 @@ readGroup += '\tPG:%s' % options.rgpg if options.rgpi: readGroup += '\tPI:%s' % options.rgpi - end_cmds += ' -R "%s" ' % readGroup - - if options.interPairEnd: - end_cmds += '-p %s ' % options.interPairEnd - if options.mark: - end_cmds += '-M ' - + start_cmds += " -R '%s'" % readGroup if options.genAlignType == 'paired': - cmd = 'bwa mem %s %s %s %s %s > %s' % ( start_cmds, ref_file_name, fastq, rfastq, end_cmds, options.output ) + cmd = "%s %s %s %s > %s" % ( start_cmds, ref_file_name, fastq, rfastq, options.output ) else: - cmd = 'bwa mem %s %s %s %s > %s' % ( start_cmds, ref_file_name, fastq, end_cmds, options.output ) + cmd = "%s %s %s > %s" % ( start_cmds, ref_file_name, fastq, options.output ) # perform alignments buffsize = 1048576 try: # need to nest try-except in try-finally to handle 2.4 try: + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + print "The cmd is %s" % cmd + proc = subprocess.Popen( args=cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + tmp_stderr.close() + # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) + stderr = '' try: - tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name - tmp_stderr = open( tmp, 'wb' ) - print "The cmd is %s" % cmd - proc = subprocess.Popen( args=cmd, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - tmp_stderr.close() - # get stderr, allowing for case where it's very large - tmp_stderr = open( tmp, 'rb' ) - stderr = '' - try: - while True: - stderr += tmp_stderr.read( buffsize ) - if not stderr or len( stderr ) % buffsize != 0: - break - except OverflowError: - pass - tmp_stderr.close() - if returncode != 0: - raise Exception, stderr + while True: + stderr += tmp_stderr.read( buffsize ) + if not stderr or len( stderr ) % buffsize != 0: + break + except OverflowError: + pass + tmp_stderr.close() + if returncode != 0: + raise Exception, stderr + except Exception, e: + raise Exception, 'Error generating alignments. ' + str( e ) + # remove header if necessary + if options.suppressHeader: + tmp_out = tempfile.NamedTemporaryFile( dir=tmp_dir) + tmp_out_name = tmp_out.name + tmp_out.close() + try: + shutil.move( options.output, tmp_out_name ) except Exception, e: - raise Exception, 'Error generating alignments. ' + str( e ) - # remove header if necessary - if options.suppressHeader == 'true': - tmp_out = tempfile.NamedTemporaryFile( dir=tmp_dir) - tmp_out_name = tmp_out.name - tmp_out.close() - try: - shutil.move( options.output, tmp_out_name ) - except Exception, e: - raise Exception, 'Error moving output file before removing headers. ' + str( e ) - fout = file( options.output, 'w' ) - for line in file( tmp_out.name, 'r' ): - if not ( line.startswith( '@HD' ) or line.startswith( '@SQ' ) or line.startswith( '@RG' ) or line.startswith( '@PG' ) or line.startswith( '@CO' ) ): - fout.write( line ) - fout.close() - # check that there are results in the output file - if os.path.getsize( options.output ) > 0: - sys.stdout.write( 'BWA run on %s-end data' % options.genAlignType ) - else: - raise Exception, 'The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.' - except Exception, e: - stop_err( 'The alignment failed.\n' + str( e ) ) + raise Exception, 'Error moving output file before removing headers. ' + str( e ) + fout = file( options.output, 'w' ) + for line in file( tmp_out.name, 'r' ): + if not ( line.startswith( '@HD' ) or line.startswith( '@SQ' ) or line.startswith( '@RG' ) or line.startswith( '@PG' ) or line.startswith( '@CO' ) ): + fout.write( line ) + fout.close() + # check that there are results in the output file + if os.path.getsize( options.output ) > 0: + sys.stdout.write( 'BWA run on %s-end data' % options.genAlignType ) + else: + raise Exception, 'The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.' finally: # clean up temp dir if os.path.exists( tmp_index_dir ): diff -r 6820983ba5d5 -r ebb02ba5987c bwa_mem.xml --- a/bwa_mem.xml Tue Mar 18 07:49:22 2014 -0400 +++ b/bwa_mem.xml Fri Mar 21 12:56:15 2014 -0400 @@ -1,26 +1,29 @@ - + bwa - bwa 2>&1 | grep "Version: " | sed -e 's/Version: //' bwa_mem.py --threads="\${GALAXY_SLOTS:-1}" --fileSource="${genomeSource.refGenomeSource}" - #if $genomeSource.refGenomeSource == "history": + #if $genomeSource.refGenomeSource == "history" ##build index on the fly --ref="${genomeSource.ownFile}" --dbkey="${dbkey}" - #else: + #else ##use precomputed indexes --ref="${genomeSource.indices.fields.path}" #end if ## input file(s) --fastq="${paired.fastq}" - #if $paired.sPaired == "paired": + #if $paired.sPaired == "single" + #if $paired.interPairEnd + --interPairEnd + #end if + #else --rfastq="${paired.rfastq}" #end if @@ -30,28 +33,60 @@ ## run parameters --genAlignType="${paired.sPaired}" --params="${params.source_select}" - #if $params.source_select != "pre_set": - --minEditDistSeed="${params.minEditDistSeed}" - --bandWidth="${params.bandWidth}" - --offDiagonal="${params.offDiagonal}" - --internalSeeds="${params.internalSeeds}" - --seedsOccurrence="${params.seedsOccurrence}" - --mateRescue="${params.mateRescue}" - --skipPairing="${params.skipPairing}" - --seqMatch="${params.seqMatch}" - --mismatch="${params.mismatch}" - --gapOpen="${params.gapOpen}" - --gapExtension="${params.gapExtension}" - --clipping="${params.clipping}" - --unpairedReadpair="${params.unpairedReadpair}" - --interPairEnd="${params.interPairEnd}" - --minScore="${params.minScore}" - --mark="${params.mark}" + #if $params.source_select != "pre_set" + #if str($params.minEditDistSeed) + --minSeedLength ${params.minEditDistSeed} + #end if + #if str($params.bandWidth) + --bandWidth ${params.bandWidth} + #end if + #if str($params.offDiagonal) + --offDiagonal ${params.offDiagonal} + #end if + #if str($params.internalSeeds) + --internalSeeds ${params.internalSeeds} + #end if + #if str($params.seedsOccurrence) + --seedsOccurrence ${params.seedsOccurrence} + #end if + #if $params.mateRescue + --mateRescue + #end if + #if $params.skipPairing + --skipPairing + #end if + #if str($params.seqMatch) + --seqMatch ${params.seqMatch} + #end if + #if str($params.mismatch) + --mismatch ${params.mismatch} + #end if + #if str($params.gapOpen) + --gapOpen ${params.gapOpen} + #end if + #if str($params.gapExtension) + --gapExtension ${params.gapExtension} + #end if + #if $params.clipping + --clipping "${params.clipping}" + #end if + #if str($params.unpairedReadpair) + --unpairedReadpair ${params.unpairedReadpair} + #end if + #if str($params.minScore) + --minScore ${params.minScore} + #end if + #if $params.outputAll + --outputAll + #end if + #if $params.mark + --mark + #end if #if $params.readGroup.specReadGroup == "yes" --rgid="${params.readGroup.rgid}" --rgsm="${params.readGroup.rgsm}" - --rgpl="${params.readGroup.rgpl}" + --rgpl ${params.readGroup.rgpl} --rglb="${params.readGroup.rglb}" --rgpu="${params.readGroup.rgpu}" --rgcn="${params.readGroup.rgcn}" @@ -65,7 +100,9 @@ #end if ## suppress output SAM header - --suppressHeader="${suppressHeader}" + #if $suppressHeader + --suppressHeader + #end if @@ -88,11 +125,12 @@ - + + @@ -106,43 +144,55 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + \d+(,\d+)?$ + + + + + - + - - - - + + + + + + + + + + + + + + + + + + - + + \*|[ACMGRSVTWYHKDBN]+$ + @@ -151,7 +201,7 @@ - + diff -r 6820983ba5d5 -r ebb02ba5987c readme.rst --- a/readme.rst Tue Mar 18 07:49:22 2014 -0400 +++ b/readme.rst Fri Mar 21 12:56:15 2014 -0400 @@ -25,9 +25,10 @@ Version history --------------- -- Release 0: Initial release in the Tool Shed. This is a fork of http://toolshed.g2.bx.psu.edu/view/yufei-luo/bwa_0_7_5 repository with the following changes: Remove .loc file, only .loc.sample should be included. Fix bwa_index.loc.sample file to contain only comments. Add suppressHeader param as in bwa_wrappers. Use $GALAXY_SLOTS environment variable when available. Add and . Remove unused import. Fix spacing and typos. Use new recommended citation. Add tool_dependencies.xml . Rename to bwa_mem. Remove definitively colorspace support. Use optparse instead of argparse since Galaxy still supports Python 2.6 . +- Release 1 (bwa_mem 0.8.0): Rewrite of param handling. interPairEnd param moved to "paired" section. Add param for '-a' option. Remove basic parallelism tag, which does not work with multiple inputs (thanks Björn Grüning for the notice). Simplify Python code. +- Release 0 (bwa_mem 0.7.7): Initial release in the Tool Shed. This is a fork of http://toolshed.g2.bx.psu.edu/view/yufei-luo/bwa_0_7_5 repository with the following changes: Remove .loc file, only .loc.sample should be included. Fix bwa_index.loc.sample file to contain only comments. Add suppressHeader param as in bwa_wrappers. Use $GALAXY_SLOTS environment variable when available. Add and . Remove unused import. Fix spacing and typos. Use new recommended citation. Add tool_dependencies.xml . Rename to bwa_mem. Remove definitively colorspace support. Use optparse instead of argparse since Galaxy still supports Python 2.6 . Add COPYING and readme.rst files. Development ----------- -Development is hosted at https://bitbucket.org/nsoranzo/bwa_mem . Contributions and bug reports are very welcome! +Development is hosted at https://bitbucket.org/crs4/orione-tools . Contributions and bug reports are very welcome!