<tool id="pca_pipeline" name="PCA Pipeline" version="1.0.0">
	<description>Iterative PCA pipeline</description>
	<requirements>
		<requirement type="package" version="2.8">Jinja2</requirement>
		<!--
		<requirement type="package" version="3.2.1">R</requirement>
		<requirement type="package" version="1.2.5">flashpcaR</requirement>
		-->
	</requirements>
	<command interpreter="python">
		<![CDATA[
			iterative_pca.py 
			$datafile
			$data_type
			$iterations 
			--sd_cutoff $sd_cutoff
			--absolute_prefix $output.files_path 
			--html_file $output
			#if $control_tag
				--control_tag $control_tag
			#end if
			#if $cases_tag
				--cases_tag $cases_tag
			#end if
			#if $data_type.value == "variant_data"
				#if $user_configfile
					--config_file $user_configfile
				#else
					--config_file $cfile
				#end if
			#end if
			#if $clustering_flag.value == "yes"
				--clustering_flag
				--clustering_method $clustering_method
				--cluster_trimming $cluster_trimming
			#end if
			#if $ethnicity_file
				--ethnicity_file $ethnicity_file
			#end if
			#if $xsamples_file
				--reject_samples $xsamples_file
			#end if
			#if $xsnps_file
				--reject_snps $xsnps_file
			#end if
			--galaxy
			]]>
	</command>
	<configfiles>
		<configfile name="cfile">#control
control_tag,#Sample,$control_tag
cases_tag,#Sample,$cases_tag
#column_names
genotype_column,$genotype_column
reference_column,$reference_column
alternate_column,$alternate_column
sample_id_column,$sample_id_column
chromosome_column,$chromosome_column
position_column,$position_column
variant_id_column,$variant_id_column
#numeric_filters
#for $i, $f in enumerate($numeric_filters)
$f.filter_name,$f.column_name,$f.operation,$f.cutoff
#end for
#string_filters
#for $i, $s in enumerate($string_filters)
$s.filter_name,$s.column_name,$s.exact_flag,$s.accept_flag
$(','.join($s.patterns.split('\n')))
#end for</configfile>
	</configfiles>
	<inputs>
		<param name="datafile" type="data" label="Input datafile"/>
		<param name="data_type" type="select" label="Type of input data file">
			<option value="variant_data">Variant Data</option>
			<option value="numeric_ped">Numeric PED File</option>
			<option value="rdata">RData file</option>
		</param>		
		<param name="iterations" type="integer" value="1" label="Number of iterations to complete"/>
		<param name="clustering_flag" type="select" display="radio" label="Do clustering?">
			<option value="yes">Yes</option>
			<option value="no">No</option>
		</param>
		<param name="clustering_method" type="select" label="Clustering method (ignore if you selected 'No' for 'Do clustering?')">
			<option value="dbscan">DBSCAN</option>
			<option value="hclust">Hierarchical Clustering</option>
		</param>
		<param name="cluster_trimming" type="select" label="Algorithm used to identify and remove cluster outliers (ignore if you selected 'No' for 'Do clustering?')">
			<option value="sd">Standard Deviations</option>
			<option value="mcd">Mean Cluster Distance</option>
			<option value="dbscan_outliers_only">DBSCAN outliers only (Only valid if DBSCAN is selected as 'Algorithm used to find clusters'</option>
		</param>
		<param name="sd_cutoff" type="float" value="2" label="Strictness of outlier trimming. Lower = more outliers cut at each stage, Higher = less outliers cut at each stage."/>
		<!-- Control and cases tag info -->
		<param name="control_tag" type="text" value="LP" label="Control Tag"/>
		<param name="cases_tag" type="text" value="HAPS" label="Cases Tag"/>
		<param name="user_configfile" type="data" format="txt" optional="true" label="Optional user provided config file. 
			NB: 
			- If this is set, ALL the fields below will be ignored. 
			- If no input is provided, and the input data is a text file containing variant data, ALL the fields below except the filters MUST be filled in"/>
		<param name="ethnicity_file" type="data" format="txt" optional="true" label="Optional file containing data about ethnicity of samples"/>
		<param name="xsamples_file" type="data" format="txt" optional="true" label="Optional file containing EXACT ids of samples to exclude"/>
		<param name="xsnps_file" type="data" format="txt" optional="true" label="Optional file containing EXACT ids of SNPs to exclude"/>
		<!-- Column headers -->
		<param name="sample_id_column" type="text" value="#Sample" label="Sample ID Column"/>
		<param name="variant_id_column" type="text" value="ID" label="Variant ID Column"/>
		<param name="chromosome_column" type="text" value="CHROM" label="Chromosome Column"/>
		<param name="position_column" type="text" value="POS" label="Position Column"/>
		<param name="genotype_column" type="text" value="GT" label="Genotype Column"/>
		<param name="reference_column" type="text" value="REF" label="Reference Allele Column"/>
		<param name="alternate_column" type="text" value="ALT" label="Alternate Allele Column"/>
		<!-- Numeric Filters -->
		<repeat name="numeric_filters" title="Optional Numeric Filters">
			<param name="filter_name" type="text" label="Filter Name"/>
			<param name="column_name" type="text" label="Name of column to filter on"/>
			<param name="operation" type="select" label="Accept if column value is:">
				<option value="g">greater than</option>
				<option value="l">less than</option>
				<option value="e">equal to</option>
				<option value="ge">greater than or equal to</option>
				<option value="le">less than or equal to</option>
			</param>
			<param name="cutoff" type="float" value="0" label="Cutoff Value"/>
		</repeat>
		<!-- String Filters -->
		<repeat name="string_filters" title="Optional String Filters">
			<param name="filter_name" type="text" label="Filter Name"/>
			<param name="column_name" type="text" label="Name of column to filter on"/>
			<param name="exact_flag" type="select" label="Exact pattern matching?">
				<option value="exact">Yes</option>
				<option value="not_exact">No</option>
			</param>			
			<param name="accept_flag" type="select" label="Action to perform after a successful match">
				<option value="accept">Accept</option>
				<option value="reject">Reject</option>
			</param>       	
			<param name="patterns" type="text" area="true" size="10x35" label="Patterns to match on" help="Enter a list of patterns here, separated by newlines"/>
		</repeat>
	</inputs>
	<outputs>
		<data name="output" format="html"> 
			<label>PCA summary: "${datafile.name}"</label>
		</data>
	</outputs>
	<help><![CDATA[
.. class:: warningmark 

'''WARNING''' This tool requires the 'dbscan' (https://cran.r-project.org/web/packages/dbscan/index.html) and 'flashpcaR' (https://github.com/gabraham/flashpca/releases) R packages to be installed on the galaxy instance.

======================================
Principle Component Analysis Pipeline 
======================================

Overview
--------
A tool which performs iterative principle component analysis. 
The general idea is to seperate patient samples based on their ethnicity, by performing PCA on the variant data of each sample. 
After each analysis step, outliers are identified. The PCA is then repeated, with the outliers removed.
This process continues for a set number of iterations specified by the user. After the pipeline completes, the user can see a 
detailed summary, as well as have access to the outliers identified at each iteration.

Primary Input
---------------
As primary input the tools accepts a single file, which may be formatted in the following ways:

- **Variant data file:** This should be a tab-delimited text file, with each row containing data about a single variant site from a single person. If this option is selected, the column names which contain important information must also be specified, either via a configuration file (see below), or through the tool's form fields.
- **Numeric ped file:** See http://pngu.mgh.harvard.edu/~purcell/plink/data.shtml for detailed information on PED format. This tool requires the affection status of each site to be specified numerically i.e.:

	- 0 = homozygous reference 
	- 1 = heterozygous
	- 2 = homozygous alternate
  
  rather than consisting of pairs of genotypes for each site.
- **RData file:** File containing stored data from an R session. For this tool the input must meet certain requirements:
 	
 	- The file can only contain a SINGLE R object, which must be a list. 
 	- The list must contain a named 'bed' element. 
 	- The 'bed' element must be an n x m matrix/data frame, where n = number of samples, m = number of unique snps found in all the samples. 
 	- The A(i,j)th entry in the 'bed' matrix should indicate affectation status of the ith sample at the jth SNP site, according to the key for numeric ped files (as above). 
 	- The row names of the 'bed' matrix must contain the ids of the samples. 
 	- The column names of the 'bed' matrix must contain the ids of the SNPs. 

  If these very specific criteria are not met, the tool WILL fail.



Primary Output
---------------

HTML file containing plots of the PCA for each iteration.
Possible plots, depending on user specified options:

- **Control vs Cases Plot:** If control and/or cases tags are provided, this plot will be output. ALL samples are plotted, with controls shown in blue, cases in red, unknown samples in black.
- **Cluster Plot:** Output if user opts to do clustering. Samples are plotted, with clusters colour-coded. Outliers as identified by DBSCAN are always read and use an open circle as the icon. Trimmed clusters use a cross for the icon, instead of a circle. Both the outliers (open circles) AND the rejected clusters (crosses) will be dropped in the next iteration.
- **Outliers Plot:** Output if user does NOT opt to do clustering. Samples which are considered outliers (as described above in 'Detecting outliers without clustering') are plotted as red open circles; all other samples are plotted as green full circles.
- **Standard Deviations Plot:** Samples are colour-coded by standard deviation. Samples which fall within 1 standard devaiton of the median are red, <= 2 sds are green, <= 3 sds are blue, > 3 sds are purple. 
- **Ethnicity Plot:** Each ethnicity uses a specific colour and symbol. Fairly self-explanotory. Plot is only output if an ethnicity data file is provided as input.

Beneath the plots there are also two expandable lists. Samples excluded shows which samples were not part of the PCA for this iteration. This is cumulative. Outliers shows the outliers detected in THIS iteration. Any available data from the ethnicity file (if provided) is also displayed for each excluded sample.

Options/Secondary Inputs
------------------------
- **Type of input data file:** Either a ped file or a text file as specified above
- **Number of iterations to complete:** A single iteration would involve performing PCA on the input data, then identifying and removing outliers. Two iterations would involve performing PCA again with the outliers identified from the first iteration excluded, three iterations would exclude the outliers from the first 2 stages, and so on and so forth.
- **Detecting outliers without clustering:** This is done by obtaining the standard deviations of the first two principle components. Any samples whose scores for either of these first two components falls more than 'n' number of standard deviations away from the component median are considered outliers.
- **Clustering:** The user may select from a range of algorithms which will try to identify clusters in the data, with each cluster hopefully corresponding to an ethnic group.
- **Clustering methods:**

	- *DBSCAN (Density based spatial clustering of applications with noise):*

	  Forms clusters based on density of points, and does not require the number of clusters to be specified beforehand. Good for irregularly shaped, non-spherical clusters. Does NOT require all points to be part of clusters, and produces a set of 'outliers', i.e. points which do not belong to any clusters.
	
	- *Hierarchical Clustering:* 

	  Forms clusters based on distance between points. Tends to result in spherical clusters, but able to handle clusters of varying density. Forces all points to be part of a single cluster. The number of clusters is determined seperately, using the silhouette scores of all the points as a heuristic.

- **Cluster trimming methods:** All these methods first involve finding the centres of each cluster.

	- *Standard Deviations:* 

	  If the centroid of a cluster lies more than ‘n’ standard deviations (n is passed in as a parameter by the user) from the centroid of the entire dataset in either the x or y directions, the entire cluster is cut. If DBSCAN is selected, the outliers it identifies are also cut.

	- *Mean Cluster Distance:* 

	  Obtain the average distance between clusters, done by computing the distance between all pairs of clusters and taking the mean. For each cluster, we also compute an average “isolation” value, which is the mean of the distances between that particular cluster and all other clusters. If a cluster’s isolation value is larger than the average cluster distance (multiplied by the strictness weighting), then that cluster is considered an outlier and cut from the next iteration. If DBSCAN is selected, the outliers it identifies are also cut.

	- *DBSCAN outliers only:* 

	  Only cut the points identified by the DBSCAN algorithm as not belonging to any cluster. No entire clusters are cut. Obviously this method is only applicable if DBSCAN is selected as the clustering method. THE TOOL WILL NOT RUN IF YOU SELECT THIS OPTION TOGETHER WITH 'Hierarchical Clustering' AS THE CLUSTERING METHOD.

- **Strictness:** A multiplier used to determine how 'strict' the outlier cutting methods are. For example, if strictness = 1, and we are not doing clustering, all points which lie more than 1 sd from the median are cut. If strictness = 2, all points which lie more than 2 sd from the median are cut, etc.

- **Control Tag:** A pattern present in the ids of all the control samples, e.g. "LP"

- **Cases Tag:** A pattern present in the ids of all the cases samples, e.g. "HAPS"

- **Configuration file:** A configuration file to accompany an input variant text file. The config file has a rather specific format, an example is given below::

	#control
	control_tag,#Sample,HAPS
	cases_tag,#Sample,LP
	#column_names
	genotype_column,GT
	reference_column,REF
	alternate_column,ALT
	sample_id_column,#Sample
	chromosome_column,CHROM
	position_column,POS
	variant_id_column,ID
	#numeric_filters
	strand_bias_filter,Fraction_with_strand_bias,<,0.03
	position_bias_filter,Fraction_with_positional_bias,<,0.03
	count_filter,Num_samples_variant,>,1
	pass_filter,Fraction_samples_passed_filter,>,0.9
	#string_filters
	variant_type_filter,Variant_Type,exact,accept
	SNV
	genotype_filter,GT,exact,accept
	'0/1,'1/1

  File consists of up to four sections, the starts of which are marked by lines beginning with an octothorpe.

	- *'#control' section:* Indicates substrings found in ids of controls and cases
	- *'#column_names' section:* This is the only required section. First column indicates what column name (in the variant text file) the second column specifies. The same keys i.e. left most column values, as shown in the example must be used, e.g. sample_id_column, the RHS column names must match the names in the variant data file. If using a generated config file, only modify the RHS column, and DO NOT REMOVE ANY rows from this section.
	- *'#numeric_filters' section:* Each filter takes up a single line, and is seperated into 4 sections by commas.

		- Column 1: Name of the filter, which is arbitrary
		- Column 2: The name of the column in the variant file to filter on. If this column is not found, a warning is displayed
		- Column 3: The criteria of the filter which must be passed in order for us to accept a particular row. E.g. less than, greater than
		- Column 4: The cutoff value to be compared against.

	- *'#string_filters' section:* Each filter takes up two lines.

		- Line 1, Column 1: Arbitrary filter name
		- Line 1, Column 2: Column name to filter on 
		- Line 1, Column 3: Do the patterns have to be exact matches, or just a substrings? E.g. if pattern = "HAPS" and string being compared = "HAPS-909090", if exact was true this would not be a successfull match, whereas if not_exact was true it would be a match.
		- Line 1, Column 4: What to do with the row if a successful match is detected, e.g. accept or reject
		- Line 2: A comma seperated list of patterns to match on


- **Ethnicity file:** An ethnicity file containing ethnicity data, and possible other data, on the samples. Note this data is not used to sort the input and has no effect on the PCA itself. It is used only to label the results of the output.

  Requirements:

 	- tab delimited
 	- Must have at least two columns
 	- First column has sample ID's
 	- Second column has ethnicities
 	- First row must be a header

  First few lines of a correctly formatted ethnicity file given below::

	IID	population	Halo1.or.2.	BloodAge	SalivaAge	COB	ethnicity
	LP-10000001	AUSTRALIAN	Halo2 - LP-BC	67	NA	Australia	australian
	LP-10000003	AUSTRALIAN	Halo1	45	NA	Australia	australian  southern_european
	LP-10000005	AUSTRALIAN	Halo1	73	NA	Australia	australian  southern_european
	LP-10000008	EUROPE	Halo1	54	NA	South Eastern Europe	south_east_european
	LP-10000009	OTHER	Halo1	65	NA	Southern & East Africa	jewish

- **Exclude samples file:** A text file containing exact ids of samples to exclude from the PCA.

  Requirements:

	- single column
	- sample ids seperated by newlines
	- one sample id per line
  
  Example::

	HAPS-90573
	HAPS-90578R
	HAPS-110542
	HAPS-110605
	HAPS-110620
	HAPS-110638
	HAPS-110649
	HAPS-110668
	HAPS-110799
	HAPS-110813
	HAPS-110959
	HAPS-111186
	HAPS-111298
	HAPS-111404
	HAPS-111493
	HAPS-111512
	HAPS-111538

- **Exclude SNPS file:** A text file containing exact ids of SNPs to exclude from the PCA.
  
  Requirements:

	- single column
	- snp ids seperated by newlines
	- one snp id per line

  Example::

	rs72896283
	rs7534447
	rs4662775
	rs10932813
	rs10932816
	rs12330369
	rs1802904
	rs10902762
	rs9996817
	rs6446393
	rs871133
	rs4301095
	rs941849
	rs6917467
	rs75834296
	rs142922667

- **Required Column Headers:** If a variant text file is the primary input, the following information MUST be provided, either through the config file, or by filling out the corresponding fields in the tool submission form.

	- Sample IDs: Name of the column containing the sample ids
	- Chromosome: Name of the column indicating what chromosome the SNP is found on
	- Position: Name of the column indicating at which position on the chromosome the SNP is found
	- Genotype: The genotype of the sample for this site
	- Reference: The 'normal'/'common' genotype for this site
	- Alternate: The alternate genotype for this site
	- Variant IDs: Name of the column indicating the ID of the SNP 

- **Numeric Filters:** See Configuration file section
- **String Filters:** See Configuration file section

Other Output
-------------

- Tool will output a root folder containing the HTML file and all the plots, placed in directories seperated by iteration.
- If the input data was a variant file, the output folder will also contain a numeric ped file, generated before the first iteration, as well as a config file. The config file is either the exact one passed in by the user, or one automatically generated from the form input, which can be used for future PCA runs.

	]]>


	</help>
</tool>