Mercurial > repos > george-weingart > maaslin

comparison src/Maaslin.R @ 0:e0b5980139d9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
maaslin
author george-weingart
date Tue, 13 May 2014 22:00:40 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:e0b5980139d9
1 #!/usr/bin/env Rscript
2 #####################################################################################
3 #Copyright (C) <2012>
4 #
5 #Permission is hereby granted, free of charge, to any person obtaining a copy of
6 #this software and associated documentation files (the "Software"), to deal in the
7 #Software without restriction, including without limitation the rights to use, copy,
8 #modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
9 #and to permit persons to whom the Software is furnished to do so, subject to
10 #the following conditions:
11 #
12 #The above copyright notice and this permission notice shall be included in all copies
13 #or substantial portions of the Software.
14 #
15 #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
16 #INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
17 #PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
18 #HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 #OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20 #SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 #
22 # This file is a component of the MaAsLin (Multivariate Associations Using Linear Models),
23 # authored by the Huttenhower lab at the Harvard School of Public Health
24 # (contact Timothy Tickle, ttickle@hsph.harvard.edu).
25 #####################################################################################
26
27 inlinedocs <- function(
28 ##author<< Curtis Huttenhower <chuttenh@hsph.harvard.edu> and Timothy Tickle <ttickle@hsph.harvard.edu>
29 ##description<< Main driver script. Should be called to perform MaAsLin Analysis.
30 ) { return( pArgs ) }
31
32
33 ### Install packages if not already installed
34 vDepLibrary = c("agricolae", "gam", "gamlss", "gbm", "glmnet", "inlinedocs", "logging", "MASS", "nlme", "optparse", "outliers", "penalized", "pscl", "robustbase", "testthat")
35 for(sDepLibrary in vDepLibrary)
36 {
37 if(! require(sDepLibrary, character.only=TRUE) )
38 {
39 install.packages(pkgs=sDepLibrary, repos="http://cran.us.r-project.org")
40 }
41 }
42
43 ### Logging class
44 suppressMessages(library( logging, warn.conflicts=FALSE, quietly=TRUE, verbose=FALSE))
45 ### Class for commandline argument processing
46 suppressMessages(library( optparse, warn.conflicts=FALSE, quietly=TRUE, verbose=FALSE))
47
48
49 ### Create command line argument parser
50 pArgs <- OptionParser( usage = "%prog [options] <output.txt> <data.tsv>" )
51
52 # Input files for MaAsLin
53 ## Data configuration file
54 pArgs <- add_option( pArgs, c("-i", "--input_config"), type="character", action="store", dest="strInputConfig", metavar="data.read.config", help="Optional configuration file describing data input format.")
55 ## Data manipulation/normalization file
56 pArgs <- add_option( pArgs, c("-I", "--input_process"), type="character", action="store", dest="strInputR", metavar="data.R", help="Optional configuration script normalizing or processing data.")
57
58 # Settings for MaAsLin
59 ## Maximum false discovery rate
60 pArgs <- add_option( pArgs, c("-d", "--fdr"), type="double", action="store", dest="dSignificanceLevel", default=0.25, metavar="significance", help="The threshold to use for significance for the generated q-values (BH FDR). Anything equal to or lower than this is significant. [Default %default]")
61 ## Minimum feature relative abundance filtering
62 pArgs <- add_option( pArgs, c("-r", "--minRelativeAbundance"), type="double", action="store", dest="dMinAbd", default=0.0001, metavar="minRelativeAbundance", help="The minimum relative abundance allowed in the data. Values below this are removed and imputed as the median of the sample data. [Default %default]")
63 ## Minimum feature prevalence filtering
64 pArgs <- add_option( pArgs, c("-p", "--minPrevalence"), type="double", action="store", dest="dMinSamp", default=0.1, metavar="minPrevalence", help="The minimum percentage of samples a feature can have abundance in before being removed. Also is the minimum percentage of samples a metadata can have that are not NA before being removed. [Default %default]")
65 ## Fence for outlier, if not set Grubbs test is used
66 pArgs <- add_option( pArgs, c("-o", "--outlierFence"), type="double", action="store", dest="dOutlierFence", default=0, metavar="outlierFence", help="Outliers are defined as this number times the interquartile range added/subtracted from the 3rd/1st quartiles respectively. If set to 0 (default), outliers are defined by the Grubbs test. [Default %default]")
67 ## Significance for Grubbs test
68 pArgs <- add_option(pArgs, c("-G","--grubbsSig"), type="double", action="store", dest="dPOutlier", default=0.05, metavar="grubbsAlpha", help="This is the significance cuttoff used to indicate an outlier or not. The closer to zero, the more significant an outlier must be to be removed. [Default %default]")
69 ## Fixed (not random) covariates
70 pArgs <- add_option( pArgs, c("-R","--random"), type="character", action="store", dest="strRandomCovariates", default=NULL, metavar="fixed", help="These metadata will be treated as random covariates. Comma delimited data feature names. These features must be listed in the read.config file. Example '-R RandomMetadata1,RandomMetadata2'. [Default %default]")
71 ## Change the type of correction fo rmultiple corrections
72 pArgs <- add_option( pArgs, c("-T","--testingCorrection"), type="character", action="store", dest="strMultTestCorrection", default="BH", metavar="multipleTestingCorrection", help="This indicates which multiple hypothesis testing method will be used, available are holm, hochberg, hommel, bonferroni, BH, BY. [Default %default]")
73 ## Use a zero inflated model of the inference method indicate in -m
74 pArgs <- add_option( pArgs, c("-z","--doZeroInfated"), type="logical", action="store_true", default = FALSE, dest="fZeroInflated", metavar="fZeroInflated", help="If true, the zero inflated version of the inference model indicated in -m is used. For instance if using lm, zero-inflated regression on a gaussian distribution is used. [Default %default].")
75
76 # Arguments used in validation of MaAsLin
77 ## Model selection (enumerate) c("none","boost","penalized","forward","backward")
78 pArgs <- add_option( pArgs, c("-s", "--selection"), type="character", action="store", dest="strModelSelection", default="boost", metavar="model_selection", help="Indicates which of the variable selection techniques to use. [Default %default]")
79 ## Argument indicating which method should be ran (enumerate) c("univariate","lm","neg_binomial","quasi")
80 pArgs <- add_option( pArgs, c("-m", "--method"), type="character", action="store", dest="strMethod", default="lm", metavar="analysis_method", help="Indicates which of the statistical inference methods to run. [Default %default]")
81 ## Argument indicating which link function is used c("none","asinsqrt")
82 pArgs <- add_option( pArgs, c("-l", "--link"), type="character", action="store", dest="strTransform", default="asinsqrt", metavar="transform_method", help="Indicates which link or transformation to use with a glm, if glm is not selected this argument will be set to none. [Default %default]")
83 pArgs <- add_option( pArgs, c("-Q","--NoQC"), type="logical", action="store_true", default=FALSE, dest="fNoQC", metavar="Do_Not_Run_QC", help="Indicates if the quality control will be ran on the metadata/data. Default is true. [Default %default]")
84
85 # Arguments to suppress MaAsLin actions on certain data
86 ## Do not perform model selection on the following data
87 pArgs <- add_option( pArgs, c("-F","--forced"), type="character", action="store", dest="strForcedPredictors", default=NULL, metavar="forced_predictors", help="Metadata features that will be forced into the model seperated by commas. These features must be listed in the read.config file. Example '-F Metadata2,Metadata6,Metadata10'. [Default %default]")
88 ## Do not impute the following
89 pArgs <- add_option( pArgs, c("-n","--noImpute"), type="character", action="store", dest="strNoImpute", default=NULL, metavar="no_impute", help="These data will not be imputed. Comma delimited data feature names. Example '-n Feature1,Feature4,Feature6'. [Default %default]")
90
91 #Miscellaneouse arguments
92 ### Argument to control logging (enumerate)
93 strDefaultLogging = "DEBUG"
94 pArgs <- add_option( pArgs, c("-v", "--verbosity"), type="character", action="store", dest="strVerbosity", default=strDefaultLogging, metavar="verbosity", help="Logging verbosity [Default %default]")
95 ### Run maaslin without creating a log file
96 pArgs <- add_option( pArgs, c("-O","--omitLogFile"), type="logical", action="store_true", default=FALSE, dest="fOmitLogFile", metavar="omitlogfile",help="Including this flag will stop the creation of the output log file. [Default %default]")
97 ### Argument for inverting background to black
98 pArgs <- add_option( pArgs, c("-t", "--invert"), type="logical", action="store_true", dest="fInvert", default=FALSE, metavar="invert", help="When given, flag indicates to invert the background of figures to black. [Default %default]")
99 ### Selection Frequency
100 pArgs <- add_option( pArgs, c("-f","--selectionFrequency"), type="double", action="store", dest="dSelectionFrequency", default=NA, metavar="selectionFrequency", help="Selection Frequency for boosting (max 1 will remove almost everything). Interpreted as requiring boosting to select metadata 100% percent of the time (or less if given a number that is less). Value should be between 1 (100%) and 0 (0%), NA (default is determined by data size).")
101 ### All v All
102 pArgs <- add_option( pArgs, c("-a","--allvall"), type="logical", action="store_true", dest="fAllvAll", default=FALSE, metavar="compare_all", help="When given, the flag indicates that each fixed covariate that is not indicated as Forced is compared once at a time per data feature (bug). Made to be used with the -F option to specify one part of the model while allowing the other to cycle through a group of covariates. Does not affect Random covariates, which are always included when specified. [Default %default]")
103 pArgs <- add_option( pArgs, c("-N","--PlotNA"), type="logical", action="store_true", default=FALSE, dest="fPlotNA", metavar="plotNAs",help="Plot data that was originally NA, by default they are not plotted. [Default %default]")
104 ### Alternative methodology settings
105 pArgs <- add_option( pArgs, c("-A","--pAlpha"), type="double", action="store", dest="dPenalizedAlpha", default=0.95, metavar="PenalizedAlpha",help="The alpha for penalization (1.0=L1 regularization, LASSO; 0.0=L2 regularization, ridge regression. [Default %default]")
106 ### Pass an alternative library dir
107 pArgs <- add_option( pArgs, c("-L", "--libdir"), action="store", dest="sAlternativeLibraryLocation", default=file.path( "","usr","share","biobakery" ), metavar="AlternativeLibraryDirectory", help="An alternative location to find the lib directory. This dir and children will be searched for the first maaslin/src/lib dir.")
108
109 ### Misc biplot arguments
110 pArgs <- add_option( pArgs, c("-M","--BiplotMetadataScale"), type="double", action="store", dest="dBiplotMetadataScale", default=1, metavar="scaleForMetadata", help="A real number used to scale the metadata labels on the biplot (otherwise a default will be selected from the data). [Default %default]")
111 pArgs <- add_option( pArgs, c("-C", "--BiplotColor"), type="character", action="store", dest="strBiplotColor", default=NULL, metavar="BiplotColorCovariate", help="A continuous metadata that will be used to color samples in the biplot ordination plot (otherwise a default will be selected from the data). Example Age [Default %default]")
112 pArgs <- add_option( pArgs, c("-S", "--BiplotShapeBy"), type="character", action="store", dest="strBiplotShapeBy", default=NULL, metavar="BiplotShapeCovariate", help="A discontinuous metadata that will be used to indicate shapes of samples in the Biplot ordination plot (otherwise a default will be selected from the data). Example Sex [Default %default]")
113 pArgs <- add_option( pArgs, c("-P", "--BiplotPlotFeatures"), type="character", action="store", dest="strBiplotPlotFeatures", default=NULL, metavar="BiplotFeaturesToPlot", help="Metadata and data features to plot (otherwise a default will be selected from the data). Comma Delimited.")
114 pArgs <- add_option( pArgs, c("-D", "--BiplotRotateMetadata"), type="character", action="store", dest="sRotateByMetadata", default=NULL, metavar="BiplotRotateMetadata", help="Metadata to use to rotate the biplot. Format 'Metadata,value'. 'Age,0.5' . [Default %default]")
115 pArgs <- add_option( pArgs, c("-B", "--BiplotShapes"), type="character", action="store", dest="sShapes", default=NULL, metavar="BiplotShapes", help="Specify shapes specifically for metadata or metadata values. [Default %default]")
116 pArgs <- add_option( pArgs, c("-b", "--BugCount"), type="integer", action="store", dest="iNumberBugs", default=3, metavar="PlottedBugCount", help="The number of bugs automatically selected from the data to plot. [Default %default]")
117 pArgs <- add_option( pArgs, c("-E", "--MetadataCount"), type="integer", action="store", dest="iNumberMetadata", default=NULL, metavar="PlottedMetadataCount", help="The number of metadata automatically selected from the data to plot. [Default all significant metadata and minimum is 1]")
118
119 #pArgs <- add_option( pArgs, c("-c","--MFAFeatureCount"), type="integer", action="store", dest="iMFAMaxFeatures", default=3, metavar="maxMFAFeature", help="Number of features or number of bugs to plot (default=3; 3 metadata and 3 data).")
120
121 main <- function(
122 ### The main function manages the following:
123 ### 1. Optparse arguments are checked
124 ### 2. A logger is created if requested in the optional arguments
125 ### 3. The custom R script is sourced. This is the input *.R script named
126 ### the same as the input *.pcl file. This script contains custom formating
127 ### of data and function calls to the MFA visualization.
128 ### 4. Matrices are written to the project folder as they are read in seperately as metadata and data and merged together.
129 ### 5. Data is cleaned with custom filtering if supplied in the *.R script.
130 ### 6. Transformations occur if indicated by the optional arguments
131 ### 7. Standard quality control is performed on data
132 ### 8. Cleaned metadata and data are written to output project for documentation.
133 ### 9. A regularization method is ran (boosting by default).
134 ### 10. An analysis method is performed on the model (optionally boosted model).
135 ### 11. Data is summarized and PDFs are created for significant associations
136 ### (those whose q-values {BH FDR correction} are <= the threshold given in the optional arguments.
137 pArgs
138 ### Parsed commandline arguments
139 ){
140 lsArgs <- parse_args( pArgs, positional_arguments = TRUE )
141 #logdebug("lsArgs", c_logrMaaslin)
142 #logdebug(paste(lsArgs,sep=" "), c_logrMaaslin)
143
144 # Parse parameters
145 lsForcedParameters = NULL
146 if(!is.null(lsArgs$options$strForcedPredictors))
147 {
148 lsForcedParameters = unlist(strsplit(lsArgs$options$strForcedPredictors,","))
149 }
150 xNoImpute = NULL
151 if(!is.null(lsArgs$options$strNoImpute))
152 {
153 xNoImpute = unlist(strsplit(lsArgs$options$strNoImpute,"[,]"))
154 }
155 lsRandomCovariates = NULL
156 if(!is.null(lsArgs$options$strRandomCovariates))
157 {
158 lsRandomCovariates = unlist(strsplit(lsArgs$options$strRandomCovariates,"[,]"))
159 }
160 lsFeaturesToPlot = NULL
161 if(!is.null(lsArgs$options$strBiplotPlotFeatures))
162 {
163 lsFeaturesToPlot = unlist(strsplit(lsArgs$options$strBiplotPlotFeatures,"[,]"))
164 }
165
166 #If logging is not an allowable value, inform user and set to INFO
167 if(length(intersect(names(loglevels), c(lsArgs$options$strVerbosity))) == 0)
168 {
169 print(paste("Maaslin::Error. Did not understand the value given for logging, please use any of the following: DEBUG,INFO,WARN,ERROR."))
170 print(paste("Maaslin::Warning. Setting logging value to \"",strDefaultLogging,"\"."))
171 }
172
173 # Do not allow mixed effect models and zero inflated models, don't have implemented
174 if(lsArgs$options$fZeroInflated && !is.null(lsArgs$options$strRandomCovariates))
175 {
176 stop("MaAsLin Error:: The combination of zero inflated models and mixed effects models are not supported.")
177 }
178
179 ### Create logger
180 c_logrMaaslin <- getLogger( "maaslin" )
181 addHandler( writeToConsole, c_logrMaaslin )
182 setLevel( lsArgs$options$strVerbosity, c_logrMaaslin )
183
184 #Get positional arguments
185 if( length( lsArgs$args ) != 2 ) { stop( print_help( pArgs ) ) }
186 ### Output file name
187 strOutputTXT <- lsArgs$args[1]
188 ### Input TSV data file
189 strInputTSV <- lsArgs$args[2]
190
191 # Get analysis method options
192 # includes data transformations, model selection/regularization, regression models/links
193 lsArgs$options$strModelSelection = tolower(lsArgs$options$strModelSelection)
194 if(!lsArgs$options$strModelSelection %in% c("none","boost","penalized","forward","backward"))
195 {
196 logerror(paste("Received an invalid value for the selection argument, received '",lsArgs$options$strModelSelection,"'"), c_logrMaaslin)
197 stop( print_help( pArgs ) )
198 }
199 lsArgs$options$strMethod = tolower(lsArgs$options$strMethod)
200 if(!lsArgs$options$strMethod %in% c("univariate","lm","neg_binomial","quasi"))
201 {
202 logerror(paste("Received an invalid value for the method argument, received '",lsArgs$options$strMethod,"'"), c_logrMaaslin)
203 stop( print_help( pArgs ) )
204 }
205 lsArgs$options$strTransform = tolower(lsArgs$options$strTransform)
206 if(!lsArgs$options$strTransform %in% c("none","asinsqrt"))
207 {
208 logerror(paste("Received an invalid value for the transform/link argument, received '",lsArgs$options$strTransform,"'"), c_logrMaaslin)
209 stop( print_help( pArgs ) )
210 }
211
212 if(!lsArgs$options$strMultTestCorrection %in% c("holm", "hochberg", "hommel", "bonferroni", "BH", "BY"))
213 {
214 logerror(paste("Received an invalid value for the multiple testing correction argument, received '",lsArgs$options$strMultTestCorrection,"'"), c_logrMaaslin)
215 stop( print_help( pArgs ) )
216 }
217
218 ### Necessary local import files
219 ### Check to make sure the lib is in the expected place (where the script is)
220 ### if not, then try the alternative lib location
221 ### This will happen if, for instance the script is linked or
222 ### on the path.
223 # Get the first choice relative path
224 initial.options <- commandArgs(trailingOnly = FALSE)
225 script.name <- sub("--file=", "", initial.options[grep("--file=", initial.options)])
226 strDir = file.path( dirname( script.name ), "lib" )
227 # If this does not have the lib file then go for the alt lib
228 if( !file.exists(strDir) )
229 {
230 lsPotentialListLocations = dir( path = lsArgs$options$sAlternativeLibraryLocation, pattern = "lib", recursive = TRUE, include.dirs = TRUE)
231 if( length( lsPotentialListLocations ) > 0 )
232 {
233 sLibraryPath = file.path( "maaslin","src","lib" )
234 iLibraryPathLength = nchar( sLibraryPath )
235 for( strSearchDir in lsPotentialListLocations )
236 {
237 # Looking for the path where the end of the path is equal to the library path given earlier
238 # Also checks before hand to make sure the path is atleast as long as the library path so no errors occur
239 if ( substring( strSearchDir, 1 + nchar( strSearchDir ) - iLibraryPathLength ) == sLibraryPath )
240 {
241 strDir = file.path( lsArgs$options$sAlternativeLibraryLocation, strSearchDir )
242 break
243 }
244 }
245 }
246 }
247
248 strSelf = basename( script.name )
249 for( strR in dir( strDir, pattern = "*.R$" ) )
250 {
251 if( strR == strSelf ) {next}
252 source( file.path( strDir, strR ) )
253 }
254
255 # Get analysis modules
256 afuncVariableAnalysis = funcGetAnalysisMethods(lsArgs$options$strModelSelection,lsArgs$options$strTransform,lsArgs$options$strMethod,lsArgs$options$fZeroInflated)
257
258 # Set up parameters for variable selection
259 lxParameters = list(dFreq=lsArgs$options$dSelectionFrequency, dPAlpha=lsArgs$options$dPenalizedAlpha)
260 if((lsArgs$options$strMethod == "lm")||(lsArgs$options$strMethod == "univariate"))
261 { lxParameters$sFamily = "gaussian"
262 } else if(lsArgs$options$strMethod == "neg_binomial"){ lxParameters$sFamily = "binomial"
263 } else if(lsArgs$options$strMethod == "quasi"){ lxParameters$sFamily = "poisson"}
264
265 #Indicate start
266 logdebug("Start MaAsLin", c_logrMaaslin)
267 #Log commandline arguments
268 logdebug("Commandline Arguments", c_logrMaaslin)
269 logdebug(lsArgs, c_logrMaaslin)
270
271 ### Output directory for the study based on the requested output file
272 outputDirectory = dirname(strOutputTXT)
273 ### Base name for the project based on the read.config name
274 strBase <- sub("\\.[^.]*$", "", basename(strInputTSV))
275
276 ### Sources in the custom script
277 ### If the custom script is not there then
278 ### defaults are used and no custom scripts are ran
279 funcSourceScript <- function(strFunctionPath)
280 {
281 #If is specified, set up the custom func clean variable
282 #If the custom script is null then return
283 if(is.null(strFunctionPath)){return(NULL)}
284
285 #Check to make sure the file exists
286 if(file.exists(strFunctionPath))
287 {
288 #Read in the file
289 source(strFunctionPath)
290 } else {
291 #Handle when the file does not exist
292 stop(paste("MaAsLin Error: A custom data manipulation script was indicated but was not found at the file path: ",strFunctionPath,sep=""))
293 }
294 }
295
296 #Read file
297 inputFileData = funcReadMatrices(lsArgs$options$strInputConfig, strInputTSV, log=TRUE)
298 if(is.null(inputFileData[[c_strMatrixMetadata]])) { names(inputFileData)[1] <- c_strMatrixMetadata }
299 if(is.null(inputFileData[[c_strMatrixData]])) { names(inputFileData)[2] <- c_strMatrixData }
300
301 #Metadata and bug names
302 lsOriginalMetadataNames = names(inputFileData[[c_strMatrixMetadata]])
303 lsOriginalFeatureNames = names(inputFileData[[c_strMatrixData]])
304
305 #Dimensions of the datasets
306 liMetaData = dim(inputFileData[[c_strMatrixMetadata]])
307 liData = dim(inputFileData[[c_strMatrixData]])
308
309 #Merge data files together
310 frmeData = merge(inputFileData[[c_strMatrixMetadata]],inputFileData[[c_strMatrixData]],by.x=0,by.y=0)
311 #Reset rownames
312 row.names(frmeData) = frmeData[[1]]
313 frmeData = frmeData[-1]
314
315 #Write QC files only in certain modes of verbosity
316 # Read in and merge files
317 if( c_logrMaaslin$level <= loglevels["DEBUG"] ) {
318 # If the QC internal file does not exist, make
319 strQCDir = file.path(outputDirectory,"QC")
320 dir.create(strQCDir, showWarnings = FALSE)
321 # Write metadata matrix before merge
322 funcWriteMatrices(dataFrameList=list(Metadata = inputFileData[[c_strMatrixMetadata]]), saveFileList=c(file.path(strQCDir,"metadata.tsv")), configureFileName=c(file.path(strQCDir,"metadata.read.config")), acharDelimiter="\t")
323 # Write data matrix before merge
324 funcWriteMatrices(dataFrameList=list(Data = inputFileData[[c_strMatrixData]]), saveFileList=c(file.path(strQCDir,"data.tsv")), configureFileName=c(file.path(strQCDir,"data.read.config")), acharDelimiter="\t")
325 #Record the data as it has been read
326 funcWriteMatrices(dataFrameList=list(Merged = frmeData), saveFileList=c(file.path(strQCDir,"read-Merged.tsv")), configureFileName=c(file.path(strQCDir,"read-Merged.read.config")), acharDelimiter="\t")
327 }
328
329 #Data needed for the MaAsLin environment
330 #List of lists (one entry per file)
331 #Is contained by a container of itself
332 #lslsData = list()
333 #List
334 lsData = c()
335
336 #List of metadata indicies
337 aiMetadata = c(1:liMetaData[2])
338 lsData$aiMetadata = aiMetadata
339 #List of data indicies
340 aiData = c(1:liData[2])+liMetaData[2]
341 lsData$aiData = aiData
342 #Add a list to hold qc metrics and counts
343 lsData$lsQCCounts$aiDataInitial = aiData
344 lsData$lsQCCounts$aiMetadataInitial = aiMetadata
345
346 #Raw data
347 lsData$frmeRaw = frmeData
348
349 #Load script if it exists, stop on error
350 funcProcess <- NULL
351 if(!is.null(funcSourceScript(lsArgs$options$strInputR))){funcProcess <- get(c_strCustomProcessFunction)}
352
353 #Clean the data and update the current data list to the cleaned data list
354 funcTransformData = afuncVariableAnalysis[[c_iTransform]]
355 lsQCCounts = list(aiDataCleaned = c(), aiMetadataCleaned = c())
356 lsRet = list(frmeData=frmeData, aiData=aiData, aiMetadata=aiMetadata, lsQCCounts=lsQCCounts, liNaIndices=c())
357
358 viNotTransformedDataIndices = c()
359 if(!lsArgs$options$fNoQC)
360 {
361 c_logrMaaslin$info( "Running quality control." )
362 lsRet = funcClean( frmeData=frmeData, funcDataProcess=funcProcess, aiMetadata=aiMetadata, aiData=aiData, lsQCCounts=lsData$lsQCCounts, astrNoImpute=xNoImpute, dMinSamp = lsArgs$options$dMinSamp, dMinAbd = lsArgs$options$dMinAbd, dFence=lsArgs$options$dOutlierFence, funcTransform=funcTransformData, dPOutlier=lsArgs$options$dPOutlier)
363
364 viNotTransformedDataIndices = lsRet$viNotTransformedData
365
366 #If using a count based model make sure all are integer (QCing can add in numeric values during interpolation for example)
367 if(lsArgs$options$strMethod %in% c_vCountBasedModels)
368 {
369 c_logrMaaslin$info( "Assuring the data matrix is integer." )
370 for(iDataIndex in aiData)
371 {
372 lsRet$frmeData[ iDataIndex ] = round( lsRet$frmeData[ iDataIndex ] )
373 }
374 }
375 } else {
376 c_logrMaaslin$info( "Not running quality control, attempting transform." )
377 ### Need to do transform if the QC is not performed
378 iTransformed = 0
379 for(iDataIndex in aiData)
380 {
381 if( ! funcTransformIncreasesOutliers( lsRet$frmeData[iDataIndex], funcTransformData ) )
382 {
383 lsRet$frmeData[iDataIndex]=funcTransformData(lsRet$frmeData[iDataIndex])
384 iTransformed = iTransformed + 1
385 } else {
386 viNotTransformedDataIndices = c(viNotTransformedDataIndices, iDataIndex)
387 }
388 }
389 c_logrMaaslin$info(paste("Number of features transformed = ", iTransformed))
390 }
391
392 logdebug("lsRet", c_logrMaaslin)
393 logdebug(format(lsRet), c_logrMaaslin)
394 #Update the variables after cleaning
395 lsRet$frmeRaw = frmeData
396 lsRet$lsQCCounts$aiDataCleaned = lsRet$aiData
397 lsRet$lsQCCounts$aiMetadataCleaned = lsRet$aiMetadata
398
399 #Add List of metadata string names
400 astrMetadata = colnames(lsRet$frmeData)[lsRet$aiMetadata]
401 lsRet$astrMetadata = astrMetadata
402
403 # If plotting NA data reset the NA metadata indices to empty so they will not be excluded
404 if(lsArgs$options$fPlotNA)
405 {
406 lsRet$liNaIndices = list()
407 }
408
409 #Write QC files only in certain modes of verbosity
410 if( c_logrMaaslin$level <= loglevels["DEBUG"] ) {
411 #Record the data after cleaning
412 funcWriteMatrices(dataFrameList=list(Cleaned = lsRet$frmeData[union(lsRet$aiMetadata,lsRet$aiData)]), saveFileList=c(file.path(strQCDir,"read_cleaned.tsv")), configureFileName=c(file.path(strQCDir,"read_cleaned.read.config")), acharDelimiter="\t") }
413
414 #These variables will be used to count how many features get analysed
415 lsRet$lsQCCounts$iBoosts = 0
416 lsRet$lsQCCounts$iBoostErrors = 0
417 lsRet$lsQCCounts$iNoTerms = 0
418 lsRet$lsQCCounts$iLms = 0
419
420 #Indicate if the residuals plots should occur
421 fDoRPlot=TRUE
422 #Should not occur for univariates
423 if(lsArgs$options$strMethod %in% c("univariate")){ fDoRPlot=FALSE }
424
425 #Run analysis
426 alsRetBugs = funcBugs( frmeData=lsRet$frmeData, lsData=lsRet, aiMetadata=lsRet$aiMetadata, aiData=lsRet$aiData, aiNotTransformedData=viNotTransformedDataIndices, strData=strBase, dSig=lsArgs$options$dSignificanceLevel, fInvert=lsArgs$options$fInvert,
427 strDirOut=outputDirectory, funcReg=afuncVariableAnalysis[[c_iSelection]], funcTransform=funcTransformData, funcUnTransform=afuncVariableAnalysis[[c_iUnTransform]], lsNonPenalizedPredictors=lsForcedParameters,
428 funcAnalysis=afuncVariableAnalysis[[c_iAnalysis]], lsRandomCovariates=lsRandomCovariates, funcGetResults=afuncVariableAnalysis[[c_iResults]], fDoRPlot=fDoRPlot, fOmitLogFile=lsArgs$options$fOmitLogFile,
429 fAllvAll=lsArgs$options$fAllvAll, liNaIndices=lsRet$liNaIndices, lxParameters=lxParameters, strTestingCorrection=lsArgs$options$strMultTestCorrection,
430 fIsUnivariate=afuncVariableAnalysis[[c_iIsUnivariate]], fZeroInflated=lsArgs$options$fZeroInflated )
431
432 #Write QC files only in certain modes of verbosity
433 if( c_logrMaaslin$level <= loglevels["DEBUG"] ) {
434 funcWriteQCReport(strProcessFileName=file.path(strQCDir,"ProcessQC.txt"), lsQCData=alsRetBugs$lsQCCounts, liDataDim=liData, liMetadataDim=liMetaData)
435
436 ### Write out the parameters used in the run
437 unlink(file.path(strQCDir,"Run_Parameters.txt"))
438 funcWrite("Parameters used in the MaAsLin run", file.path(strQCDir,"Run_Parameters.txt"))
439 funcWrite(paste("Optional input read.config file=",lsArgs$options$strInputConfig), file.path(strQCDir,"Run_Parameters.txt"))
440 funcWrite(paste("Optional R file=",lsArgs$options$strInputR), file.path(strQCDir,"Run_Parameters.txt"))
441 funcWrite(paste("FDR threshold for pdf generation=",lsArgs$options$dSignificanceLevel), file.path(strQCDir,"Run_Parameters.txt"))
442 funcWrite(paste("Minimum relative abundance=",lsArgs$options$dMinAbd), file.path(strQCDir,"Run_Parameters.txt"))
443 funcWrite(paste("Minimum percentage of samples with measurements=",lsArgs$options$dMinSamp), file.path(strQCDir,"Run_Parameters.txt"))
444 funcWrite(paste("The fence used to define outliers with a quantile based analysis. If set to 0, the Grubbs test was used=",lsArgs$options$dOutlierFence), file.path(strQCDir,"Run_Parameters.txt"))
445 funcWrite(paste("Ignore if the Grubbs test was not used. The significance level used as a cut-off to define outliers=",lsArgs$options$dPOutlier), file.path(strQCDir,"Run_Parameters.txt"))
446 funcWrite(paste("These covariates are treated as random covariates and not fixed covariates=",lsArgs$options$strRandomCovariates), file.path(strQCDir,"Run_Parameters.txt"))
447 funcWrite(paste("The type of multiple testing correction used=",lsArgs$options$strMultTestCorrection), file.path(strQCDir,"Run_Parameters.txt"))
448 funcWrite(paste("Zero inflated inference models were turned on=",lsArgs$options$fZeroInflated), file.path(strQCDir,"Run_Parameters.txt"))
449 funcWrite(paste("Feature selection step=",lsArgs$options$strModelSelection), file.path(strQCDir,"Run_Parameters.txt"))
450 funcWrite(paste("Statistical inference step=",lsArgs$options$strMethod), file.path(strQCDir,"Run_Parameters.txt"))
451 funcWrite(paste("Numeric transform used=",lsArgs$options$strTransform), file.path(strQCDir,"Run_Parameters.txt"))
452 funcWrite(paste("Quality control was run=",!lsArgs$options$fNoQC), file.path(strQCDir,"Run_Parameters.txt"))
453 funcWrite(paste("These covariates were forced into each model=",lsArgs$options$strForcedPredictors), file.path(strQCDir,"Run_Parameters.txt"))
454 funcWrite(paste("These features' data were not changed by QC processes=",lsArgs$options$strNoImpute), file.path(strQCDir,"Run_Parameters.txt"))
455 funcWrite(paste("Output verbosity=",lsArgs$options$strVerbosity), file.path(strQCDir,"Run_Parameters.txt"))
456 funcWrite(paste("Log file was generated=",!lsArgs$options$fOmitLogFile), file.path(strQCDir,"Run_Parameters.txt"))
457 funcWrite(paste("Data plots were inverted=",lsArgs$options$fInvert), file.path(strQCDir,"Run_Parameters.txt"))
458 funcWrite(paste("Ignore unless boosting was used. The threshold for the rel.inf used to select features=",lsArgs$options$dSelectionFrequency), file.path(strQCDir,"Run_Parameters.txt"))
459 funcWrite(paste("All verses all inference method was used=",lsArgs$options$fAllvAll), file.path(strQCDir,"Run_Parameters.txt"))
460 funcWrite(paste("Ignore unless penalized feature selection was used. Alpha to determine the type of penalty=",lsArgs$options$dPenalizedAlpha), file.path(strQCDir,"Run_Parameters.txt"))
461 funcWrite(paste("Biplot parameter, user defined metadata scale=",lsArgs$options$dBiplotMetadataScale), file.path(strQCDir,"Run_Parameters.txt"))
462 funcWrite(paste("Biplot parameter, user defined metadata used to color the plot=",lsArgs$options$strBiplotColor), file.path(strQCDir,"Run_Parameters.txt"))
463 funcWrite(paste("Biplot parameter, user defined metadata used to dictate the shapes of the plot markers=",lsArgs$options$strBiplotShapeBy), file.path(strQCDir,"Run_Parameters.txt"))
464 funcWrite(paste("Biplot parameter, user defined user requested features to plot=",lsArgs$options$strBiplotPlotFeatures), file.path(strQCDir,"Run_Parameters.txt"))
465 funcWrite(paste("Biplot parameter, user defined metadata used to rotate the plot ordination=",lsArgs$options$sRotateByMetadata), file.path(strQCDir,"Run_Parameters.txt"))
466 funcWrite(paste("Biplot parameter, user defined custom shapes for metadata=",lsArgs$options$sShapes), file.path(strQCDir,"Run_Parameters.txt"))
467 funcWrite(paste("Biplot parameter, user defined number of bugs to plot =",lsArgs$options$iNumberBugs), file.path(strQCDir,"Run_Parameters.txt"))
468 }
469
470 ### Write summary table
471 # Summarize output files based on a keyword and a significance threshold
472 # Look for less than or equal to the threshold (appropriate for p-value and q-value type measurements)
473 # DfSummary is sorted by the q.value when it is returned
474 dfSummary = funcSummarizeDirectory(astrOutputDirectory=outputDirectory,
475 strBaseName=strBase,
476 astrSummaryFileName=file.path(outputDirectory,paste(strBase,c_sSummaryFileSuffix, sep="")),
477 astrKeyword=c_strKeywordEvaluatedForInclusion,
478 afSignificanceLevel=lsArgs$options$dSignificanceLevel)
479
480 if( !is.null( dfSummary ) )
481 {
482 ### Start biplot
483 # Get metadata of interest and reduce to default size
484 lsSigMetadata = unique(dfSummary[[1]])
485 if( is.null( lsArgs$options$iNumberMetadata ) )
486 {
487 lsSigMetadata = lsSigMetadata[ 1:length( lsSigMetadata ) ]
488 } else {
489 lsSigMetadata = lsSigMetadata[ 1:min( length( lsSigMetadata ), max( lsArgs$options$iNumberMetadata, 1 ) ) ]
490 }
491
492 # Convert to indices (ordered numerically here)
493 liSigMetadata = which( colnames( lsRet$frmeData ) %in% lsSigMetadata )
494
495 # Get bugs of interest and reduce to default size
496 lsSigBugs = unique(dfSummary[[2]])
497
498 # Reduce the bugs to the right size
499 if(lsArgs$options$iNumberBugs < 1)
500 {
501 lsSigBugs = c()
502 } else if( is.null( lsArgs$options$iNumberBugs ) ) {
503 lsSigBugs = lsSigBugs[ 1 : length( lsSigBugs ) ]
504 } else {
505 lsSigBugs = lsSigBugs[ 1 : lsArgs$options$iNumberBugs ]
506 }
507
508 # Set color by and shape by features if not given
509 # Selects the continuous (for color) and factor (for shape) data with the most significant association
510 if(is.null(lsArgs$options$strBiplotColor)||is.null(lsArgs$options$strBiplotShapeBy))
511 {
512 for(sMetadata in lsSigMetadata)
513 {
514 if(is.factor(lsRet$frmeRaw[[sMetadata]]))
515 {
516 if(is.null(lsArgs$options$strBiplotShapeBy))
517 {
518 lsArgs$options$strBiplotShapeBy = sMetadata
519 if(!is.null(lsArgs$options$strBiplotColor))
520 {
521 break
522 }
523 }
524 }
525 if(is.numeric(lsRet$frmeRaw[[sMetadata]]))
526 {
527 if(is.null(lsArgs$options$strBiplotColor))
528 {
529 lsArgs$options$strBiplotColor = sMetadata
530 if(!is.null(lsArgs$options$strBiplotShapeBy))
531 {
532 break
533 }
534 }
535 }
536 }
537 }
538
539 #If a user defines a feature, make sure it is in the bugs/data indices
540 if(!is.null(lsFeaturesToPlot) || !is.null(lsArgs$options$strBiplotColor) || !is.null(lsArgs$options$strBiplotShapeBy))
541 {
542 lsCombinedFeaturesToPlot = unique(c(lsFeaturesToPlot,lsArgs$options$strBiplotColor,lsArgs$options$strBiplotShapeBy))
543 lsCombinedFeaturesToPlot = lsCombinedFeaturesToPlot[!is.null(lsCombinedFeaturesToPlot)]
544
545 # If bugs to plot were given then do not use the significant bugs from the MaAsLin output which is default
546 if(!is.null(lsFeaturesToPlot))
547 {
548 lsSigBugs = c()
549 liSigMetadata = c()
550 }
551 liSigMetadata = unique(c(liSigMetadata,which(colnames(lsRet$frmeData) %in% setdiff(lsCombinedFeaturesToPlot, lsOriginalFeatureNames))))
552 lsSigBugs = unique(c(lsSigBugs, intersect(lsCombinedFeaturesToPlot, lsOriginalFeatureNames)))
553 }
554
555 # Convert bug names and metadata names to comma delimited strings
556 vsBugs = paste(lsSigBugs,sep=",",collapse=",")
557 vsMetadata = paste(colnames(lsRet$frmeData)[liSigMetadata],sep=",",collapse=",")
558 vsMetadataByLevel = c()
559
560 # Possibly remove the NA levels depending on the preferences
561 vsRemoveNA = c(NA, "NA", "na", "Na", "nA")
562 if(!lsArgs$options$fPlotNA){ vsRemoveNA = c() }
563 for(aiMetadataIndex in liSigMetadata)
564 {
565 lxCurMetadata = lsRet$frmeData[[aiMetadataIndex]]
566 sCurName = names(lsRet$frmeData[aiMetadataIndex])
567 if(is.factor(lxCurMetadata))
568 {
569 vsMetadataByLevel = c(vsMetadataByLevel,paste(sCurName, setdiff( levels(lxCurMetadata), vsRemoveNA),sep="_"))
570 } else {
571 vsMetadataByLevel = c(vsMetadataByLevel,sCurName)
572 }
573 }
574
575 # If NAs should not be plotted, make them the background color
576 # Unless explicitly asked to be plotted
577 sPlotNAColor = "white"
578 if(lsArgs$options$fInvert){sPlotNAColor = "black"}
579 if(lsArgs$options$fPlotNA){sPlotNAColor = "grey"}
580 sLastMetadata = lsOriginalMetadataNames[max(which(lsOriginalMetadataNames %in% names(lsRet$frmeData)))]
581
582 # Plot biplot
583 logdebug("PlotBiplot:Started")
584 funcDoBiplot(
585 sBugs = vsBugs,
586 sMetadata = vsMetadataByLevel,
587 sColorBy = lsArgs$options$strBiplotColor,
588 sPlotNAColor = sPlotNAColor,
589 sShapeBy = lsArgs$options$strBiplotShapeBy,
590 sShapes = lsArgs$options$sShapes,
591 sDefaultMarker = "16",
592 sRotateByMetadata = lsArgs$options$sRotateByMetadata,
593 dResizeArrow = lsArgs$options$dBiplotMetadataScale,
594 sInputFileName = lsRet$frmeRaw,
595 sLastMetadata = sLastMetadata,
596 sOutputFileName = file.path(outputDirectory,paste(strBase,"-biplot.pdf",sep="")))
597 logdebug("PlotBiplot:Stopped")
598 }
599 }
600
601 # This is the equivalent of __name__ == "__main__" in Python.
602 # That is, if it's true we're being called as a command line script;
603 # if it's false, we're being sourced or otherwise included, such as for
604 # library or inlinedocs.
605 if( identical( environment( ), globalenv( ) ) &&
606 !length( grep( "^source\\(", sys.calls( ) ) ) ) {
607 main( pArgs ) }