|
542
|
1 from __future__ import division
|
|
|
2 import csv
|
|
|
3 from enum import Enum
|
|
|
4 import re
|
|
|
5 import sys
|
|
|
6 import numpy as np
|
|
|
7 import pandas as pd
|
|
|
8 import itertools as it
|
|
|
9 import scipy.stats as st
|
|
|
10 import lxml.etree as ET
|
|
|
11 import math
|
|
|
12 try:
|
|
|
13 from .utils import general_utils as utils
|
|
|
14 except:
|
|
|
15 import utils.general_utils as utils
|
|
|
16 from PIL import Image
|
|
|
17 import os
|
|
|
18 import copy
|
|
|
19 import argparse
|
|
|
20 import pyvips
|
|
|
21 from PIL import Image
|
|
|
22 from typing import Tuple, Union, Optional, List, Dict
|
|
|
23 import matplotlib.pyplot as plt
|
|
543
|
24 import os
|
|
542
|
25
|
|
|
26 ERRORS = []
|
|
|
27 ########################## argparse ##########################################
|
|
|
28 ARGS :argparse.Namespace
|
|
|
29 def process_args(args:List[str] = None) -> argparse.Namespace:
|
|
|
30 """
|
|
|
31 Interfaces the script of a module with its frontend, making the user's choices for various parameters available as values in code.
|
|
|
32
|
|
|
33 Args:
|
|
|
34 args : Always obtained (in file) from sys.argv
|
|
|
35
|
|
|
36 Returns:
|
|
|
37 Namespace : An object containing the parsed arguments
|
|
|
38 """
|
|
|
39 parser = argparse.ArgumentParser(
|
|
|
40 usage = "%(prog)s [options]",
|
|
|
41 description = "process some value's genes to create a comparison's map.")
|
|
|
42
|
|
|
43 #General:
|
|
|
44 parser.add_argument(
|
|
|
45 '-td', '--tool_dir',
|
|
|
46 type = str,
|
|
|
47 default = os.path.dirname(os.path.abspath(__file__)),
|
|
|
48 help = 'your tool directory (default: auto-detected package location)')
|
|
|
49
|
|
|
50 parser.add_argument('-on', '--control', type = str)
|
|
|
51 parser.add_argument('-ol', '--out_log', help = "Output log")
|
|
|
52
|
|
|
53 #Computation details:
|
|
|
54 parser.add_argument(
|
|
|
55 '-co', '--comparison',
|
|
|
56 type = str,
|
|
|
57 default = 'manyvsmany',
|
|
|
58 choices = ['manyvsmany', 'onevsrest', 'onevsmany'])
|
|
|
59
|
|
|
60 parser.add_argument(
|
|
|
61 '-te' ,'--test',
|
|
|
62 type = str,
|
|
|
63 default = 'ks',
|
|
|
64 choices = ['ks', 'ttest_p', 'ttest_ind', 'wilcoxon', 'mw'],
|
|
|
65 help = 'Statistical test to use (default: %(default)s)')
|
|
|
66
|
|
|
67 parser.add_argument(
|
|
|
68 '-pv' ,'--pValue',
|
|
|
69 type = float,
|
|
|
70 default = 0.1,
|
|
|
71 help = 'P-Value threshold (default: %(default)s)')
|
|
|
72
|
|
|
73 parser.add_argument(
|
|
|
74 '-adj' ,'--adjusted',
|
|
|
75 type = utils.Bool("adjusted"), default = False,
|
|
|
76 help = 'Apply the FDR (Benjamini-Hochberg) correction (default: %(default)s)')
|
|
|
77
|
|
|
78 parser.add_argument(
|
|
|
79 '-fc', '--fChange',
|
|
|
80 type = float,
|
|
|
81 default = 1.5,
|
|
|
82 help = 'Fold-Change threshold (default: %(default)s)')
|
|
|
83
|
|
|
84 parser.add_argument(
|
|
|
85 '-op', '--option',
|
|
|
86 type = str,
|
|
|
87 choices = ['datasets', 'dataset_class'],
|
|
|
88 help='dataset or dataset and class')
|
|
|
89
|
|
|
90 parser.add_argument(
|
|
|
91 '-idf', '--input_data_fluxes',
|
|
|
92 type = str,
|
|
|
93 help = 'input dataset fluxes')
|
|
|
94
|
|
|
95 parser.add_argument(
|
|
|
96 '-icf', '--input_class_fluxes',
|
|
|
97 type = str,
|
|
|
98 help = 'sample group specification fluxes')
|
|
|
99
|
|
|
100 parser.add_argument(
|
|
|
101 '-idsf', '--input_datas_fluxes',
|
|
|
102 type = str,
|
|
|
103 nargs = '+',
|
|
|
104 help = 'input datasets fluxes')
|
|
|
105
|
|
|
106 parser.add_argument(
|
|
|
107 '-naf', '--names_fluxes',
|
|
|
108 type = str,
|
|
|
109 nargs = '+',
|
|
|
110 help = 'input names fluxes')
|
|
|
111
|
|
|
112 #Output:
|
|
|
113 parser.add_argument(
|
|
|
114 "-gs", "--generate_svg",
|
|
|
115 type = utils.Bool("generate_svg"), default = True,
|
|
|
116 help = "choose whether to generate svg")
|
|
|
117
|
|
|
118 parser.add_argument(
|
|
|
119 "-gp", "--generate_pdf",
|
|
|
120 type = utils.Bool("generate_pdf"), default = True,
|
|
|
121 help = "choose whether to generate pdf")
|
|
|
122
|
|
|
123 parser.add_argument(
|
|
|
124 '-cm', '--custom_map',
|
|
|
125 type = str,
|
|
|
126 help='custom map to use')
|
|
|
127
|
|
|
128 parser.add_argument(
|
|
|
129 '-mc', '--choice_map',
|
|
|
130 type = utils.Model, default = utils.Model.HMRcore,
|
|
|
131 choices = [utils.Model.HMRcore, utils.Model.ENGRO2, utils.Model.Custom])
|
|
|
132
|
|
|
133 parser.add_argument(
|
|
|
134 '-colorm', '--color_map',
|
|
|
135 type = str,
|
|
|
136 choices = ["jet", "viridis"])
|
|
|
137
|
|
|
138 parser.add_argument(
|
|
|
139 '-idop', '--output_path',
|
|
|
140 type = str,
|
|
|
141 default='result',
|
|
|
142 help = 'output path for maps')
|
|
|
143
|
|
|
144 args :argparse.Namespace = parser.parse_args(args)
|
|
|
145 args.net = True # TODO SICCOME I FLUSSI POSSONO ESSERE ANCHE NEGATIVI SONO SEMPRE CONSIDERATI NETTI
|
|
|
146
|
|
|
147 return args
|
|
|
148
|
|
|
149 ############################ dataset input ####################################
|
|
|
150 def read_dataset(data :str, name :str) -> pd.DataFrame:
|
|
|
151 """
|
|
|
152 Tries to read the dataset from its path (data) as a tsv and turns it into a DataFrame.
|
|
|
153
|
|
|
154 Args:
|
|
|
155 data : filepath of a dataset (from frontend input params or literals upon calling)
|
|
|
156 name : name associated with the dataset (from frontend input params or literals upon calling)
|
|
|
157
|
|
|
158 Returns:
|
|
|
159 pd.DataFrame : dataset in a runtime operable shape
|
|
|
160
|
|
|
161 Raises:
|
|
|
162 sys.exit : if there's no data (pd.errors.EmptyDataError) or if the dataset has less than 2 columns
|
|
|
163 """
|
|
|
164 try:
|
|
|
165 dataset = pd.read_csv(data, sep = '\t', header = 0, engine='python')
|
|
|
166 except pd.errors.EmptyDataError:
|
|
|
167 sys.exit('Execution aborted: wrong format of ' + name + '\n')
|
|
|
168 if len(dataset.columns) < 2:
|
|
|
169 sys.exit('Execution aborted: wrong format of ' + name + '\n')
|
|
|
170 return dataset
|
|
|
171
|
|
|
172 ############################ dataset name #####################################
|
|
|
173 def name_dataset(name_data :str, count :int) -> str:
|
|
|
174 """
|
|
|
175 Produces a unique name for a dataset based on what was provided by the user. The default name for any dataset is "Dataset", thus if the user didn't change it this function appends f"_{count}" to make it unique.
|
|
|
176
|
|
|
177 Args:
|
|
|
178 name_data : name associated with the dataset (from frontend input params)
|
|
|
179 count : counter from 1 to make these names unique (external)
|
|
|
180
|
|
|
181 Returns:
|
|
|
182 str : the name made unique
|
|
|
183 """
|
|
|
184 if str(name_data) == 'Dataset':
|
|
|
185 return str(name_data) + '_' + str(count)
|
|
|
186 else:
|
|
|
187 return str(name_data)
|
|
|
188
|
|
|
189 ############################ map_methods ######################################
|
|
|
190 FoldChange = Union[float, int, str] # Union[float, Literal[0, "-INF", "INF"]]
|
|
|
191 def fold_change(avg1 :float, avg2 :float) -> FoldChange:
|
|
|
192 """
|
|
|
193 Calculates the fold change between two gene expression values.
|
|
|
194
|
|
|
195 Args:
|
|
|
196 avg1 : average expression value from one dataset avg2 : average expression value from the other dataset
|
|
|
197
|
|
|
198 Returns:
|
|
|
199 FoldChange :
|
|
|
200 0 : when both input values are 0
|
|
|
201 "-INF" : when avg1 is 0
|
|
|
202 "INF" : when avg2 is 0
|
|
|
203 float : for any other combination of values
|
|
|
204 """
|
|
|
205 if avg1 == 0 and avg2 == 0:
|
|
|
206 return 0
|
|
|
207 elif avg1 == 0:
|
|
|
208 return '-INF'
|
|
|
209 elif avg2 == 0:
|
|
|
210 return 'INF'
|
|
|
211 else: # (threshold_F_C - 1) / (abs(threshold_F_C) + 1) con threshold_F_C > 1
|
|
|
212 return (avg1 - avg2) / (abs(avg1) + abs(avg2))
|
|
|
213
|
|
|
214 def getElementById(reactionId :str, metabMap :ET.ElementTree) -> utils.Result[ET.Element, utils.Result.ResultErr]:
|
|
|
215 """
|
|
|
216 Finds any element in the given map with the given ID. ID uniqueness in an svg file is recommended but
|
|
|
217 not enforced, if more than one element with the exact ID is found only the first will be returned.
|
|
|
218
|
|
|
219 Args:
|
|
|
220 reactionId (str): exact ID of the requested element.
|
|
|
221 metabMap (ET.ElementTree): metabolic map containing the element.
|
|
|
222
|
|
|
223 Returns:
|
|
|
224 utils.Result[ET.Element, ResultErr]: result of the search, either the first match found or a ResultErr.
|
|
|
225 """
|
|
|
226 return utils.Result.Ok(
|
|
|
227 f"//*[@id=\"{reactionId}\"]").map(
|
|
|
228 lambda xPath : metabMap.xpath(xPath)[0]).mapErr(
|
|
|
229 lambda _ : utils.Result.ResultErr(f"No elements with ID \"{reactionId}\" found in map"))
|
|
|
230 # ^^^ we shamelessly ignore the contents of the IndexError, it offers nothing to the user.
|
|
|
231
|
|
|
232 def styleMapElement(element :ET.Element, styleStr :str) -> None:
|
|
|
233 currentStyles :str = element.get("style", "")
|
|
|
234 if re.search(r";stroke:[^;]+;stroke-width:[^;]+;stroke-dasharray:[^;]+$", currentStyles):
|
|
|
235 currentStyles = ';'.join(currentStyles.split(';')[:-3])
|
|
|
236
|
|
|
237 element.set("style", currentStyles + styleStr)
|
|
|
238
|
|
|
239 class ReactionDirection(Enum):
|
|
|
240 Unknown = ""
|
|
|
241 Direct = "_F"
|
|
|
242 Inverse = "_B"
|
|
|
243
|
|
|
244 @classmethod
|
|
|
245 def fromDir(cls, s :str) -> "ReactionDirection":
|
|
|
246 # vvv as long as there's so few variants I actually condone the if spam:
|
|
|
247 if s == ReactionDirection.Direct.value: return ReactionDirection.Direct
|
|
|
248 if s == ReactionDirection.Inverse.value: return ReactionDirection.Inverse
|
|
|
249 return ReactionDirection.Unknown
|
|
|
250
|
|
|
251 @classmethod
|
|
|
252 def fromReactionId(cls, reactionId :str) -> "ReactionDirection":
|
|
|
253 return ReactionDirection.fromDir(reactionId[-2:])
|
|
|
254
|
|
|
255 def getArrowBodyElementId(reactionId :str) -> str:
|
|
|
256 if reactionId.endswith("_RV"): reactionId = reactionId[:-3] #TODO: standardize _RV
|
|
|
257 elif ReactionDirection.fromReactionId(reactionId) is not ReactionDirection.Unknown: reactionId = reactionId[:-2]
|
|
|
258 return f"R_{reactionId}"
|
|
|
259
|
|
|
260 def getArrowHeadElementId(reactionId :str) -> Tuple[str, str]:
|
|
|
261 """
|
|
|
262 We attempt extracting the direction information from the provided reaction ID, if unsuccessful we provide the IDs of both directions.
|
|
|
263
|
|
|
264 Args:
|
|
|
265 reactionId : the provided reaction ID.
|
|
|
266
|
|
|
267 Returns:
|
|
|
268 Tuple[str, str]: either a single str ID for the correct arrow head followed by an empty string or both options to try.
|
|
|
269 """
|
|
|
270 if reactionId.endswith("_RV"): reactionId = reactionId[:-3] #TODO: standardize _RV
|
|
|
271 elif ReactionDirection.fromReactionId(reactionId) is not ReactionDirection.Unknown: return reactionId[:-3:-1] + reactionId[:-2], ""
|
|
|
272 return f"F_{reactionId}", f"B_{reactionId}"
|
|
|
273
|
|
|
274 class ArrowColor(Enum):
|
|
|
275 """
|
|
|
276 Encodes possible arrow colors based on their meaning in the enrichment process.
|
|
|
277 """
|
|
|
278 Invalid = "#BEBEBE" # gray, fold-change under treshold or not significant p-value
|
|
|
279 Transparent = "#ffffff00" # transparent, to make some arrow segments disappear
|
|
|
280 UpRegulated = "#ecac68" # red, up-regulated reaction
|
|
|
281 DownRegulated = "#6495ed" # blue, down-regulated reaction
|
|
|
282
|
|
|
283 UpRegulatedInv = "#FF0000"
|
|
|
284 # ^^^ different shade of red (actually orange), up-regulated net value for a reversible reaction with
|
|
|
285 # conflicting enrichment in the two directions.
|
|
|
286
|
|
|
287 DownRegulatedInv = "#0000FF"
|
|
|
288 # ^^^ different shade of blue (actually purple), down-regulated net value for a reversible reaction with
|
|
|
289 # conflicting enrichment in the two directions.
|
|
|
290
|
|
|
291 @classmethod
|
|
|
292 def fromFoldChangeSign(cls, foldChange :float, *, useAltColor = False) -> "ArrowColor":
|
|
|
293 colors = (cls.DownRegulated, cls.DownRegulatedInv) if foldChange < 0 else (cls.UpRegulated, cls.UpRegulatedInv)
|
|
|
294 return colors[useAltColor]
|
|
|
295
|
|
|
296 def __str__(self) -> str: return self.value
|
|
|
297
|
|
|
298 class Arrow:
|
|
|
299 """
|
|
|
300 Models the properties of a reaction arrow that change based on enrichment.
|
|
|
301 """
|
|
|
302 MIN_W = 2
|
|
|
303 MAX_W = 12
|
|
|
304
|
|
|
305 def __init__(self, width :int, col: ArrowColor, *, isDashed = False) -> None:
|
|
|
306 """
|
|
|
307 (Private) Initializes an instance of Arrow.
|
|
|
308
|
|
|
309 Args:
|
|
|
310 width : width of the arrow, ideally to be kept within Arrow.MIN_W and Arrow.MAX_W (not enforced).
|
|
|
311 col : color of the arrow.
|
|
|
312 isDashed : whether the arrow should be dashed, meaning the associated pValue resulted not significant.
|
|
|
313
|
|
|
314 Returns:
|
|
|
315 None : practically, a Arrow instance.
|
|
|
316 """
|
|
|
317 self.w = width
|
|
|
318 self.col = col
|
|
|
319 self.dash = isDashed
|
|
|
320
|
|
|
321 def applyTo(self, reactionId :str, metabMap :ET.ElementTree, styleStr :str) -> None:
|
|
|
322 if getElementById(reactionId, metabMap).map(lambda el : styleMapElement(el, styleStr)).isErr:
|
|
|
323 ERRORS.append(reactionId)
|
|
|
324
|
|
|
325 def styleReactionElements(self, metabMap :ET.ElementTree, reactionId :str, *, mindReactionDir = True) -> None:
|
|
|
326 if not mindReactionDir:
|
|
|
327 return self.applyTo(getArrowBodyElementId(reactionId), metabMap, self.toStyleStr())
|
|
|
328
|
|
|
329 # Now we style the arrow head(s):
|
|
|
330 idOpt1, idOpt2 = getArrowHeadElementId(reactionId)
|
|
|
331 self.applyTo(idOpt1, metabMap, self.toStyleStr(downSizedForTips = True))
|
|
|
332 if idOpt2: self.applyTo(idOpt2, metabMap, self.toStyleStr(downSizedForTips = True))
|
|
|
333
|
|
|
334 def styleReactionElementsMeanMedian(self, metabMap :ET.ElementTree, reactionId :str, isNegative:bool) -> None:
|
|
|
335
|
|
|
336 self.applyTo(getArrowBodyElementId(reactionId), metabMap, self.toStyleStr())
|
|
|
337 idOpt1, idOpt2 = getArrowHeadElementId(reactionId)
|
|
|
338
|
|
|
339 if(isNegative):
|
|
|
340 self.applyTo(idOpt2, metabMap, self.toStyleStr(downSizedForTips = True))
|
|
|
341 self.col = ArrowColor.Transparent
|
|
|
342 self.applyTo(idOpt1, metabMap, self.toStyleStr(downSizedForTips = True)) #trasp
|
|
|
343 else:
|
|
|
344 self.applyTo(idOpt1, metabMap, self.toStyleStr(downSizedForTips = True))
|
|
|
345 self.col = ArrowColor.Transparent
|
|
|
346 self.applyTo(idOpt2, metabMap, self.toStyleStr(downSizedForTips = True)) #trasp
|
|
|
347
|
|
|
348
|
|
|
349
|
|
|
350 def getMapReactionId(self, reactionId :str, mindReactionDir :bool) -> str:
|
|
|
351 """
|
|
|
352 Computes the reaction ID as encoded in the map for a given reaction ID from the dataset.
|
|
|
353
|
|
|
354 Args:
|
|
|
355 reactionId: the reaction ID, as encoded in the dataset.
|
|
|
356 mindReactionDir: if True forward (F_) and backward (B_) directions will be encoded in the result.
|
|
|
357
|
|
|
358 Returns:
|
|
|
359 str : the ID of an arrow's body or tips in the map.
|
|
|
360 """
|
|
|
361 # we assume the reactionIds also don't encode reaction dir if they don't mind it when styling the map.
|
|
|
362 if not mindReactionDir: return "R_" + reactionId
|
|
|
363
|
|
|
364 #TODO: this is clearly something we need to make consistent in fluxes
|
|
|
365 return (reactionId[:-3:-1] + reactionId[:-2]) if reactionId[:-2] in ["_F", "_B"] else f"F_{reactionId}" # "Pyr_F" --> "F_Pyr"
|
|
|
366
|
|
|
367 def toStyleStr(self, *, downSizedForTips = False) -> str:
|
|
|
368 """
|
|
|
369 Collapses the styles of this Arrow into a str, ready to be applied as part of the "style" property on an svg element.
|
|
|
370
|
|
|
371 Returns:
|
|
|
372 str : the styles string.
|
|
|
373 """
|
|
|
374 width = self.w
|
|
|
375 if downSizedForTips: width *= 0.8
|
|
|
376 return f";stroke:{self.col};stroke-width:{width};stroke-dasharray:{'5,5' if self.dash else 'none'}"
|
|
|
377
|
|
|
378 # vvv These constants could be inside the class itself a static properties, but python
|
|
|
379 # was built by brainless organisms so here we are!
|
|
|
380 INVALID_ARROW = Arrow(Arrow.MIN_W, ArrowColor.Invalid)
|
|
|
381 INSIGNIFICANT_ARROW = Arrow(Arrow.MIN_W, ArrowColor.Invalid, isDashed = True)
|
|
|
382
|
|
|
383 def applyFluxesEnrichmentToMap(fluxesEnrichmentRes :Dict[str, Union[Tuple[float, FoldChange], Tuple[float, FoldChange, float, float]]], metabMap :ET.ElementTree, maxNumericZScore :float) -> None:
|
|
|
384 """
|
|
|
385 Applies fluxes enrichment results to the provided metabolic map.
|
|
|
386
|
|
|
387 Args:
|
|
|
388 fluxesEnrichmentRes : fluxes enrichment results.
|
|
|
389 metabMap : the metabolic map to edit.
|
|
|
390 maxNumericZScore : biggest finite z-score value found.
|
|
|
391
|
|
|
392 Side effects:
|
|
|
393 metabMap : mut
|
|
|
394
|
|
|
395 Returns:
|
|
|
396 None
|
|
|
397 """
|
|
|
398 for reactionId, values in fluxesEnrichmentRes.items():
|
|
|
399 pValue = values[0]
|
|
|
400 foldChange = values[1]
|
|
|
401 z_score = values[2]
|
|
|
402
|
|
|
403 if math.isnan(pValue) or (isinstance(foldChange, float) and math.isnan(foldChange)):
|
|
|
404 continue
|
|
|
405
|
|
|
406 if isinstance(foldChange, str): foldChange = float(foldChange)
|
|
|
407 if pValue > ARGS.pValue: # pValue above tresh: dashed arrow
|
|
|
408 INSIGNIFICANT_ARROW.styleReactionElements(metabMap, reactionId)
|
|
|
409 INSIGNIFICANT_ARROW.styleReactionElements(metabMap, reactionId, mindReactionDir = False)
|
|
|
410
|
|
|
411 continue
|
|
|
412
|
|
|
413 if abs(foldChange) < (ARGS.fChange - 1) / (abs(ARGS.fChange) + 1):
|
|
|
414 INVALID_ARROW.styleReactionElements(metabMap, reactionId)
|
|
|
415 INVALID_ARROW.styleReactionElements(metabMap, reactionId, mindReactionDir = False)
|
|
|
416
|
|
|
417 continue
|
|
|
418
|
|
|
419 width = Arrow.MAX_W
|
|
|
420 if not math.isinf(z_score):
|
|
|
421 try:
|
|
|
422 width = min(
|
|
|
423 max(abs(z_score * Arrow.MAX_W) / maxNumericZScore, Arrow.MIN_W),
|
|
|
424 Arrow.MAX_W)
|
|
|
425
|
|
|
426 except ZeroDivisionError: pass
|
|
|
427 # TODO CHECK RV
|
|
|
428 #if not reactionId.endswith("_RV"): # RV stands for reversible reactions
|
|
|
429 # Arrow(width, ArrowColor.fromFoldChangeSign(foldChange)).styleReactionElements(metabMap, reactionId)
|
|
|
430 # continue
|
|
|
431
|
|
|
432 #reactionId = reactionId[:-3] # Remove "_RV"
|
|
|
433
|
|
|
434 inversionScore = (values[3] < 0) + (values[4] < 0) # Compacts the signs of averages into 1 easy to check score
|
|
|
435 if inversionScore == 2: foldChange *= -1
|
|
|
436 # ^^^ Style the inverse direction with the opposite sign netValue
|
|
|
437
|
|
|
438 # If the score is 1 (opposite signs) we use alternative colors vvv
|
|
|
439 arrow = Arrow(width, ArrowColor.fromFoldChangeSign(foldChange, useAltColor = inversionScore == 1))
|
|
|
440
|
|
|
441 # vvv These 2 if statements can both be true and can both happen
|
|
|
442 if ARGS.net: # style arrow head(s):
|
|
|
443 arrow.styleReactionElements(metabMap, reactionId + ("_B" if inversionScore == 2 else "_F"))
|
|
|
444 arrow.applyTo(("F_" if inversionScore == 2 else "B_") + reactionId, metabMap, f";stroke:{ArrowColor.Transparent};stroke-width:0;stroke-dasharray:None")
|
|
|
445
|
|
|
446 arrow.styleReactionElements(metabMap, reactionId, mindReactionDir = False)
|
|
|
447
|
|
|
448
|
|
|
449 ############################ split class ######################################
|
|
|
450 def split_class(classes :pd.DataFrame, resolve_rules :Dict[str, List[float]]) -> Dict[str, List[List[float]]]:
|
|
|
451 """
|
|
|
452 Generates a :dict that groups together data from a :DataFrame based on classes the data is related to.
|
|
|
453
|
|
|
454 Args:
|
|
|
455 classes : a :DataFrame of only string values, containing class information (rows) and keys to query the resolve_rules :dict
|
|
|
456 resolve_rules : a :dict containing :float data
|
|
|
457
|
|
|
458 Returns:
|
|
|
459 dict : the dict with data grouped by class
|
|
|
460
|
|
|
461 Side effects:
|
|
|
462 classes : mut
|
|
|
463 """
|
|
|
464 class_pat :Dict[str, List[List[float]]] = {}
|
|
|
465 for i in range(len(classes)):
|
|
|
466 classe :str = classes.iloc[i, 1]
|
|
|
467 if pd.isnull(classe): continue
|
|
|
468
|
|
|
469 l :List[List[float]] = []
|
|
|
470 for j in range(i, len(classes)):
|
|
|
471 if classes.iloc[j, 1] == classe:
|
|
|
472 pat_id :str = classes.iloc[j, 0]
|
|
|
473 tmp = resolve_rules.get(pat_id, None)
|
|
|
474 if tmp != None:
|
|
|
475 l.append(tmp)
|
|
|
476 classes.iloc[j, 1] = None
|
|
|
477
|
|
|
478 if l:
|
|
|
479 class_pat[classe] = list(map(list, zip(*l)))
|
|
|
480 continue
|
|
|
481
|
|
|
482 utils.logWarning(
|
|
|
483 f"Warning: no sample found in class \"{classe}\", the class has been disregarded", ARGS.out_log)
|
|
|
484
|
|
|
485 return class_pat
|
|
|
486
|
|
|
487 ############################ conversion ##############################################
|
|
|
488 #conversion from svg to png
|
|
|
489 def svg_to_png_with_background(svg_path :utils.FilePath, png_path :utils.FilePath, dpi :int = 72, scale :int = 1, size :Optional[float] = None) -> None:
|
|
|
490 """
|
|
|
491 Internal utility to convert an SVG to PNG (forced opaque) to aid in PDF conversion.
|
|
|
492
|
|
|
493 Args:
|
|
|
494 svg_path : path to SVG file
|
|
|
495 png_path : path for new PNG file
|
|
|
496 dpi : dots per inch of the generated PNG
|
|
|
497 scale : scaling factor for the generated PNG, computed internally when a size is provided
|
|
|
498 size : final effective width of the generated PNG
|
|
|
499
|
|
|
500 Returns:
|
|
|
501 None
|
|
|
502 """
|
|
|
503 if size:
|
|
|
504 image = pyvips.Image.new_from_file(svg_path.show(), dpi=dpi, scale=1)
|
|
|
505 scale = size / image.width
|
|
|
506 image = image.resize(scale)
|
|
|
507 else:
|
|
|
508 image = pyvips.Image.new_from_file(svg_path.show(), dpi=dpi, scale=scale)
|
|
|
509
|
|
|
510 white_background = pyvips.Image.black(image.width, image.height).new_from_image([255, 255, 255])
|
|
|
511 white_background = white_background.affine([scale, 0, 0, scale])
|
|
|
512
|
|
|
513 if white_background.bands != image.bands:
|
|
|
514 white_background = white_background.extract_band(0)
|
|
|
515
|
|
|
516 composite_image = white_background.composite2(image, 'over')
|
|
|
517 composite_image.write_to_file(png_path.show())
|
|
|
518
|
|
|
519 #funzione unica, lascio fuori i file e li passo in input
|
|
|
520 #conversion from png to pdf
|
|
|
521 def convert_png_to_pdf(png_file :utils.FilePath, pdf_file :utils.FilePath) -> None:
|
|
|
522 """
|
|
|
523 Internal utility to convert a PNG to PDF to aid from SVG conversion.
|
|
|
524
|
|
|
525 Args:
|
|
|
526 png_file : path to PNG file
|
|
|
527 pdf_file : path to new PDF file
|
|
|
528
|
|
|
529 Returns:
|
|
|
530 None
|
|
|
531 """
|
|
|
532 image = Image.open(png_file.show())
|
|
|
533 image = image.convert("RGB")
|
|
|
534 image.save(pdf_file.show(), "PDF", resolution=100.0)
|
|
|
535
|
|
|
536 #function called to reduce redundancy in the code
|
|
|
537 def convert_to_pdf(file_svg :utils.FilePath, file_png :utils.FilePath, file_pdf :utils.FilePath) -> None:
|
|
|
538 """
|
|
|
539 Converts the SVG map at the provided path to PDF.
|
|
|
540
|
|
|
541 Args:
|
|
|
542 file_svg : path to SVG file
|
|
|
543 file_png : path to PNG file
|
|
|
544 file_pdf : path to new PDF file
|
|
|
545
|
|
|
546 Returns:
|
|
|
547 None
|
|
|
548 """
|
|
|
549 svg_to_png_with_background(file_svg, file_png)
|
|
|
550 try:
|
|
|
551 convert_png_to_pdf(file_png, file_pdf)
|
|
|
552 print(f'PDF file {file_pdf.filePath} successfully generated.')
|
|
|
553
|
|
|
554 except Exception as e:
|
|
|
555 raise utils.DataErr(file_pdf.show(), f'Error generating PDF file: {e}')
|
|
|
556
|
|
|
557 ############################ map ##############################################
|
|
|
558 def buildOutputPath(dataset1Name :str, dataset2Name = "rest", *, details = "", ext :utils.FileFormat) -> utils.FilePath:
|
|
|
559 """
|
|
|
560 Builds a FilePath instance from the names of confronted datasets ready to point to a location in the
|
|
|
561 "result/" folder, used by this tool for output files in collections.
|
|
|
562
|
|
|
563 Args:
|
|
|
564 dataset1Name : _description_
|
|
|
565 dataset2Name : _description_. Defaults to "rest".
|
|
|
566 details : _description_
|
|
|
567 ext : _description_
|
|
|
568
|
|
|
569 Returns:
|
|
|
570 utils.FilePath : _description_
|
|
|
571 """
|
|
|
572 # This function returns a util data structure but is extremely specific to this module.
|
|
|
573 # RAS also uses collections as output and as such might benefit from a method like this, but I'd wait
|
|
|
574 # TODO: until a third tool with multiple outputs appears before porting this to utils.
|
|
|
575 return utils.FilePath(
|
|
|
576 f"{dataset1Name}_vs_{dataset2Name}" + (f" ({details})" if details else ""),
|
|
|
577 # ^^^ yes this string is built every time even if the form is the same for the same 2 datasets in
|
|
|
578 # all output files: I don't care, this was never the performance bottleneck of the tool and
|
|
|
579 # there is no other net gain in saving and re-using the built string.
|
|
|
580 ext,
|
|
|
581 prefix = ARGS.output_path)
|
|
|
582
|
|
|
583 FIELD_NOT_AVAILABLE = '/'
|
|
|
584 def writeToCsv(rows: List[list], fieldNames :List[str], outPath :utils.FilePath) -> None:
|
|
|
585 fieldsAmt = len(fieldNames)
|
|
|
586 with open(outPath.show(), "w", newline = "") as fd:
|
|
|
587 writer = csv.DictWriter(fd, fieldnames = fieldNames, delimiter = '\t')
|
|
|
588 writer.writeheader()
|
|
|
589
|
|
|
590 for row in rows:
|
|
|
591 sizeMismatch = fieldsAmt - len(row)
|
|
|
592 if sizeMismatch > 0: row.extend([FIELD_NOT_AVAILABLE] * sizeMismatch)
|
|
|
593 writer.writerow({ field : data for field, data in zip(fieldNames, row) })
|
|
|
594
|
|
|
595 OldEnrichedScores = Dict[str, List[Union[float, FoldChange]]] #TODO: try to use Tuple whenever possible
|
|
|
596 def writeTabularResult(enrichedScores : OldEnrichedScores, outPath :utils.FilePath) -> None:
|
|
|
597 fieldNames = ["ids", "P_Value", "fold change", "z-score"]
|
|
|
598 fieldNames.extend(["average_1", "average_2"])
|
|
|
599
|
|
|
600 writeToCsv([ [reactId] + values for reactId, values in enrichedScores.items() ], fieldNames, outPath)
|
|
|
601
|
|
|
602 def temp_thingsInCommon(tmp :Dict[str, List[Union[float, FoldChange]]], core_map :ET.ElementTree, max_z_score :float, dataset1Name :str, dataset2Name = "rest") -> None:
|
|
|
603 # this function compiles the things always in common between comparison modes after enrichment.
|
|
|
604 # TODO: organize, name better.
|
|
|
605 writeTabularResult(tmp, buildOutputPath(dataset1Name, dataset2Name, details = "Tabular Result", ext = utils.FileFormat.TSV))
|
|
|
606 for reactId, enrichData in tmp.items(): tmp[reactId] = tuple(enrichData)
|
|
|
607 applyFluxesEnrichmentToMap(tmp, core_map, max_z_score)
|
|
|
608
|
|
|
609 def computePValue(dataset1Data: List[float], dataset2Data: List[float]) -> Tuple[float, float]:
|
|
|
610 """
|
|
|
611 Computes the statistical significance score (P-value) of the comparison between coherent data
|
|
|
612 from two datasets. The data is supposed to, in both datasets:
|
|
|
613 - be related to the same reaction ID;
|
|
|
614 - be ordered by sample, such that the item at position i in both lists is related to the
|
|
|
615 same sample or cell line.
|
|
|
616
|
|
|
617 Args:
|
|
|
618 dataset1Data : data from the 1st dataset.
|
|
|
619 dataset2Data : data from the 2nd dataset.
|
|
|
620
|
|
|
621 Returns:
|
|
|
622 tuple: (P-value, Z-score)
|
|
|
623 - P-value from the selected test on the provided data.
|
|
|
624 - Z-score of the difference between means of the two datasets.
|
|
|
625 """
|
|
|
626
|
|
|
627 match ARGS.test:
|
|
|
628 case "ks":
|
|
|
629 # Perform Kolmogorov-Smirnov test
|
|
|
630 _, p_value = st.ks_2samp(dataset1Data, dataset2Data)
|
|
|
631 case "ttest_p":
|
|
|
632 # Datasets should have same size
|
|
|
633 if len(dataset1Data) != len(dataset2Data):
|
|
|
634 raise ValueError("Datasets must have the same size for paired t-test.")
|
|
|
635 # Perform t-test for paired samples
|
|
|
636 _, p_value = st.ttest_rel(dataset1Data, dataset2Data)
|
|
|
637 case "ttest_ind":
|
|
|
638 # Perform t-test for independent samples
|
|
|
639 _, p_value = st.ttest_ind(dataset1Data, dataset2Data)
|
|
|
640 case "wilcoxon":
|
|
|
641 # Datasets should have same size
|
|
|
642 if len(dataset1Data) != len(dataset2Data):
|
|
|
643 raise ValueError("Datasets must have the same size for Wilcoxon signed-rank test.")
|
|
|
644 # Perform Wilcoxon signed-rank test
|
|
|
645 np.random.seed(42) # Ensure reproducibility since zsplit method is used
|
|
|
646 _, p_value = st.wilcoxon(dataset1Data, dataset2Data, zero_method="zsplit")
|
|
|
647 case "mw":
|
|
|
648 # Perform Mann-Whitney U test
|
|
|
649 _, p_value = st.mannwhitneyu(dataset1Data, dataset2Data)
|
|
|
650
|
|
|
651 # Calculate means and standard deviations
|
|
|
652 mean1 = np.nanmean(dataset1Data)
|
|
|
653 mean2 = np.nanmean(dataset2Data)
|
|
|
654 std1 = np.nanstd(dataset1Data, ddof=1)
|
|
|
655 std2 = np.nanstd(dataset2Data, ddof=1)
|
|
|
656
|
|
|
657 n1 = len(dataset1Data)
|
|
|
658 n2 = len(dataset2Data)
|
|
|
659
|
|
|
660 # Calculate Z-score
|
|
|
661 z_score = (mean1 - mean2) / np.sqrt((std1**2 / n1) + (std2**2 / n2))
|
|
|
662
|
|
|
663 return p_value, z_score
|
|
|
664
|
|
|
665 def compareDatasetPair(dataset1Data :List[List[float]], dataset2Data :List[List[float]], ids :List[str]) -> Tuple[Dict[str, List[Union[float, FoldChange]]], float]:
|
|
|
666 #TODO: the following code still suffers from "dumbvarnames-osis"
|
|
|
667 comparisonResult :Dict[str, List[Union[float, FoldChange]]] = {}
|
|
|
668 count = 0
|
|
|
669 max_z_score = 0
|
|
|
670 for l1, l2 in zip(dataset1Data, dataset2Data):
|
|
|
671 reactId = ids[count]
|
|
|
672 count += 1
|
|
|
673 if not reactId: continue # we skip ids that have already been processed
|
|
|
674
|
|
|
675 try:
|
|
|
676 p_value, z_score = computePValue(l1, l2)
|
|
|
677 avg1 = sum(l1) / len(l1)
|
|
|
678 avg2 = sum(l2) / len(l2)
|
|
|
679 f_c = fold_change(avg1, avg2)
|
|
|
680 if np.isfinite(z_score) and max_z_score < abs(z_score): max_z_score = abs(z_score)
|
|
|
681
|
|
|
682 comparisonResult[reactId] = [float(p_value), f_c, z_score, avg1, avg2]
|
|
|
683 except (TypeError, ZeroDivisionError): continue
|
|
|
684
|
|
|
685 # Apply multiple testing correction if set by the user
|
|
|
686 if ARGS.adjusted:
|
|
|
687
|
|
|
688 # Retrieve the p-values from the comparisonResult dictionary, they have to be different from NaN
|
|
|
689 validPValues = [(reactId, result[0]) for reactId, result in comparisonResult.items() if not np.isnan(result[0])]
|
|
|
690
|
|
|
691 if not validPValues:
|
|
|
692 return comparisonResult, max_z_score
|
|
|
693
|
|
|
694 # Unpack the valid p-values
|
|
|
695 reactIds, pValues = zip(*validPValues)
|
|
|
696 # Adjust the p-values using the Benjamini-Hochberg method
|
|
|
697 adjustedPValues = st.false_discovery_control(pValues)
|
|
|
698 # Update the comparisonResult dictionary with the adjusted p-values
|
|
|
699 for reactId , adjustedPValue in zip(reactIds, adjustedPValues):
|
|
|
700 comparisonResult[reactId][0] = adjustedPValue
|
|
|
701
|
|
|
702 return comparisonResult, max_z_score
|
|
|
703
|
|
|
704 def computeEnrichment(class_pat :Dict[str, List[List[float]]], ids :List[str]) -> List[Tuple[str, str, dict, float]]:
|
|
|
705 """
|
|
|
706 Compares clustered data based on a given comparison mode and applies enrichment-based styling on the
|
|
|
707 provided metabolic map.
|
|
|
708
|
|
|
709 Args:
|
|
|
710 class_pat : the clustered data.
|
|
|
711 ids : ids for data association.
|
|
|
712
|
|
|
713
|
|
|
714 Returns:
|
|
|
715 List[Tuple[str, str, dict, float]]: List of tuples with pairs of dataset names, comparison dictionary, and max z-score.
|
|
|
716
|
|
|
717 Raises:
|
|
|
718 sys.exit : if there are less than 2 classes for comparison
|
|
|
719
|
|
|
720 """
|
|
|
721 class_pat = { k.strip() : v for k, v in class_pat.items() }
|
|
|
722 #TODO: simplfy this stuff vvv and stop using sys.exit (raise the correct utils error)
|
|
|
723 if (not class_pat) or (len(class_pat.keys()) < 2): sys.exit('Execution aborted: classes provided for comparisons are less than two\n')
|
|
|
724
|
|
|
725 enrichment_results = []
|
|
|
726
|
|
|
727
|
|
|
728 if ARGS.comparison == "manyvsmany":
|
|
|
729 for i, j in it.combinations(class_pat.keys(), 2):
|
|
|
730 comparisonDict, max_z_score = compareDatasetPair(class_pat.get(i), class_pat.get(j), ids)
|
|
|
731 enrichment_results.append((i, j, comparisonDict, max_z_score))
|
|
|
732
|
|
|
733 elif ARGS.comparison == "onevsrest":
|
|
|
734 for single_cluster in class_pat.keys():
|
|
|
735 rest = [item for k, v in class_pat.items() if k != single_cluster for item in v]
|
|
|
736
|
|
|
737 comparisonDict, max_z_score = compareDatasetPair(class_pat.get(single_cluster), rest, ids)
|
|
|
738 enrichment_results.append((single_cluster, "rest", comparisonDict, max_z_score))
|
|
|
739
|
|
|
740 #elif ARGS.comparison == "onevsmany":
|
|
|
741 # controlItems = class_pat.get(ARGS.control)
|
|
|
742 # for otherDataset in class_pat.keys():
|
|
|
743 # if otherDataset == ARGS.control:
|
|
|
744 # continue
|
|
|
745 # comparisonDict, max_z_score = compareDatasetPair(controlItems, class_pat.get(otherDataset), ids)
|
|
|
746 # enrichment_results.append((ARGS.control, otherDataset, comparisonDict, max_z_score))
|
|
|
747 elif ARGS.comparison == "onevsmany":
|
|
|
748 controlItems = class_pat.get(ARGS.control)
|
|
|
749 for otherDataset in class_pat.keys():
|
|
|
750 if otherDataset == ARGS.control:
|
|
|
751 continue
|
|
|
752 comparisonDict, max_z_score = compareDatasetPair(class_pat.get(otherDataset),controlItems, ids)
|
|
|
753 enrichment_results.append(( otherDataset,ARGS.control, comparisonDict, max_z_score))
|
|
|
754
|
|
|
755 return enrichment_results
|
|
|
756
|
|
|
757 def createOutputMaps(dataset1Name :str, dataset2Name :str, core_map :ET.ElementTree) -> None:
|
|
|
758 svgFilePath = buildOutputPath(dataset1Name, dataset2Name, details="SVG Map", ext=utils.FileFormat.SVG)
|
|
|
759 utils.writeSvg(svgFilePath, core_map)
|
|
|
760
|
|
|
761 if ARGS.generate_pdf:
|
|
|
762 pngPath = buildOutputPath(dataset1Name, dataset2Name, details="PNG Map", ext=utils.FileFormat.PNG)
|
|
|
763 pdfPath = buildOutputPath(dataset1Name, dataset2Name, details="PDF Map", ext=utils.FileFormat.PDF)
|
|
|
764 convert_to_pdf(svgFilePath, pngPath, pdfPath)
|
|
|
765
|
|
|
766 if not ARGS.generate_svg:
|
|
|
767 os.remove(svgFilePath.show())
|
|
|
768
|
|
|
769 ClassPat = Dict[str, List[List[float]]]
|
|
|
770 def getClassesAndIdsFromDatasets(datasetsPaths :List[str], datasetPath :str, classPath :str, names :List[str]) -> Tuple[List[str], ClassPat]:
|
|
|
771 # TODO: I suggest creating dicts with ids as keys instead of keeping class_pat and ids separate,
|
|
|
772 # for the sake of everyone's sanity.
|
|
|
773 class_pat :ClassPat = {}
|
|
|
774 if ARGS.option == 'datasets':
|
|
|
775 num = 1 #TODO: the dataset naming function could be a generator
|
|
|
776 for path, name in zip(datasetsPaths, names):
|
|
|
777 name = name_dataset(name, num)
|
|
|
778 resolve_rules_float, ids = getDatasetValues(path, name)
|
|
|
779 if resolve_rules_float != None:
|
|
|
780 class_pat[name] = list(map(list, zip(*resolve_rules_float.values())))
|
|
|
781
|
|
|
782 num += 1
|
|
|
783
|
|
|
784 elif ARGS.option == "dataset_class":
|
|
|
785 classes = read_dataset(classPath, "class")
|
|
|
786 classes = classes.astype(str)
|
|
|
787 resolve_rules_float, ids = getDatasetValues(datasetPath, "Dataset Class (not actual name)")
|
|
|
788 #check if classes have match on ids
|
|
|
789 if not all(classes.iloc[:, 0].isin(ids)):
|
|
|
790 utils.logWarning(
|
|
|
791 "No match between classes and sample IDs", ARGS.out_log)
|
|
|
792 if resolve_rules_float != None: class_pat = split_class(classes, resolve_rules_float)
|
|
|
793
|
|
|
794 return ids, class_pat
|
|
|
795 #^^^ TODO: this could be a match statement over an enum, make it happen future marea dev with python 3.12! (it's why I kept the ifs)
|
|
|
796
|
|
|
797 #TODO: create these damn args as FilePath objects
|
|
|
798 def getDatasetValues(datasetPath :str, datasetName :str) -> Tuple[ClassPat, List[str]]:
|
|
|
799 """
|
|
|
800 Opens the dataset at the given path and extracts the values (expected nullable numerics) and the IDs.
|
|
|
801
|
|
|
802 Args:
|
|
|
803 datasetPath : path to the dataset
|
|
|
804 datasetName (str): dataset name, used in error reporting
|
|
|
805
|
|
|
806 Returns:
|
|
|
807 Tuple[ClassPat, List[str]]: values and IDs extracted from the dataset
|
|
|
808 """
|
|
|
809 dataset = read_dataset(datasetPath, datasetName)
|
|
|
810
|
|
|
811 # Ensure the first column is treated as the reaction name
|
|
|
812 dataset = dataset.set_index(dataset.columns[0])
|
|
|
813
|
|
|
814 # Check if required reactions exist in the dataset
|
|
|
815 required_reactions = ['EX_lac__L_e', 'EX_glc__D_e', 'EX_gln__L_e', 'EX_glu__L_e']
|
|
|
816 missing_reactions = [reaction for reaction in required_reactions if reaction not in dataset.index]
|
|
|
817
|
|
|
818 if missing_reactions:
|
|
|
819 sys.exit(f'Execution aborted: Missing required reactions {missing_reactions} in {datasetName}\n')
|
|
|
820
|
|
|
821 # Calculate new rows using safe division
|
|
|
822 lact_glc = np.divide(
|
|
|
823 np.clip(dataset.loc['EX_lac__L_e'].to_numpy(), a_min=0, a_max=None),
|
|
|
824 np.clip(dataset.loc['EX_glc__D_e'].to_numpy(), a_min=None, a_max=0),
|
|
|
825 out=np.full_like(dataset.loc['EX_lac__L_e'].to_numpy(), np.nan), # Prepara un array con NaN come output di default
|
|
|
826 where=dataset.loc['EX_glc__D_e'].to_numpy() != 0 # Condizione per evitare la divisione per zero
|
|
|
827 )
|
|
|
828 lact_gln = np.divide(
|
|
|
829 np.clip(dataset.loc['EX_lac__L_e'].to_numpy(), a_min=0, a_max=None),
|
|
|
830 np.clip(dataset.loc['EX_gln__L_e'].to_numpy(), a_min=None, a_max=0),
|
|
|
831 out=np.full_like(dataset.loc['EX_lac__L_e'].to_numpy(), np.nan),
|
|
|
832 where=dataset.loc['EX_gln__L_e'].to_numpy() != 0
|
|
|
833 )
|
|
|
834 lact_o2 = np.divide(
|
|
|
835 np.clip(dataset.loc['EX_lac__L_e'].to_numpy(), a_min=0, a_max=None),
|
|
|
836 np.clip(dataset.loc['EX_o2_e'].to_numpy(), a_min=None, a_max=0),
|
|
|
837 out=np.full_like(dataset.loc['EX_lac__L_e'].to_numpy(), np.nan),
|
|
|
838 where=dataset.loc['EX_o2_e'].to_numpy() != 0
|
|
|
839 )
|
|
|
840 glu_gln = np.divide(
|
|
|
841 dataset.loc['EX_glu__L_e'].to_numpy(),
|
|
|
842 np.clip(dataset.loc['EX_gln__L_e'].to_numpy(), a_min=None, a_max=0),
|
|
|
843 out=np.full_like(dataset.loc['EX_lac__L_e'].to_numpy(), np.nan),
|
|
|
844 where=dataset.loc['EX_gln__L_e'].to_numpy() != 0
|
|
|
845 )
|
|
|
846
|
|
|
847
|
|
|
848 values = {'lact_glc': lact_glc, 'lact_gln': lact_gln, 'lact_o2': lact_o2, 'glu_gln': glu_gln}
|
|
|
849
|
|
|
850 # Sostituzione di inf e NaN con 0 se necessario
|
|
|
851 for key in values:
|
|
|
852 values[key] = np.nan_to_num(values[key], nan=0.0, posinf=0.0, neginf=0.0)
|
|
|
853
|
|
|
854 # Creazione delle nuove righe da aggiungere al dataset
|
|
|
855 new_rows = pd.DataFrame({
|
|
|
856 dataset.index.name: ['LactGlc', 'LactGln', 'LactO2', 'GluGln'],
|
|
|
857 **{col: [values['lact_glc'][i], values['lact_gln'][i], values['lact_o2'][i], values['glu_gln'][i]]
|
|
|
858 for i, col in enumerate(dataset.columns)}
|
|
|
859 })
|
|
|
860
|
|
|
861 #print(new_rows)
|
|
|
862
|
|
|
863 # Ritorna il dataset originale con le nuove righe
|
|
|
864 dataset.reset_index(inplace=True)
|
|
|
865 dataset = pd.concat([dataset, new_rows], ignore_index=True)
|
|
|
866
|
|
|
867 IDs = pd.Series.tolist(dataset.iloc[:, 0].astype(str))
|
|
|
868
|
|
|
869 dataset = dataset.drop(dataset.columns[0], axis = "columns").to_dict("list")
|
|
|
870 return { id : list(map(utils.Float("Dataset values, not an argument"), values)) for id, values in dataset.items() }, IDs
|
|
|
871
|
|
|
872 def rgb_to_hex(rgb):
|
|
|
873 """
|
|
|
874 Convert RGB values (0-1 range) to hexadecimal color format.
|
|
|
875
|
|
|
876 Args:
|
|
|
877 rgb (numpy.ndarray): An array of RGB color components (in the range [0, 1]).
|
|
|
878
|
|
|
879 Returns:
|
|
|
880 str: The color in hexadecimal format (e.g., '#ff0000' for red).
|
|
|
881 """
|
|
|
882 # Convert RGB values (0-1 range) to hexadecimal format
|
|
|
883 rgb = (np.array(rgb) * 255).astype(int)
|
|
|
884 return '#{:02x}{:02x}{:02x}'.format(rgb[0], rgb[1], rgb[2])
|
|
|
885
|
|
|
886 def save_colormap_image(min_value: float, max_value: float, path: utils.FilePath, colorMap:str="viridis"):
|
|
|
887 """
|
|
|
888 Create and save an image of the colormap showing the gradient and its range.
|
|
|
889
|
|
|
890 Args:
|
|
|
891 min_value (float): The minimum value of the colormap range.
|
|
|
892 max_value (float): The maximum value of the colormap range.
|
|
|
893 filename (str): The filename for saving the image.
|
|
|
894 """
|
|
|
895
|
|
|
896 # Create a colormap using matplotlib
|
|
|
897 cmap = plt.get_cmap(colorMap)
|
|
|
898
|
|
|
899 # Create a figure and axis
|
|
|
900 fig, ax = plt.subplots(figsize=(6, 1))
|
|
|
901 fig.subplots_adjust(bottom=0.5)
|
|
|
902
|
|
|
903 # Create a gradient image
|
|
|
904 gradient = np.linspace(0, 1, 256)
|
|
|
905 gradient = np.vstack((gradient, gradient))
|
|
|
906
|
|
|
907 # Add min and max value annotations
|
|
|
908 ax.text(0, 0.5, f'{np.round(min_value, 3)}', va='center', ha='right', transform=ax.transAxes, fontsize=12, color='black')
|
|
|
909 ax.text(1, 0.5, f'{np.round(max_value, 3)}', va='center', ha='left', transform=ax.transAxes, fontsize=12, color='black')
|
|
|
910
|
|
|
911
|
|
|
912 # Display the gradient image
|
|
|
913 ax.imshow(gradient, aspect='auto', cmap=cmap)
|
|
|
914 ax.set_axis_off()
|
|
|
915
|
|
|
916 # Save the image
|
|
|
917 plt.savefig(path.show(), bbox_inches='tight', pad_inches=0)
|
|
|
918 plt.close()
|
|
|
919 pass
|
|
|
920
|
|
|
921 def min_nonzero_abs(arr):
|
|
|
922 # Flatten the array and filter out zeros, then find the minimum of the remaining values
|
|
|
923 non_zero_elements = np.abs(arr)[np.abs(arr) > 0]
|
|
|
924 return np.min(non_zero_elements) if non_zero_elements.size > 0 else None
|
|
|
925
|
|
|
926 def computeEnrichmentMeanMedian(metabMap: ET.ElementTree, class_pat: Dict[str, List[List[float]]], ids: List[str], colormap:str) -> None:
|
|
|
927 """
|
|
|
928 Compute and visualize the metabolic map based on mean and median of the input fluxes.
|
|
|
929 The fluxes are normalised across classes/datasets and visualised using the given colormap.
|
|
|
930
|
|
|
931 Args:
|
|
|
932 metabMap (ET.ElementTree): An XML tree representing the metabolic map.
|
|
|
933 class_pat (Dict[str, List[List[float]]]): A dictionary where keys are class names and values are lists of enrichment values.
|
|
|
934 ids (List[str]): A list of reaction IDs to be used for coloring arrows.
|
|
|
935
|
|
|
936 Returns:
|
|
|
937 None
|
|
|
938 """
|
|
|
939 # Create copies only if they are needed
|
|
|
940 metabMap_mean = copy.deepcopy(metabMap)
|
|
|
941 metabMap_median = copy.deepcopy(metabMap)
|
|
|
942
|
|
|
943 # Compute medians and means
|
|
|
944 medians = {key: np.round(np.nanmedian(np.array(value), axis=1), 6) for key, value in class_pat.items()}
|
|
|
945 means = {key: np.round(np.nanmean(np.array(value), axis=1),6) for key, value in class_pat.items()}
|
|
|
946
|
|
|
947 # Normalize medians and means
|
|
|
948 max_flux_medians = max(np.max(np.abs(arr)) for arr in medians.values())
|
|
|
949 max_flux_means = max(np.max(np.abs(arr)) for arr in means.values())
|
|
|
950
|
|
|
951 min_flux_medians = min(min_nonzero_abs(arr) for arr in medians.values())
|
|
|
952 min_flux_means = min(min_nonzero_abs(arr) for arr in means.values())
|
|
|
953
|
|
|
954 medians = {key: median/max_flux_medians for key, median in medians.items()}
|
|
|
955 means = {key: mean/max_flux_means for key, mean in means.items()}
|
|
|
956
|
|
|
957 save_colormap_image(min_flux_medians, max_flux_medians, utils.FilePath("Color map median", ext=utils.FileFormat.PNG, prefix=ARGS.output_path), colormap)
|
|
|
958 save_colormap_image(min_flux_means, max_flux_means, utils.FilePath("Color map mean", ext=utils.FileFormat.PNG, prefix=ARGS.output_path), colormap)
|
|
|
959
|
|
|
960 cmap = plt.get_cmap(colormap)
|
|
|
961
|
|
|
962 min_width = 2.0 # Minimum arrow width
|
|
|
963 max_width = 15.0 # Maximum arrow width
|
|
|
964
|
|
|
965 for key in class_pat:
|
|
|
966 # Create color mappings for median and mean
|
|
|
967 colors_median = {
|
|
|
968 rxn_id: rgb_to_hex(cmap(abs(medians[key][i]))) if medians[key][i] != 0 else '#bebebe' #grey blocked
|
|
|
969 for i, rxn_id in enumerate(ids)
|
|
|
970 }
|
|
|
971
|
|
|
972 colors_mean = {
|
|
|
973 rxn_id: rgb_to_hex(cmap(abs(means[key][i]))) if means[key][i] != 0 else '#bebebe' #grey blocked
|
|
|
974 for i, rxn_id in enumerate(ids)
|
|
|
975 }
|
|
|
976
|
|
|
977 for i, rxn_id in enumerate(ids):
|
|
|
978 # Calculate arrow width for median
|
|
|
979 width_median = np.interp(abs(medians[key][i]), [0, 1], [min_width, max_width])
|
|
|
980 isNegative = medians[key][i] < 0
|
|
|
981 apply_arrow(metabMap_median, rxn_id, colors_median[rxn_id], isNegative, width_median)
|
|
|
982
|
|
|
983 # Calculate arrow width for mean
|
|
|
984 width_mean = np.interp(abs(means[key][i]), [0, 1], [min_width, max_width])
|
|
|
985 isNegative = means[key][i] < 0
|
|
|
986 apply_arrow(metabMap_mean, rxn_id, colors_mean[rxn_id], isNegative, width_mean)
|
|
|
987
|
|
|
988 # Save and convert the SVG files
|
|
|
989 save_and_convert(metabMap_mean, "mean", key)
|
|
|
990 save_and_convert(metabMap_median, "median", key)
|
|
|
991
|
|
|
992 def apply_arrow(metabMap, rxn_id, color, isNegative, width=5):
|
|
|
993 """
|
|
|
994 Apply an arrow to a specific reaction in the metabolic map with a given color.
|
|
|
995
|
|
|
996 Args:
|
|
|
997 metabMap (ET.ElementTree): An XML tree representing the metabolic map.
|
|
|
998 rxn_id (str): The ID of the reaction to which the arrow will be applied.
|
|
|
999 color (str): The color of the arrow in hexadecimal format.
|
|
|
1000 isNegative (bool): A boolean indicating if the arrow represents a negative value.
|
|
|
1001 width (int): The width of the arrow.
|
|
|
1002
|
|
|
1003 Returns:
|
|
|
1004 None
|
|
|
1005 """
|
|
|
1006 arrow = Arrow(width=width, col=color)
|
|
|
1007 arrow.styleReactionElementsMeanMedian(metabMap, rxn_id, isNegative)
|
|
|
1008 pass
|
|
|
1009
|
|
|
1010 def save_and_convert(metabMap, map_type, key):
|
|
|
1011 """
|
|
|
1012 Save the metabolic map as an SVG file and optionally convert it to PNG and PDF formats.
|
|
|
1013
|
|
|
1014 Args:
|
|
|
1015 metabMap (ET.ElementTree): An XML tree representing the metabolic map.
|
|
|
1016 map_type (str): The type of map ('mean' or 'median').
|
|
|
1017 key (str): The key identifying the specific map.
|
|
|
1018
|
|
|
1019 Returns:
|
|
|
1020 None
|
|
|
1021 """
|
|
|
1022 svgFilePath = utils.FilePath(f"SVG Map {map_type} - {key}", ext=utils.FileFormat.SVG, prefix=ARGS.output_path)
|
|
|
1023 utils.writeSvg(svgFilePath, metabMap)
|
|
|
1024 if ARGS.generate_pdf:
|
|
|
1025 pngPath = utils.FilePath(f"PNG Map {map_type} - {key}", ext=utils.FileFormat.PNG, prefix=ARGS.output_path)
|
|
|
1026 pdfPath = utils.FilePath(f"PDF Map {map_type} - {key}", ext=utils.FileFormat.PDF, prefix=ARGS.output_path)
|
|
|
1027 convert_to_pdf(svgFilePath, pngPath, pdfPath)
|
|
|
1028 if not ARGS.generate_svg:
|
|
|
1029 os.remove(svgFilePath.show())
|
|
|
1030
|
|
|
1031 ############################ MAIN #############################################
|
|
|
1032 def main(args:List[str] = None) -> None:
|
|
|
1033 """
|
|
|
1034 Initializes everything and sets the program in motion based on the fronted input arguments.
|
|
|
1035
|
|
|
1036 Returns:
|
|
|
1037 None
|
|
|
1038
|
|
|
1039 Raises:
|
|
|
1040 sys.exit : if a user-provided custom map is in the wrong format (ET.XMLSyntaxError, ET.XMLSchemaParseError)
|
|
|
1041 """
|
|
|
1042
|
|
|
1043 global ARGS
|
|
|
1044 ARGS = process_args(args)
|
|
|
1045
|
|
|
1046 if ARGS.custom_map == 'None':
|
|
|
1047 ARGS.custom_map = None
|
|
|
1048
|
|
|
1049 if os.path.isdir(ARGS.output_path) == False: os.makedirs(ARGS.output_path)
|
|
|
1050
|
|
|
1051 core_map :ET.ElementTree = ARGS.choice_map.getMap(
|
|
|
1052 ARGS.tool_dir,
|
|
|
1053 utils.FilePath.fromStrPath(ARGS.custom_map) if ARGS.custom_map else None)
|
|
|
1054 # TODO: ^^^ ugly but fine for now, the argument is None if the model isn't custom because no file was given.
|
|
|
1055 # getMap will None-check the customPath and panic when the model IS custom but there's no file (good). A cleaner
|
|
|
1056 # solution can be derived from my comment in FilePath.fromStrPath
|
|
|
1057
|
|
|
1058 ids, class_pat = getClassesAndIdsFromDatasets(ARGS.input_datas_fluxes, ARGS.input_data_fluxes, ARGS.input_class_fluxes, ARGS.names_fluxes)
|
|
|
1059
|
|
|
1060 if(ARGS.choice_map == utils.Model.HMRcore):
|
|
|
1061 temp_map = utils.Model.HMRcore_no_legend
|
|
|
1062 computeEnrichmentMeanMedian(temp_map.getMap(ARGS.tool_dir), class_pat, ids, ARGS.color_map)
|
|
|
1063 elif(ARGS.choice_map == utils.Model.ENGRO2):
|
|
|
1064 temp_map = utils.Model.ENGRO2_no_legend
|
|
|
1065 computeEnrichmentMeanMedian(temp_map.getMap(ARGS.tool_dir), class_pat, ids, ARGS.color_map)
|
|
|
1066 else:
|
|
|
1067 computeEnrichmentMeanMedian(core_map, class_pat, ids, ARGS.color_map)
|
|
|
1068
|
|
|
1069
|
|
|
1070 enrichment_results = computeEnrichment(class_pat, ids)
|
|
|
1071 for i, j, comparisonDict, max_z_score in enrichment_results:
|
|
|
1072 map_copy = copy.deepcopy(core_map)
|
|
|
1073 temp_thingsInCommon(comparisonDict, map_copy, max_z_score, i, j)
|
|
|
1074 createOutputMaps(i, j, map_copy)
|
|
|
1075
|
|
|
1076 if not ERRORS: return
|
|
|
1077 utils.logWarning(
|
|
|
1078 f"The following reaction IDs were mentioned in the dataset but weren't found in the map: {ERRORS}",
|
|
|
1079 ARGS.out_log)
|
|
|
1080
|
|
|
1081 print('Execution succeded')
|
|
|
1082
|
|
|
1083 ###############################################################################
|
|
|
1084 if __name__ == "__main__":
|
|
|
1085 main()
|
|
|
1086
|
