Mercurial > repos > bgruening > sklearn_feature_selection
comparison search_model_validation.py @ 18:ec25331946b8 draft
planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/sklearn commit c0a3a186966888e5787335a7628bf0a4382637e7
| author | bgruening |
|---|---|
| date | Tue, 14 May 2019 18:17:57 -0400 |
| parents | 2bbbac61e48d |
| children | 0b88494bdcac |
comparison
equal
deleted
inserted
replaced
| 17:2bbbac61e48d | 18:ec25331946b8 |
|---|---|
| 1 import argparse | |
| 2 import collections | |
| 1 import imblearn | 3 import imblearn |
| 2 import json | 4 import json |
| 3 import numpy as np | 5 import numpy as np |
| 4 import os | |
| 5 import pandas | 6 import pandas |
| 6 import pickle | 7 import pickle |
| 7 import skrebate | 8 import skrebate |
| 8 import sklearn | 9 import sklearn |
| 9 import sys | 10 import sys |
| 10 import xgboost | 11 import xgboost |
| 11 import warnings | 12 import warnings |
| 13 import iraps_classifier | |
| 14 import model_validations | |
| 15 import preprocessors | |
| 16 import feature_selectors | |
| 12 from imblearn import under_sampling, over_sampling, combine | 17 from imblearn import under_sampling, over_sampling, combine |
| 13 from imblearn.pipeline import Pipeline as imbPipeline | 18 from scipy.io import mmread |
| 14 from sklearn import (cluster, compose, decomposition, ensemble, feature_extraction, | 19 from mlxtend import classifier, regressor |
| 15 feature_selection, gaussian_process, kernel_approximation, metrics, | 20 from sklearn import (cluster, compose, decomposition, ensemble, |
| 16 model_selection, naive_bayes, neighbors, pipeline, preprocessing, | 21 feature_extraction, feature_selection, |
| 17 svm, linear_model, tree, discriminant_analysis) | 22 gaussian_process, kernel_approximation, metrics, |
| 23 model_selection, naive_bayes, neighbors, | |
| 24 pipeline, preprocessing, svm, linear_model, | |
| 25 tree, discriminant_analysis) | |
| 18 from sklearn.exceptions import FitFailedWarning | 26 from sklearn.exceptions import FitFailedWarning |
| 19 from sklearn.externals import joblib | 27 from sklearn.externals import joblib |
| 20 from utils import get_cv, get_scoring, get_X_y, load_model, read_columns, SafeEval | 28 from sklearn.model_selection._validation import _score |
| 21 | 29 |
| 22 | 30 from utils import (SafeEval, get_cv, get_scoring, get_X_y, |
| 23 N_JOBS = int(os.environ.get('GALAXY_SLOTS', 1)) | 31 load_model, read_columns) |
| 24 | 32 from model_validations import train_test_split |
| 25 | 33 |
| 26 def get_search_params(params_builder): | 34 |
| 35 N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1)) | |
| 36 CACHE_DIR = './cached' | |
| 37 NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', 'steps', | |
| 38 'nthread', 'verbose') | |
| 39 | |
| 40 | |
| 41 def _eval_search_params(params_builder): | |
| 27 search_params = {} | 42 search_params = {} |
| 28 safe_eval = SafeEval(load_scipy=True, load_numpy=True) | |
| 29 safe_eval_es = SafeEval(load_estimators=True) | |
| 30 | 43 |
| 31 for p in params_builder['param_set']: | 44 for p in params_builder['param_set']: |
| 32 search_p = p['search_param_selector']['search_p'] | 45 search_list = p['sp_list'].strip() |
| 33 if search_p.strip() == '': | 46 if search_list == '': |
| 34 continue | 47 continue |
| 35 param_type = p['search_param_selector']['selected_param_type'] | 48 |
| 36 | 49 param_name = p['sp_name'] |
| 37 lst = search_p.split(':') | 50 if param_name.lower().endswith(NON_SEARCHABLE): |
| 38 assert (len(lst) == 2), "Error, make sure there is one and only one colon in search parameter input." | 51 print("Warning: `%s` is not eligible for search and was " |
| 39 literal = lst[1].strip() | 52 "omitted!" % param_name) |
| 40 param_name = lst[0].strip() | 53 continue |
| 41 if param_name: | 54 |
| 42 if param_name.lower() == 'n_jobs': | 55 if not search_list.startswith(':'): |
| 43 sys.exit("Parameter `%s` is invalid for search." %param_name) | 56 safe_eval = SafeEval(load_scipy=True, load_numpy=True) |
| 44 elif not param_name.endswith('-'): | 57 ev = safe_eval(search_list) |
| 45 ev = safe_eval(literal) | 58 search_params[param_name] = ev |
| 46 if param_type == 'final_estimator_p': | 59 else: |
| 47 search_params['estimator__' + param_name] = ev | 60 # Have `:` before search list, asks for estimator evaluatio |
| 48 else: | 61 safe_eval_es = SafeEval(load_estimators=True) |
| 49 search_params['preprocessing_' + param_type[5:6] + '__' + param_name] = ev | 62 search_list = search_list[1:].strip() |
| 50 else: | 63 # TODO maybe add regular express check |
| 51 # only for estimator eval, add `-` to the end of param | 64 ev = safe_eval_es(search_list) |
| 52 #TODO maybe add regular express check | 65 preprocessors = ( |
| 53 ev = safe_eval_es(literal) | 66 preprocessing.StandardScaler(), preprocessing.Binarizer(), |
| 54 for obj in ev: | 67 preprocessing.Imputer(), preprocessing.MaxAbsScaler(), |
| 55 if 'n_jobs' in obj.get_params(): | 68 preprocessing.Normalizer(), preprocessing.MinMaxScaler(), |
| 56 obj.set_params( n_jobs=N_JOBS ) | 69 preprocessing.PolynomialFeatures(), |
| 57 if param_type == 'final_estimator_p': | 70 preprocessing.RobustScaler(), feature_selection.SelectKBest(), |
| 58 search_params['estimator__' + param_name[:-1]] = ev | 71 feature_selection.GenericUnivariateSelect(), |
| 59 else: | 72 feature_selection.SelectPercentile(), |
| 60 search_params['preprocessing_' + param_type[5:6] + '__' + param_name[:-1]] = ev | 73 feature_selection.SelectFpr(), feature_selection.SelectFdr(), |
| 61 elif param_type != 'final_estimator_p': | 74 feature_selection.SelectFwe(), |
| 62 #TODO regular express check ? | 75 feature_selection.VarianceThreshold(), |
| 63 ev = safe_eval_es(literal) | 76 decomposition.FactorAnalysis(random_state=0), |
| 64 preprocessors = [preprocessing.StandardScaler(), preprocessing.Binarizer(), preprocessing.Imputer(), | 77 decomposition.FastICA(random_state=0), |
| 65 preprocessing.MaxAbsScaler(), preprocessing.Normalizer(), preprocessing.MinMaxScaler(), | 78 decomposition.IncrementalPCA(), |
| 66 preprocessing.PolynomialFeatures(),preprocessing.RobustScaler(), | 79 decomposition.KernelPCA(random_state=0, n_jobs=N_JOBS), |
| 67 feature_selection.SelectKBest(), feature_selection.GenericUnivariateSelect(), | 80 decomposition.LatentDirichletAllocation( |
| 68 feature_selection.SelectPercentile(), feature_selection.SelectFpr(), feature_selection.SelectFdr(), | 81 random_state=0, n_jobs=N_JOBS), |
| 69 feature_selection.SelectFwe(), feature_selection.VarianceThreshold(), | 82 decomposition.MiniBatchDictionaryLearning( |
| 70 decomposition.FactorAnalysis(random_state=0), decomposition.FastICA(random_state=0), decomposition.IncrementalPCA(), | 83 random_state=0, n_jobs=N_JOBS), |
| 71 decomposition.KernelPCA(random_state=0, n_jobs=N_JOBS), decomposition.LatentDirichletAllocation(random_state=0, n_jobs=N_JOBS), | 84 decomposition.MiniBatchSparsePCA( |
| 72 decomposition.MiniBatchDictionaryLearning(random_state=0, n_jobs=N_JOBS), | 85 random_state=0, n_jobs=N_JOBS), |
| 73 decomposition.MiniBatchSparsePCA(random_state=0, n_jobs=N_JOBS), decomposition.NMF(random_state=0), | 86 decomposition.NMF(random_state=0), |
| 74 decomposition.PCA(random_state=0), decomposition.SparsePCA(random_state=0, n_jobs=N_JOBS), | 87 decomposition.PCA(random_state=0), |
| 75 decomposition.TruncatedSVD(random_state=0), | 88 decomposition.SparsePCA(random_state=0, n_jobs=N_JOBS), |
| 76 kernel_approximation.Nystroem(random_state=0), kernel_approximation.RBFSampler(random_state=0), | 89 decomposition.TruncatedSVD(random_state=0), |
| 77 kernel_approximation.AdditiveChi2Sampler(), kernel_approximation.SkewedChi2Sampler(random_state=0), | 90 kernel_approximation.Nystroem(random_state=0), |
| 78 cluster.FeatureAgglomeration(), | 91 kernel_approximation.RBFSampler(random_state=0), |
| 79 skrebate.ReliefF(n_jobs=N_JOBS), skrebate.SURF(n_jobs=N_JOBS), skrebate.SURFstar(n_jobs=N_JOBS), | 92 kernel_approximation.AdditiveChi2Sampler(), |
| 80 skrebate.MultiSURF(n_jobs=N_JOBS), skrebate.MultiSURFstar(n_jobs=N_JOBS), | 93 kernel_approximation.SkewedChi2Sampler(random_state=0), |
| 81 imblearn.under_sampling.ClusterCentroids(random_state=0, n_jobs=N_JOBS), | 94 cluster.FeatureAgglomeration(), |
| 82 imblearn.under_sampling.CondensedNearestNeighbour(random_state=0, n_jobs=N_JOBS), | 95 skrebate.ReliefF(n_jobs=N_JOBS), |
| 83 imblearn.under_sampling.EditedNearestNeighbours(random_state=0, n_jobs=N_JOBS), | 96 skrebate.SURF(n_jobs=N_JOBS), |
| 84 imblearn.under_sampling.RepeatedEditedNearestNeighbours(random_state=0, n_jobs=N_JOBS), | 97 skrebate.SURFstar(n_jobs=N_JOBS), |
| 85 imblearn.under_sampling.AllKNN(random_state=0, n_jobs=N_JOBS), | 98 skrebate.MultiSURF(n_jobs=N_JOBS), |
| 86 imblearn.under_sampling.InstanceHardnessThreshold(random_state=0, n_jobs=N_JOBS), | 99 skrebate.MultiSURFstar(n_jobs=N_JOBS), |
| 87 imblearn.under_sampling.NearMiss(random_state=0, n_jobs=N_JOBS), | 100 imblearn.under_sampling.ClusterCentroids( |
| 88 imblearn.under_sampling.NeighbourhoodCleaningRule(random_state=0, n_jobs=N_JOBS), | 101 random_state=0, n_jobs=N_JOBS), |
| 89 imblearn.under_sampling.OneSidedSelection(random_state=0, n_jobs=N_JOBS), | 102 imblearn.under_sampling.CondensedNearestNeighbour( |
| 90 imblearn.under_sampling.RandomUnderSampler(random_state=0), | 103 random_state=0, n_jobs=N_JOBS), |
| 91 imblearn.under_sampling.TomekLinks(random_state=0, n_jobs=N_JOBS), | 104 imblearn.under_sampling.EditedNearestNeighbours( |
| 92 imblearn.over_sampling.ADASYN(random_state=0, n_jobs=N_JOBS), | 105 random_state=0, n_jobs=N_JOBS), |
| 93 imblearn.over_sampling.RandomOverSampler(random_state=0), | 106 imblearn.under_sampling.RepeatedEditedNearestNeighbours( |
| 94 imblearn.over_sampling.SMOTE(random_state=0, n_jobs=N_JOBS), | 107 random_state=0, n_jobs=N_JOBS), |
| 95 imblearn.over_sampling.SVMSMOTE(random_state=0, n_jobs=N_JOBS), | 108 imblearn.under_sampling.AllKNN(random_state=0, n_jobs=N_JOBS), |
| 96 imblearn.over_sampling.BorderlineSMOTE(random_state=0, n_jobs=N_JOBS), | 109 imblearn.under_sampling.InstanceHardnessThreshold( |
| 97 imblearn.over_sampling.SMOTENC(categorical_features=[], random_state=0, n_jobs=N_JOBS), | 110 random_state=0, n_jobs=N_JOBS), |
| 98 imblearn.combine.SMOTEENN(random_state=0), imblearn.combine.SMOTETomek(random_state=0)] | 111 imblearn.under_sampling.NearMiss( |
| 112 random_state=0, n_jobs=N_JOBS), | |
| 113 imblearn.under_sampling.NeighbourhoodCleaningRule( | |
| 114 random_state=0, n_jobs=N_JOBS), | |
| 115 imblearn.under_sampling.OneSidedSelection( | |
| 116 random_state=0, n_jobs=N_JOBS), | |
| 117 imblearn.under_sampling.RandomUnderSampler( | |
| 118 random_state=0), | |
| 119 imblearn.under_sampling.TomekLinks( | |
| 120 random_state=0, n_jobs=N_JOBS), | |
| 121 imblearn.over_sampling.ADASYN(random_state=0, n_jobs=N_JOBS), | |
| 122 imblearn.over_sampling.RandomOverSampler(random_state=0), | |
| 123 imblearn.over_sampling.SMOTE(random_state=0, n_jobs=N_JOBS), | |
| 124 imblearn.over_sampling.SVMSMOTE(random_state=0, n_jobs=N_JOBS), | |
| 125 imblearn.over_sampling.BorderlineSMOTE( | |
| 126 random_state=0, n_jobs=N_JOBS), | |
| 127 imblearn.over_sampling.SMOTENC( | |
| 128 categorical_features=[], random_state=0, n_jobs=N_JOBS), | |
| 129 imblearn.combine.SMOTEENN(random_state=0), | |
| 130 imblearn.combine.SMOTETomek(random_state=0)) | |
| 99 newlist = [] | 131 newlist = [] |
| 100 for obj in ev: | 132 for obj in ev: |
| 101 if obj is None: | 133 if obj is None: |
| 102 newlist.append(None) | 134 newlist.append(None) |
| 103 elif obj == 'all_0': | 135 elif obj == 'all_0': |
| 112 newlist.extend(preprocessors[26:30]) | 144 newlist.extend(preprocessors[26:30]) |
| 113 elif obj == 'reb_all': | 145 elif obj == 'reb_all': |
| 114 newlist.extend(preprocessors[31:36]) | 146 newlist.extend(preprocessors[31:36]) |
| 115 elif obj == 'imb_all': | 147 elif obj == 'imb_all': |
| 116 newlist.extend(preprocessors[36:55]) | 148 newlist.extend(preprocessors[36:55]) |
| 117 elif type(obj) is int and -1 < obj < len(preprocessors): | 149 elif type(obj) is int and -1 < obj < len(preprocessors): |
| 118 newlist.append(preprocessors[obj]) | 150 newlist.append(preprocessors[obj]) |
| 119 elif hasattr(obj, 'get_params'): # user object | 151 elif hasattr(obj, 'get_params'): # user uploaded object |
| 120 if 'n_jobs' in obj.get_params(): | 152 if 'n_jobs' in obj.get_params(): |
| 121 newlist.append( obj.set_params(n_jobs=N_JOBS) ) | 153 newlist.append(obj.set_params(n_jobs=N_JOBS)) |
| 122 else: | 154 else: |
| 123 newlist.append(obj) | 155 newlist.append(obj) |
| 124 else: | 156 else: |
| 125 sys.exit("Unsupported preprocessor type: %r" %(obj)) | 157 sys.exit("Unsupported estimator type: %r" % (obj)) |
| 126 search_params['preprocessing_' + param_type[5:6]] = newlist | 158 |
| 127 else: | 159 search_params[param_name] = newlist |
| 128 sys.exit("Parameter name of the final estimator can't be skipped!") | |
| 129 | 160 |
| 130 return search_params | 161 return search_params |
| 131 | 162 |
| 132 | 163 |
| 133 if __name__ == '__main__': | 164 def main(inputs, infile_estimator, infile1, infile2, |
| 165 outfile_result, outfile_object=None, groups=None): | |
| 166 """ | |
| 167 Parameter | |
| 168 --------- | |
| 169 inputs : str | |
| 170 File path to galaxy tool parameter | |
| 171 | |
| 172 infile_estimator : str | |
| 173 File path to estimator | |
| 174 | |
| 175 infile1 : str | |
| 176 File path to dataset containing features | |
| 177 | |
| 178 infile2 : str | |
| 179 File path to dataset containing target values | |
| 180 | |
| 181 outfile_result : str | |
| 182 File path to save the results, either cv_results or test result | |
| 183 | |
| 184 outfile_object : str, optional | |
| 185 File path to save searchCV object | |
| 186 | |
| 187 groups : str | |
| 188 File path to dataset containing groups labels | |
| 189 """ | |
| 134 | 190 |
| 135 warnings.simplefilter('ignore') | 191 warnings.simplefilter('ignore') |
| 136 | 192 |
| 137 input_json_path = sys.argv[1] | 193 with open(inputs, 'r') as param_handler: |
| 138 with open(input_json_path, 'r') as param_handler: | |
| 139 params = json.load(param_handler) | 194 params = json.load(param_handler) |
| 140 | 195 if groups: |
| 141 infile_pipeline = sys.argv[2] | 196 (params['search_schemes']['options']['cv_selector'] |
| 142 infile1 = sys.argv[3] | 197 ['groups_selector']['infile_g']) = groups |
| 143 infile2 = sys.argv[4] | |
| 144 outfile_result = sys.argv[5] | |
| 145 if len(sys.argv) > 6: | |
| 146 outfile_estimator = sys.argv[6] | |
| 147 else: | |
| 148 outfile_estimator = None | |
| 149 | 198 |
| 150 params_builder = params['search_schemes']['search_params_builder'] | 199 params_builder = params['search_schemes']['search_params_builder'] |
| 151 | 200 |
| 152 input_type = params['input_options']['selected_input'] | 201 input_type = params['input_options']['selected_input'] |
| 153 if input_type == 'tabular': | 202 if input_type == 'tabular': |
| 154 header = 'infer' if params['input_options']['header1'] else None | 203 header = 'infer' if params['input_options']['header1'] else None |
| 155 column_option = params['input_options']['column_selector_options_1']['selected_column_selector_option'] | 204 column_option = (params['input_options']['column_selector_options_1'] |
| 156 if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: | 205 ['selected_column_selector_option']) |
| 206 if column_option in ['by_index_number', 'all_but_by_index_number', | |
| 207 'by_header_name', 'all_but_by_header_name']: | |
| 157 c = params['input_options']['column_selector_options_1']['col1'] | 208 c = params['input_options']['column_selector_options_1']['col1'] |
| 158 else: | 209 else: |
| 159 c = None | 210 c = None |
| 160 X = read_columns( | 211 X = read_columns( |
| 161 infile1, | 212 infile1, |
| 162 c = c, | 213 c=c, |
| 163 c_option = column_option, | 214 c_option=column_option, |
| 164 sep='\t', | 215 sep='\t', |
| 165 header=header, | 216 header=header, |
| 166 parse_dates=True | 217 parse_dates=True).astype(float) |
| 167 ) | |
| 168 else: | 218 else: |
| 169 X = mmread(open(infile1, 'r')) | 219 X = mmread(open(infile1, 'r')) |
| 170 | 220 |
| 171 header = 'infer' if params['input_options']['header2'] else None | 221 header = 'infer' if params['input_options']['header2'] else None |
| 172 column_option = params['input_options']['column_selector_options_2']['selected_column_selector_option2'] | 222 column_option = (params['input_options']['column_selector_options_2'] |
| 173 if column_option in ['by_index_number', 'all_but_by_index_number', 'by_header_name', 'all_but_by_header_name']: | 223 ['selected_column_selector_option2']) |
| 224 if column_option in ['by_index_number', 'all_but_by_index_number', | |
| 225 'by_header_name', 'all_but_by_header_name']: | |
| 174 c = params['input_options']['column_selector_options_2']['col2'] | 226 c = params['input_options']['column_selector_options_2']['col2'] |
| 175 else: | 227 else: |
| 176 c = None | 228 c = None |
| 177 y = read_columns( | 229 y = read_columns( |
| 178 infile2, | 230 infile2, |
| 179 c = c, | 231 c=c, |
| 180 c_option = column_option, | 232 c_option=column_option, |
| 181 sep='\t', | 233 sep='\t', |
| 182 header=header, | 234 header=header, |
| 183 parse_dates=True | 235 parse_dates=True) |
| 184 ) | |
| 185 y = y.ravel() | 236 y = y.ravel() |
| 186 | 237 |
| 187 optimizer = params['search_schemes']['selected_search_scheme'] | 238 optimizer = params['search_schemes']['selected_search_scheme'] |
| 188 optimizer = getattr(model_selection, optimizer) | 239 optimizer = getattr(model_selection, optimizer) |
| 189 | 240 |
| 190 options = params['search_schemes']['options'] | 241 options = params['search_schemes']['options'] |
| 242 | |
| 191 splitter, groups = get_cv(options.pop('cv_selector')) | 243 splitter, groups = get_cv(options.pop('cv_selector')) |
| 192 if groups is None: | 244 options['cv'] = splitter |
| 193 options['cv'] = splitter | |
| 194 elif groups == '': | |
| 195 options['cv'] = list( splitter.split(X, y, groups=None) ) | |
| 196 else: | |
| 197 options['cv'] = list( splitter.split(X, y, groups=groups) ) | |
| 198 options['n_jobs'] = N_JOBS | 245 options['n_jobs'] = N_JOBS |
| 199 primary_scoring = options['scoring']['primary_scoring'] | 246 primary_scoring = options['scoring']['primary_scoring'] |
| 200 options['scoring'] = get_scoring(options['scoring']) | 247 options['scoring'] = get_scoring(options['scoring']) |
| 201 if options['error_score']: | 248 if options['error_score']: |
| 202 options['error_score'] = 'raise' | 249 options['error_score'] = 'raise' |
| 203 else: | 250 else: |
| 204 options['error_score'] = np.NaN | 251 options['error_score'] = np.NaN |
| 205 if options['refit'] and isinstance(options['scoring'], dict): | 252 if options['refit'] and isinstance(options['scoring'], dict): |
| 206 options['refit'] = 'primary' | 253 options['refit'] = primary_scoring |
| 207 if 'pre_dispatch' in options and options['pre_dispatch'] == '': | 254 if 'pre_dispatch' in options and options['pre_dispatch'] == '': |
| 208 options['pre_dispatch'] = None | 255 options['pre_dispatch'] = None |
| 209 | 256 |
| 210 with open(infile_pipeline, 'rb') as pipeline_handler: | 257 with open(infile_estimator, 'rb') as estimator_handler: |
| 211 pipeline = load_model(pipeline_handler) | 258 estimator = load_model(estimator_handler) |
| 212 | 259 |
| 213 search_params = get_search_params(params_builder) | 260 memory = joblib.Memory(location=CACHE_DIR, verbose=0) |
| 214 searcher = optimizer(pipeline, search_params, **options) | 261 # cache iraps_core fits could increase search speed significantly |
| 262 if estimator.__class__.__name__ == 'IRAPSClassifier': | |
| 263 estimator.set_params(memory=memory) | |
| 264 else: | |
| 265 for p, v in estimator.get_params().items(): | |
| 266 if p.endswith('memory'): | |
| 267 if len(p) > 8 and p[:-8].endswith('irapsclassifier'): | |
| 268 # cache iraps_core fits could increase search | |
| 269 # speed significantly | |
| 270 new_params = {p: memory} | |
| 271 estimator.set_params(**new_params) | |
| 272 elif v: | |
| 273 new_params = {p, None} | |
| 274 estimator.set_params(**new_params) | |
| 275 elif p.endswith('n_jobs'): | |
| 276 new_params = {p: 1} | |
| 277 estimator.set_params(**new_params) | |
| 278 | |
| 279 param_grid = _eval_search_params(params_builder) | |
| 280 searcher = optimizer(estimator, param_grid, **options) | |
| 281 | |
| 282 # do train_test_split | |
| 283 do_train_test_split = params['train_test_split'].pop('do_split') | |
| 284 if do_train_test_split == 'yes': | |
| 285 # make sure refit is choosen | |
| 286 if not options['refit']: | |
| 287 raise ValueError("Refit must be `True` for shuffle splitting!") | |
| 288 split_options = params['train_test_split'] | |
| 289 | |
| 290 # splits | |
| 291 if split_options['shuffle'] == 'stratified': | |
| 292 split_options['labels'] = y | |
| 293 X, X_test, y, y_test = train_test_split(X, y, **split_options) | |
| 294 elif split_options['shuffle'] == 'group': | |
| 295 if not groups: | |
| 296 raise ValueError("No group based CV option was " | |
| 297 "choosen for group shuffle!") | |
| 298 split_options['labels'] = groups | |
| 299 X, X_test, y, y_test, groups, _ =\ | |
| 300 train_test_split(X, y, **split_options) | |
| 301 else: | |
| 302 if split_options['shuffle'] == 'None': | |
| 303 split_options['shuffle'] = None | |
| 304 X, X_test, y, y_test =\ | |
| 305 train_test_split(X, y, **split_options) | |
| 306 # end train_test_split | |
| 215 | 307 |
| 216 if options['error_score'] == 'raise': | 308 if options['error_score'] == 'raise': |
| 217 searcher.fit(X, y) | 309 searcher.fit(X, y, groups=groups) |
| 218 else: | 310 else: |
| 219 warnings.simplefilter('always', FitFailedWarning) | 311 warnings.simplefilter('always', FitFailedWarning) |
| 220 with warnings.catch_warnings(record=True) as w: | 312 with warnings.catch_warnings(record=True) as w: |
| 221 try: | 313 try: |
| 222 searcher.fit(X, y) | 314 searcher.fit(X, y, groups=groups) |
| 223 except ValueError: | 315 except ValueError: |
| 224 pass | 316 pass |
| 225 for warning in w: | 317 for warning in w: |
| 226 print(repr(warning.message)) | 318 print(repr(warning.message)) |
| 227 | 319 |
| 228 cv_result = pandas.DataFrame(searcher.cv_results_) | 320 if do_train_test_split == 'no': |
| 229 cv_result.rename(inplace=True, columns={'mean_test_primary': 'mean_test_'+primary_scoring, 'rank_test_primary': 'rank_test_'+primary_scoring}) | 321 # save results |
| 230 cv_result.to_csv(path_or_buf=outfile_result, sep='\t', header=True, index=False) | 322 cv_results = pandas.DataFrame(searcher.cv_results_) |
| 231 | 323 cv_results = cv_results[sorted(cv_results.columns)] |
| 232 if outfile_estimator: | 324 cv_results.to_csv(path_or_buf=outfile_result, sep='\t', |
| 233 with open(outfile_estimator, 'wb') as output_handler: | 325 header=True, index=False) |
| 234 pickle.dump(searcher.best_estimator_, output_handler, pickle.HIGHEST_PROTOCOL) | 326 |
| 327 # output test result using best_estimator_ | |
| 328 else: | |
| 329 best_estimator_ = searcher.best_estimator_ | |
| 330 if isinstance(options['scoring'], collections.Mapping): | |
| 331 is_multimetric = True | |
| 332 else: | |
| 333 is_multimetric = False | |
| 334 | |
| 335 test_score = _score(best_estimator_, X_test, | |
| 336 y_test, options['scoring'], | |
| 337 is_multimetric=is_multimetric) | |
| 338 if not is_multimetric: | |
| 339 test_score = {primary_scoring: test_score} | |
| 340 for key, value in test_score.items(): | |
| 341 test_score[key] = [value] | |
| 342 result_df = pandas.DataFrame(test_score) | |
| 343 result_df.to_csv(path_or_buf=outfile_result, sep='\t', | |
| 344 header=True, index=False) | |
| 345 | |
| 346 memory.clear(warn=False) | |
| 347 | |
| 348 if outfile_object: | |
| 349 with open(outfile_object, 'wb') as output_handler: | |
| 350 pickle.dump(searcher, output_handler, pickle.HIGHEST_PROTOCOL) | |
| 351 | |
| 352 | |
| 353 if __name__ == '__main__': | |
| 354 aparser = argparse.ArgumentParser() | |
| 355 aparser.add_argument("-i", "--inputs", dest="inputs", required=True) | |
| 356 aparser.add_argument("-e", "--estimator", dest="infile_estimator") | |
| 357 aparser.add_argument("-X", "--infile1", dest="infile1") | |
| 358 aparser.add_argument("-y", "--infile2", dest="infile2") | |
| 359 aparser.add_argument("-r", "--outfile_result", dest="outfile_result") | |
| 360 aparser.add_argument("-o", "--outfile_object", dest="outfile_object") | |
| 361 aparser.add_argument("-g", "--groups", dest="groups") | |
| 362 args = aparser.parse_args() | |
| 363 | |
| 364 main(args.inputs, args.infile_estimator, args.infile1, args.infile2, | |
| 365 args.outfile_result, outfile_object=args.outfile_object, | |
| 366 groups=args.groups) |
