Update hyperparameter_importance.py

updates changes made locally

publications/2023-neurips/experiments/surf/snellius/analysis/hyperparameter_importance.py

@@ -16,21 +16,22 @@
 def parse_args():
  parser = argparse.ArgumentParser()
  parser.add_argument('--n_trees', type=int, default=16)
- parser.add_argument('--openml_ids', type=int, nargs='+', default=None)
+ parser.add_argument('--openml_ids', type=int, nargs='+', default=[3])
  parser.add_argument('--workflow_name', type=str, default="lcdb.workflow.sklearn.LibLinearWorkflow")
  parser.add_argument('--openml_taskid_name', type=str, default="m:openmlid")
  parser.add_argument('--output_directory', type=str, default=os.path.expanduser('~/experiments/lcdb'))
  parser.add_argument('--output_filetype', type=str, choices=['pdf', 'png'], default='png')
  parser.add_argument('--max_load', type=int, default=None)
+ parser.add_argument('--anchor_value', type=int, default=2048)
  return parser.parse_args()
 
 
 def numeric_encode(df, config_space):
  # https://automl.github.io/ConfigSpace/latest/api/ConfigSpace/configuration_space/
- result = np.zeros((len(df), len(config_space.get_hyperparameters())), dtype=float)
+ result = np.zeros((len(df), len(config_space.values())), dtype=float)
 
  for hyperparameter_name, hyperparameter in config_space.items():
- index = config_space.get_idx_by_hyperparameter_name(hyperparameter_name)
+ index = config_space.index_of[hyperparameter_name]
  if isinstance(hyperparameter, ConfigSpace.hyperparameters.NumericalHyperparameter):
  result[:, index] = df[hyperparameter_name].to_numpy()
  elif isinstance(hyperparameter, ConfigSpace.hyperparameters.Constant):
@@ -43,49 +44,50 @@ def numeric_encode(df, config_space):
  return result
 
 
-def fanova_on_task(task_results, performance_column_name, curve_data_column, config_space, n_trees):
+def fanova_on_task(task_results, performance_column_name, config_space, n_trees):
  fanova_results = []
 
- # query_confusion_matrix_values = lcdb.analysis.json.QueryMetricValuesFromAnchors("confusion_matrix", split_name="val")
- # out = task_results[performance_column_name].apply(query_confusion_matrix_values)
- # print(out)
- # balanced_error_rate_values_for_config = np.array(
- # out.apply(lambda x: list(map(lambda x: 1 - lcdb.analysis.score.balanced_accuracy_from_confusion_matrix(x), x))).to_list())
- # print(balanced_error_rate_values_for_config.mean(axis=0))
- # print(out)
  evaluator = fanova.fanova.fANOVA(
  X=numeric_encode(task_results, config_space),
  Y=task_results[performance_column_name].to_numpy(),
  config_space=config_space,
  n_trees=n_trees,
  )
- for idx, pname in enumerate(config_space.get_hyperparameter_names()):
+ for idx, pname in enumerate(config_space.keys()):
  logging.info('-- hyperparameter %d %s' % (idx, pname))
  unique_values = task_results[pname].unique()
  logging.info('-- UNIQUE VALUES: %d (%s)' % (len(unique_values), unique_values))
  importance = evaluator.quantify_importance([idx])
 
- fanova_results.append(
- {
- "hyperparameter": pname,
- "fanova": importance[(idx,)]["individual importance"],
- }
- )
-
+ fanova_results.append({
+ "hyperparameter": pname,
+ "fanova": importance[(idx,)]["individual importance"],
+ })
  return fanova_results
 
 
 def run(args):
  fanova_all_results = []
  performance_column = "objective"
- curve_data_column = "m:json"
+ anchor_size_column = "anchor_sizes"
+ learning_curve_column = "learning_curve_data"
 
  WorkflowClass = lcdb.builder.utils.import_attr_from_module(args.workflow_name)
  config_space = WorkflowClass.config_space()
- workflow_hyperparameter_mapping = {"p:" + name: name for name in config_space.get_hyperparameter_names()}
+ workflow_hyperparameter_mapping = {"p:" + name: name for name in config_space.keys()}
  id_results = dict()
 
- all_results_all_workflows = lcdb.db.LCDB().query(workflows=[args.workflow_name], openmlids=args.openml_ids)
+ all_results_all_workflows = lcdb.db.LCDB().query(
+ workflows=[args.workflow_name],
+ openmlids=args.openml_ids,
+ processors={
+ anchor_size_column: lcdb.analysis.json.QueryAnchorValues(),
+ learning_curve_column: lambda x: list(map(
+ lambda x: 1 - lcdb.analysis.score.balanced_accuracy_from_confusion_matrix(x),
+ lcdb.analysis.json.QueryMetricValuesFromAnchors("confusion_matrix", split_name="val")(x)
+ ))
+ }
+ )
  load_count = 0
  for frame_workflow_job_task in all_results_all_workflows:
  workflow_ids = frame_workflow_job_task['m:workflow'].unique()
@@ -95,6 +97,25 @@ def run(args):
  raise ValueError('Should not happen. %s %s' % (str(workflow_ids), str(openml_task_ids)))
  if (workflow_ids[0], openml_task_ids[0]) not in id_results:
  id_results[(workflow_ids[0], openml_task_ids[0])] = list()
+
+ performance_values_new = list()
+ for index, row in frame_workflow_job_task.iterrows():
+ anchor_sizes = row[anchor_size_column]
+ performance_value_at_anchor = np.nan
+ if args.anchor_value is not None:
+ if args.anchor_value not in anchor_sizes:
+ logging.warning('Anchor %d not available in task %d workflow %s'
+ % (args.anchor_value, openml_task_ids[0], workflow_ids[0])
+ )
+ else:
+ anchor_index = anchor_sizes.index(args.anchor_value)
+ performance_value_at_anchor = row[learning_curve_column][anchor_index]
+ else:
+ performance_value_at_anchor = row[learning_curve_column][-1]
+ performance_values_new.append(performance_value_at_anchor)
+ performance_values_new = np.array(performance_values_new, dtype=float)
+ frame_workflow_job_task[performance_column] = pd.Series(performance_values_new)
+
  id_results[(workflow_ids[0], openml_task_ids[0])].append(frame_workflow_job_task)
 
  load_count += 1
@@ -106,11 +127,11 @@ def run(args):
  task_ids.add(task_id)
  task_results = pd.concat(id_results[(workflow_name, task_id)])
  task_results = task_results.rename(workflow_hyperparameter_mapping, axis=1)
- relevant_columns = list(workflow_hyperparameter_mapping.values()) + [performance_column, curve_data_column]
+ relevant_columns = list(workflow_hyperparameter_mapping.values()) + [performance_column]
  task_results = task_results[relevant_columns]
 
  logging.info("Starting with task %d (%d/%d)" % (task_id, idx + 1, len(id_results)))
- fanova_task_results = fanova_on_task(task_results, performance_column, curve_data_column, config_space, args.n_trees)
+ fanova_task_results = fanova_on_task(task_results, performance_column, config_space, args.n_trees)
  fanova_all_results.extend(fanova_task_results)
 
  fanova_all_results = pd.DataFrame(fanova_all_results)
@@ -125,7 +146,10 @@ def run(args):
  plt.tight_layout()
 
  # save plot to file
- output_file = args.output_directory + '/fanova_%s.%s' % (args.workflow_name, args.output_filetype)
+ filename_suffix = ""
+ if args.anchor_value is not None:
+ filename_suffix = "_anchor_%d" % args.anchor_value
+ output_file = args.output_directory + '/fanova_%s%s.%s' % (args.workflow_name, filename_suffix, args.output_filetype)
  os.makedirs(args.output_directory, exist_ok=True)
  plt.savefig(output_file)
  logging.info('saved to %s' % output_file)