Merge pull request #551 from ISA-tools/extended-511

Extended 511

isatools/constants.py

@@ -19,6 +19,9 @@
 DATA_FILE_LABELS = [
  'Raw Data File',
  'Raw Spectral Data File',
+ 'Free Induction Decay Data File',
+ 'Image File',
+ 'Derived Data File',
  'Derived Spectral Data File',
  'Derived Array Data File',
  'Derived Array Data Matrix File',
@@ -27,16 +30,16 @@
  'Peptide Assignment File',
  'Post Translational Modification Assignment File',
  'Acquisition Parameter Data File',
- 'Free Induction Decay Data File',
- 'Image File',
- 'Derived Data File',
  'Metabolite Assignment File',
  'Metabolite Identification File'
 ]
 
 _LABELS_DATA_NODES = [
  'Raw Data File',
  'Raw Spectral Data File',
+ 'Free Induction Decay Data File',
+ 'Image File',
+ 'Derived Data File',
  'Derived Spectral Data File',
  'Derived Array Data File',
  'Derived Array Data Matrix File',
@@ -45,9 +48,6 @@
  'Peptide Assignment File',
  'Post Translational Modification Assignment File',
  'Acquisition Parameter Data File',
- 'Free Induction Decay Data File',
- 'Image File',
- 'Derived Data File',
  'Metabolite Assignment File',
  'Metabolite Identification File'
 ]
@@ -65,16 +65,6 @@
  'Data Transformation Name'
 ]
 
-_LABELS_ASSAY_NODES = [
- 'Assay Name',
- 'MS Assay Name',
- 'NMR Assay Name',
- 'Hybridization Assay Name',
- 'Scan Name',
- 'Normalization Name',
- 'Data Transformation Name'
-]
-
 QUALIFIER_LABELS = [
  'Protocol REF',
  'Material Type',
@@ -83,6 +73,19 @@
  'Unit'
 ]
 
+ALLOWED_NODES = NODE_LABELS.append('Protocol REF')
+
 ALL_LABELS = NODE_LABELS + ASSAY_LABELS + QUALIFIER_LABELS
 
 ALL_LABELS.append('Protocol REF')
+ALL_LABELS.append('Label')
+
+_LABELS_ASSAY_NODES = [
+ 'Assay Name',
+ 'MS Assay Name',
+ 'NMR Assay Name',
+ 'Hybridization Assay Name',
+ 'Scan Name',
+ 'Normalization Name',
+ 'Data Transformation Name'
+]

isatools/create/model.py

@@ -3283,4 +3283,4 @@ def compute_single_arm_design_multi_element_cell(treatments, sample_assay_plan,
  elements=[follow_up_map[0]]), follow_up_map[1]])
  arm = StudyArm('ARM_00', group_size=group_size, arm_map=OrderedDict(arm_map))
  design.add_study_arm(arm)
- return design
+ return design

isatools/isajson/validate.py

@@ -810,14 +810,16 @@ def check_study_groups(study_or_assay):
 def validate(
  fp,
  config_dir=default_config_dir,
- log_level=None,
+ log_level=logging.INFO,
  base_schemas_dir="isa_model_version_1_0_schemas"
 ):
  if config_dir is None:
  config_dir = default_config_dir
- if log_level in (
- logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING,
- logging.ERROR, logging.CRITICAL):
+ if log_level is None: #(
+ # logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING,
+ # logging.ERROR, logging.CRITICAL):
+ log.disabled = True
+ else:
  log.setLevel(log_level)
  log.info("ISA JSON Validator from ISA tools API v0.12.")
  stream = StringIO()

isatools/isatab/dump/core.py

@@ -41,7 +41,7 @@ def dump(isa_obj, output_path,
  raise NameError('Investigation file must match pattern i_*.txt, got {}'.format(i_file_name))
 
  if path.exists(output_path):
- fp = open(path.join(output_path, i_file_name), 'w', encoding='utf-8')
+ fp = open(path.join(output_path, i_file_name), 'wb')
  else:
  log.debug('output_path=', i_file_name)
  raise FileNotFoundError("Can't find " + output_path)
@@ -55,7 +55,7 @@ def dump(isa_obj, output_path,
 
  # Write ONTOLOGY SOURCE REFERENCE section
  ontology_source_references_df = _build_ontology_reference_section(investigation.ontology_source_references)
- fp.write('ONTOLOGY SOURCE REFERENCE\n')
+ fp.write(bytearray('ONTOLOGY SOURCE REFERENCE\n', 'utf-8'))
  # Need to set index_label as top left cell
  ontology_source_references_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Term Source Name')
@@ -80,7 +80,7 @@ def dump(isa_obj, output_path,
  inv_df_rows.append(comment.value)
  investigation_df.loc[0] = inv_df_rows
  investigation_df = investigation_df.set_index('Investigation Identifier').T
- fp.write('INVESTIGATION\n')
+ fp.write(bytearray('INVESTIGATION\n', 'utf-8'))
  investigation_df.to_csv(
  path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Investigation Identifier')
@@ -90,14 +90,15 @@ def dump(isa_obj, output_path,
  prefix='Investigation',
  publications=investigation.publications
  )
- fp.write('INVESTIGATION PUBLICATIONS\n')
+ fp.write(bytearray('INVESTIGATION PUBLICATIONS\n', 'utf-8'))
  investigation_publications_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Investigation PubMed ID')
 
  # Write INVESTIGATION CONTACTS section
  investigation_contacts_df = _build_contacts_section_df(
  contacts=investigation.contacts)
- fp.write('INVESTIGATION CONTACTS\n')
+ fp.write(bytearray('INVESTIGATION CONTACTS\n', 'utf-8'))
+
  investigation_contacts_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Investigation Person Last Name')
 
@@ -127,40 +128,40 @@ def dump(isa_obj, output_path,
  study_df_row.append(comment.value)
  study_df.loc[0] = study_df_row
  study_df = study_df.set_index('Study Identifier').T
- fp.write('STUDY\n')
+ fp.write(bytearray('STUDY\n', 'utf-8'))
  study_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8', index_label='Study Identifier')
  study_design_descriptors_df = _build_design_descriptors_section(design_descriptors=study.design_descriptors)
- fp.write('STUDY DESIGN DESCRIPTORS\n')
+ fp.write(bytearray('STUDY DESIGN DESCRIPTORS\n', 'utf-8'))
  study_design_descriptors_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study Design Type')
 
  # Write STUDY PUBLICATIONS section
  study_publications_df = _build_publications_section_df(prefix='Study', publications=study.publications)
- fp.write('STUDY PUBLICATIONS\n')
+ fp.write(bytearray('STUDY PUBLICATIONS\n', 'utf-8'))
  study_publications_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study PubMed ID')
 
  # Write STUDY FACTORS section
  study_factors_df = _build_factors_section_df(factors=study.factors)
- fp.write('STUDY FACTORS\n')
+ fp.write(bytearray('STUDY FACTORS\n', 'utf-8'))
  study_factors_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study Factor Name')
 
  study_assays_df = _build_assays_section_df(assays=study.assays)
- fp.write('STUDY ASSAYS\n')
+ fp.write(bytearray('STUDY ASSAYS\n', 'utf-8'))
  study_assays_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study Assay File Name')
 
  # Write STUDY PROTOCOLS section
  study_protocols_df = _build_protocols_section_df(protocols=study.protocols)
- fp.write('STUDY PROTOCOLS\n')
+ fp.write(bytearray('STUDY PROTOCOLS\n', 'utf-8'))
  study_protocols_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study Protocol Name')
 
  # Write STUDY CONTACTS section
  study_contacts_df = _build_contacts_section_df(
  prefix='Study', contacts=study.contacts)
- fp.write('STUDY CONTACTS\n')
+ fp.write(bytearray('STUDY CONTACTS\n', 'utf-8'))
  study_contacts_df.to_csv(path_or_buf=fp, mode='a', sep='\t', encoding='utf-8',
  index_label='Study Person Last Name')
 

isatools/isatab/dump/write.py

@@ -59,7 +59,6 @@ def flatten(current_list):
  paths = _all_end_to_end_paths(
  s_graph,
  [x for x in s_graph.nodes() if isinstance(s_graph.indexes[x], Source)])
- log.warning(s_graph.nodes())
 
  sample_in_path_count = 0
  protocol_in_path_count = 0
@@ -220,7 +219,7 @@ def flatten(current_list):
  DF = DF.replace('', nan)
  DF = DF.dropna(axis=1, how='all')
 
- with open(path.join(output_dir, study_obj.filename), 'w') as out_fp:
+ with open(path.join(output_dir, study_obj.filename), 'wb') as out_fp:
  DF.to_csv(
  path_or_buf=out_fp, index=False, sep='\t', encoding='utf-8')
 
@@ -480,7 +479,7 @@ def pbar(x):
  DF = DF.dropna(axis=1, how='all')
 
  with open(path.join(
- output_dir, assay_obj.filename), 'w') as out_fp:
+ output_dir, assay_obj.filename), 'wb') as out_fp:
  DF.to_csv(path_or_buf=out_fp, index=False, sep='\t',
  encoding='utf-8')
 

isatools/isatab/graph.py

@@ -17,7 +17,7 @@ def _all_end_to_end_paths(G, start_nodes):
  num_start_nodes = len(start_nodes)
  message = 'Calculating for paths for {} start nodes: '.format(
  num_start_nodes)
- log.info(start_nodes)
+ # log.info(start_nodes)
  start_node = G.indexes[start_nodes[0]]
  if isinstance(start_node, Source):
  message = 'Calculating for paths for {} sources: '.format(
@@ -61,7 +61,6 @@ def _longest_path_and_attrs(paths, indexes):
  :return: The longest path and attributes
  """
  longest = (0, None)
- log.info(paths)
  for path in paths:
  length = len(path)
  for node in path:

isatools/isatab/load/ProcessSequenceFactory.py

@@ -376,6 +376,12 @@ def get_node_by_label_and_key(labl, this_key):
  if comment_key not in [x.name for x in process.comments]:
  process.comments.append(Comment(name=comment_key, value=str(object_series[comment_column])))
 
+ for performer in [c for c in column_group if c == 'Performer']:
+ process.performer = str(object_series[performer])
+
+ for date in [c for c in column_group if c == 'Date']:
+ process.date = str(object_series[date])
+
  for _, object_series in DF.iterrows(): # don't drop duplicates
  process_key_sequence = list()
  source_node_context = None

isatools/isatab/load/core.py

@@ -390,7 +390,7 @@ def merge_study_with_assay_tables(study_file_path, assay_file_path, target_file_
  log.info("Merging DataFrames...")
  merged_DF = merge(study_DF, assay_DF, on='Sample Name')
  log.info("Writing merged DataFrame to file %s", target_file_path)
- with open(target_file_path, 'w', encoding='utf-8') as fp:
+ with open(target_file_path, 'wb') as fp:
  merged_DF.to_csv(fp, sep='\t', index=False, header=study_DF.isatab_header + assay_DF.isatab_header[1:])
 
 

isatools/isatab/validate/core.py

@@ -3,6 +3,7 @@
 
 from os import path
 from glob import glob
+import logging
 
 from pandas.errors import ParserError
 
@@ -45,7 +46,7 @@ def check_labels(section, labels_expected, df):
  if _RX_COMMENT.match(label) is None:
  msg = "Invalid label found in investigation file"
  spl = "In {} section, label {} is not allowed".format(section, label)
- message_handler.add_error(message=msg, supplemental=spl, code= 5)
+ message_handler.add_error(message=msg, supplemental=spl, code=5)
  elif len(_RX_COMMENT.findall(label)) == 0:
  spl = "In {} section, label {} is missing a name".format(section, label)
  msg = "Missing name in Comment[] label"
@@ -179,8 +180,10 @@ def validate(fp: TextIO,
  :param log_level: optional log level (default: INFO)
  :return: a dictionary of the validation results (errors, warnings and info)
  """
- if not log_level:
+ if log_level is None:
  log.disabled = True
+ else:
+ log.setLevel(log_level)
  message_handler.reset_store()
  validated = False
 

isatools/isatab/validate/rules/rules_10xx.py

@@ -12,7 +12,7 @@
 def check_samples_not_declared_in_study_used_in_assay(i_df_dict, dir_context):
  """Checks if samples found in assay tables are found in the study-sample table
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :param dir_context: Path to where the investigation file is found
  :return: None
  """
@@ -43,7 +43,7 @@ def check_samples_not_declared_in_study_used_in_assay(i_df_dict, dir_context):
 def check_study_factor_usage(i_df_dict, dir_context):
  """Used for rules 1008 and 1021
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :param dir_context: Path to where the investigation file is found
  :return: None
  """
@@ -112,7 +112,7 @@ def check_study_factor_usage(i_df_dict, dir_context):
 def check_protocol_usage(i_df_dict, dir_context):
  """Used for rules 1007 and 1019
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :param dir_context: Path to where the investigation file is found
  :return: None
  """
@@ -186,7 +186,7 @@ def check_protocol_usage(i_df_dict, dir_context):
 def check_protocol_parameter_usage(i_df_dict, dir_context):
  """Used for rules 1009 and 1020
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :param dir_context: Path to where the investigation file is found
  :return: None
  """
@@ -282,7 +282,7 @@ def check_protocol_names(i_df_dict):
 def check_protocol_parameter_names(i_df_dict):
  """Used for rule 1011
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
  for study_protocols_df in i_df_dict['s_protocols']:
@@ -301,7 +301,7 @@ def check_protocol_parameter_names(i_df_dict):
 def check_study_factor_names(i_df_dict):
  """Used for rule 1012
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
  for study_factors_df in i_df_dict['s_factors']:

isatools/isatab/validate/rules/rules_30xx.py

@@ -8,7 +8,7 @@
 def check_filenames_present(i_df_dict: dict) -> None:
  """ Used for rule 3005
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
  for s_pos, study_df in enumerate(i_df_dict['studies']):
@@ -25,7 +25,7 @@ def check_filenames_present(i_df_dict: dict) -> None:
 def check_date_formats(i_df_dict):
  """ Used for rule 3001
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
 
@@ -61,7 +61,7 @@ def check_iso8601_date(date_str):
 def check_dois(i_df_dict):
  """ Used for rule 3002
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
 
@@ -87,7 +87,7 @@ def check_doi(doi_str):
 def check_pubmed_ids_format(i_df_dict):
  """ Used for rule 3003
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
 
@@ -113,7 +113,7 @@ def check_pubmed_id(pubmed_id_str):
 def check_ontology_sources(i_df_dict):
  """ Used for rule 3008
 
- :param i_df_dict: A dictionary of DataFrames and lists of DataFrames representing the investigation file
+ :param i_df_dict: A dictionary of  DataFrame and list of Dataframes representing the Investigation file
  :return: None
  """
  term_source_refs = []