Merge pull request #211 from Ensembl/mbarba/restructure

Restructure: move and rename python files and entrypoints

pipelines/nextflow/modules/database/db_factory.nf

@@ -25,10 +25,10 @@ process DB_FACTORY {
         path "dbs.json"
 
     script:
-        brc_mode = params.brc_mode ? '--brc_mode 1' : ''
-        dbname_re = filter_map.dbname_re ? "--dbname_re $filter_map.dbname_re" : ''
+        brc_mode = params.brc_mode ? '--brc_mode' : ''
+        dbname_re = filter_map.dbname_re ? "--db_regex $filter_map.dbname_re" : ''
         """
-        db_factory --host '${server.host}' \
+        database_factory --host '${server.host}' \
             --port '${server.port}' \
             --user '${server.user}' \
             --password '${server.password}' \

pipelines/nextflow/modules/download/download_asm_data.nf

@@ -36,6 +36,6 @@ process DOWNLOAD_ASM_DATA {
 
     shell:
         '''
-        retrieve_assembly_data --accession !{meta.accession} --asm_download_dir ./
+        assembly_download --accession !{meta.accession} --asm_download_dir ./
         '''
 }

pipelines/nextflow/modules/download/download_genbank.nf

@@ -27,6 +27,6 @@ process DOWNLOAD_GENBANK {
     shell:
     output_file = "output.gb"
     '''
-    download_genbank --accession !{meta.accession} --output_file !{output_file}
+    genbank_download --accession !{meta.accession} --output_file !{output_file}
     '''
 }

pipelines/nextflow/modules/events/dump_events.nf

@@ -28,7 +28,7 @@ process DUMP_EVENTS {
     script:
         """
         touch "events.txt"
-        events_dumper --host '${server.host}' \
+        events_dump --host '${server.host}' \
             --port '${server.port}' \
             --user '${server.user}' \
             --password '${server.password}' \

pipelines/nextflow/modules/fasta/process_fasta_data.nf

@@ -29,7 +29,7 @@ process PROCESS_FASTA {
     output_fasta = pep_mode == 1 ? "fasta_pep.fa" : "fasta_dna.fa"
     pep_mode_flag = pep_mode == 0 ? "" : "--peptide_mode"
     '''
-    prep_fasta_data --fasta_infile !{fasta_file} \
+    fasta_process --fasta_infile !{fasta_file} \
         --genbank_infile !{gbff_file} \
         --fasta_outfile !{output_fasta} \
         !{pep_mode_flag}

pipelines/nextflow/modules/files/collect_files_db.nf

@@ -52,7 +52,7 @@ process MANIFEST {
 
     script:
         """
-        manifest_maker --manifest_dir ${collect_dir}
+        manifest_generate --manifest_dir ${collect_dir}
         """
 }
 

pipelines/nextflow/modules/genbank/extract_from_gb.nf

@@ -29,7 +29,7 @@ process EXTRACT_FROM_GB {
 
     shell:
     '''
-    extract_from_gb --prefix !{meta.prefix} --prod_name !{meta.production_name} --gb_file !{gb_file}
+    genbank_extract_data --prefix !{meta.prefix} --prod_name !{meta.production_name} --gb_file !{gb_file}
     '''
 }
 

pipelines/nextflow/modules/genome_metadata/amend_genome_data.nf

@@ -26,7 +26,7 @@ process AMEND_GENOME_DATA {
 
     shell:
         '''
-        amend_genomic_data --genome_infile !{genome_json} \
+        genome_metadata_extend --genome_infile !{genome_json} \
             --report_file !{asm_report} \
             --genbank_infile !{genbank_gbff} \
             --genome_outfile genome.json

pipelines/nextflow/modules/genome_metadata/dump_genome_meta.nf

@@ -28,8 +28,7 @@ process DUMP_GENOME_META {
 
     script:
         """
-        touch $output
-        genome_meta_dumper --host '${server.host}' \
+        genome_metadata_dump --host '${server.host}' \
             --port '${server.port}' \
             --user '${server.user}' \
             --password '${server.password}' \

pipelines/nextflow/modules/genome_metadata/dump_genome_stats.nf

@@ -28,7 +28,7 @@ process DUMP_GENOME_STATS {
 
     script:
         """
-        genome_stats_dumper --host '${server.host}' \
+        genome_stats_dump --host '${server.host}' \
             --port '${server.port}' \
             --user '${server.user}' \
             --password '${server.password}' \

pipelines/nextflow/modules/genome_metadata/prepare_genome_metadata.nf

@@ -25,6 +25,6 @@ process PREPARE_GENOME_METADATA {
     shell:
     output_json = "genome.json"
     '''
-    prepare_genome_metadata --input_file !{input_json} --output_file !{output_json}
+    genome_metadata_prepare --input_file !{input_json} --output_file !{output_json}
     '''
 }

pipelines/nextflow/modules/gff3/process_gff3.nf

@@ -29,7 +29,7 @@ process PROCESS_GFF3 {
         out_func = "functional_annotation.json"
         out_gff = "gene_models.gff3"
         '''
-        process_gff3 --genome_data !{genome} --in_gff_path !{gff3} --out_gff_path !{out_gff} \
+        gff3_process --genome_data !{genome} --in_gff_path !{gff3} --out_gff_path !{out_gff} \
             --out_func_path !{out_func}
         '''
 }

pipelines/nextflow/modules/manifest/check_integrity_db.nf

@@ -27,8 +27,8 @@ process CHECK_INTEGRITY {
         tuple val(db), path(manifest_dir, includeInputs: true)
 
     script:
-        brc_mode = params.brc_mode ? '--brc_mode 1' : ''
+        brc_mode = params.brc_mode ? '--brc_mode' : ''
         """
-        check_integrity --manifest_file ${manifest_dir}/manifest.json $brc_mode
+        manifest_check_integrity --manifest_file ${manifest_dir}/manifest.json $brc_mode
         """
 }

pipelines/nextflow/modules/manifest/integrity.nf

@@ -29,6 +29,6 @@ process CHECK_INTEGRITY {
     shell:
         brc_mode = params.brc_mode ? '--brc_mode' : ''
         '''
-        check_integrity --manifest_file ./manifest.json !{brc_mode}
+        manifest_check_integrity --manifest_file ./manifest.json !{brc_mode}
         '''
 }

pipelines/nextflow/modules/manifest/manifest_maker.nf

@@ -25,6 +25,6 @@ process MANIFEST {
 
     shell:
         '''
-        manifest_maker --manifest_dir .
+        manifest_generate --manifest_dir .
         '''
 }

pipelines/nextflow/modules/manifest/manifest_stats.nf

@@ -31,6 +31,6 @@ process MANIFEST_STATS {
         if [ "!{ncbi_check}" == 1 ]; then
             CHECK_ASSEMBLY=" --datasets_bin !{datasets} --accession !{meta.accession}"
         fi
-        manifest_stats --manifest_dir ./ $CHECK_ASSEMBLY
+        manifest_compute_stats --manifest_dir ./ $CHECK_ASSEMBLY
         '''
 }

pipelines/nextflow/modules/patch_build/format_events.nf

@@ -28,7 +28,7 @@ process FORMAT_EVENTS {
     script:
     def final_events = "events.tab"
     """
-    format_events \\
+    events_format \\
     --input_file $events \\
     --map $new_genes \\
     --deletes $deletes \\

pipelines/nextflow/modules/patch_build/load_events.nf

@@ -22,7 +22,7 @@ process LOAD_EVENTS {
 
     script:
     """
-    events_loader \\
+    events_load \\
     --host $server.host \\
     --user $server.user \\
     --port $server.port \\

pipelines/nextflow/modules/schema/check_json_schema.nf

@@ -26,6 +26,6 @@ process CHECK_JSON_SCHEMA {
     shell:
         schema = params.json_schemas[json_file.baseName]
         '''
-        check_json_schema --json_file !{json_file} --json_schema !{schema}
+        schemas_json_validate --json_file !{json_file} --json_schema !{schema}
         '''
 }

pipelines/nextflow/modules/schema/check_json_schema_db.nf

@@ -30,6 +30,6 @@ process CHECK_JSON_SCHEMA {
         script_dir = workflow.projectDir.toString()
         schema_path = new File(script_dir + "/../../../../schemas", schema_name + "_schema.json")
         """
-        check_json_schema --json_file ${json_file} --json_schema ${schema_path}
+        schemas_json_validate --json_file ${json_file} --json_schema ${schema_path}
         """
 }

pipelines/nextflow/modules/schema/json_schema_factory.nf

@@ -26,6 +26,6 @@ process JSON_SCHEMA_FACTORY {
 
     script:
         """
-        json_schema_factory --manifest_dir ${manifest_dir} --metadata_types "${metadata_types}"
+        schemas_json_factory --manifest_dir ${manifest_dir} --metadata_types "${metadata_types}"
         """
 }

pipelines/nextflow/modules/seq_region/dump_seq_regions.nf

@@ -27,7 +27,7 @@ process DUMP_SEQ_REGIONS {
     script:
         """
         touch seq_region.json
-        seq_region_dumper --host '${server.host}' \
+        seq_region_dump --host '${server.host}' \
             --port '${server.port}' \
             --user '${server.user}' \
             --password '${server.password}' \

pipelines/nextflow/modules/seq_region/process_seq_region.nf

@@ -30,7 +30,7 @@ process PROCESS_SEQ_REGION {
     shell:
         brc_mode = params.brc_mode ? '--brc_mode' : ''
         '''
-        prepare_seq_region \
+        seq_region_prepare \
             --genome_file !{genome_json} \
             --report_file !{assembly_report} \
             --gbff_file !{genomic_gbff} \

pyproject.toml

@@ -78,37 +78,41 @@ homepage = "https://www.ensembl.org"
 repository = "https://github.com/Ensembl/ensembl-genomio"
 
 [project.scripts]
-# Download
-retrieve_assembly_data = "ensembl.io.genomio.assembly.get_assembly_data:main"
-# Genbank
-extract_from_gb = "ensembl.io.genomio.genbank.extract_from_genbank:main"
-download_genbank = "ensembl.io.genomio.genbank.get_genbank:main"
-# Prepare
-amend_genomic_data = "ensembl.io.genomio.metadata.update_genome_metadata:main"
-prep_fasta_data = "ensembl.io.genomio.fastaprep.process_fasta:main"
-prepare_genome_metadata = "ensembl.io.genomio.metadata.prepare_genome:main"
-prepare_seq_region = "ensembl.io.genomio.metadata.prepare_seq_region:main"
-process_gff3 = "ensembl.io.genomio.gff3.process_gff3:main"
-# Schema
-check_json_schema = "ensembl.io.genomio.schemas.json_schema_validator:main"
-json_schema_factory = "ensembl.io.genomio.schemas.json_schema_factory:main"
+# Assembly
+assembly_download = "ensembl.io.genomio.assembly.download:main"
+# Database
+database_factory = "ensembl.io.genomio.database.factory:main"
+# Events
+events_dump = "ensembl.io.genomio.events.dump:main"
+events_load = "ensembl.io.genomio.events.load:main"
+events_format = "ensembl.io.genomio.events.format:main"
+# Fasta
+fasta_process = "ensembl.io.genomio.fasta.process:main"
+# GenBank
+genbank_download = "ensembl.io.genomio.genbank.download:main"
+genbank_extract_data = "ensembl.io.genomio.genbank.extract_data:main"
+# Genome metadata
+genome_metadata_dump = "ensembl.io.genomio.genome_metadata.dump:main"
+genome_metadata_extend = "ensembl.io.genomio.genome_metadata.extend:main"
+genome_metadata_prepare = "ensembl.io.genomio.genome_metadata.prepare:main"
+# Genome statistics
+genome_stats_compare = "ensembl.io.genomio.genome_stats.compare:main"
+genome_stats_dump = "ensembl.io.genomio.genome_stats.dump:main"
+# GFF3
+gff3_extract_annotation = "ensembl.io.genomio.gff3.extract_annotation:main"
+gff3_process = "ensembl.io.genomio.gff3.process:main"
+# Manifest
+manifest_check_integrity = "ensembl.io.genomio.manifest.check_integrity:main"
+manifest_compute_stats = "ensembl.io.genomio.manifest.compute_stats:main"
+manifest_generate = "ensembl.io.genomio.manifest.generate:main"
+# Schemas
+schemas_json_factory = "ensembl.io.genomio.schemas.json.factory:main"
+schemas_json_validate = "ensembl.io.genomio.schemas.json.validate:main"
+# Sequence region
+seq_region_dump = "ensembl.io.genomio.seq_region.dump:main"
+seq_region_prepare = "ensembl.io.genomio.seq_region.prepare:main"
 # Utils
 extract_file = "ensembl.io.genomio.utils.archive_utils:extract_file_cli"
-# Manifest
-check_integrity = "ensembl.io.genomio.integrity:main"
-manifest_maker = "ensembl.io.genomio.manifest_maker:main"
-manifest_stats = "ensembl.io.genomio.manifest_stats:main"
-# DB
-db_factory = "ensembl.io.genomio.db_factory:main"
-# Dump
-seq_region_dumper = "ensembl.io.genomio.seq_region_dumper:main"
-genome_meta_dumper = "ensembl.io.genomio.genome_metadata.dump_genome_metadata:main"
-genome_stats_dumper = "ensembl.io.genomio.genome_metadata.dump_genome_stats:main"
-genome_stats_compare = "ensembl.io.genomio.genome_metadata.compare_genome_stats:main"
-# Events
-events_dumper = "ensembl.io.genomio.events_dumper:main"
-events_loader = "ensembl.io.genomio.events_loader:main"
-format_events = "ensembl.io.genomio.events.format_events:main"
 
 [tool.setuptools]
 package-dir = {"" = "src/python"}

src/python/ensembl/io/genomio/__init__.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-"""GenomIO modules."""
+"""Genome Input/Output (GenomIO) handling library."""
 
 __version__ = "0.1"

src/python/ensembl/io/genomio/assembly/__init__.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Assembly preparation module."""
 
-"""GenomIO assembly preparation modules."""
-
-from .get_assembly_data import *
+from .download import *

src/python/ensembl/io/genomio/fastaprep/__init__.py → src/python/ensembl/io/genomio/database/__init__.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Ensembl core database interface module."""
 
-"""GenomIO FASTA preparation modules."""
-
-from .process_fasta import *
+from .factory import *

src/python/ensembl/io/genomio/metadata/__init__.py → src/python/ensembl/io/genomio/events/__init__.py

@@ -12,9 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Gene events handling module."""
 
-"""GenomIO prepare modules."""
-
-from .prepare_genome import *
-from .prepare_seq_region import *
-from .update_genome_metadata import *
+from .dump import *
+from .format import *
+from .load import *

src/python/ensembl/io/genomio/events/format_events.py → src/python/ensembl/io/genomio/events/format.py

@@ -21,7 +21,7 @@
 import re
 from typing import Dict, List
 
-from ensembl.io.genomio.events_loader import EventCollection
+from ensembl.io.genomio.events.load import EventCollection
 from ensembl.utils.argparse import ArgumentParser
 
 

src/python/ensembl/io/genomio/fasta/__init__.py

@@ -0,0 +1,17 @@
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Fasta files processing module."""
+
+from .process import *

src/python/ensembl/io/genomio/genbank/__init__.py

@@ -0,0 +1,18 @@
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GenBank fetching and data manipulation module."""
+
+from .download import *
+from .extract_data import *

src/python/ensembl/io/genomio/genome_metadata/__init__.py

@@ -0,0 +1,19 @@
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Genome metadata handling module."""
+
+from .dump import *
+from .prepare import *
+from .extend import *