From e26076d498947d42d19290a00fbd201ca204bee6 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 16 Jul 2024 14:37:37 +0200 Subject: [PATCH 01/14] added jolt-physics with recipe v1 --- recipes/jolt-physics/recipe.yaml | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 recipes/jolt-physics/recipe.yaml diff --git a/recipes/jolt-physics/recipe.yaml b/recipes/jolt-physics/recipe.yaml new file mode 100644 index 0000000000000..1f437594c3ba1 --- /dev/null +++ b/recipes/jolt-physics/recipe.yaml @@ -0,0 +1,62 @@ +context: + version: 5.0.0 + +package: + name: jolt-physics + version: ${{ version }} + +source: + url: https://github.com/jrouwe/JoltPhysics/archive/refs/tags/v${{ version }}.zip + sha256: 2c98750316d565d553cb2c15d494b6b2ea4fbe55eb2ca18df536c9f4ac340d36 + +requirements: + build: + - ${{ compiler("cxx") }} + - cmake + - ninja + +build: + number: 0 + script: + - if: win + then: | + cmake -GNinja ^ + %CMAKE_ARGS% ^ + -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_BUILD_TYPE=Distribution ^ + -DCROSS_PLATFORM_DETERMINISTIC=ON ^ + -DTARGET_VIEWER=OFF ^ + -DTARGET_SAMPLES=OFF ^ + -DTARGET_HELLO_WORLD=OFF ^ + -DTARGET_UNIT_TESTS=OFF ^ + -DTARGET_PERFORMANCE_TEST=OFF ^ + -S %SRC_DIR%\Build + cmake --build . --target install + else: | + cmake -GNinja \ + $CMAKE_ARGS \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Distribution \ + -DCROSS_PLATFORM_DETERMINISTIC=ON \ + -DTARGET_VIEWER=OFF \ + -DTARGET_SAMPLES=OFF \ + -DTARGET_HELLO_WORLD=OFF \ + -DTARGET_UNIT_TESTS=OFF \ + -DTARGET_PERFORMANCE_TEST=OFF \ + -S $SRC_DIR/Build + cmake --build . --target install + +about: + homepage: https://github.com/jrouwe/JoltPhysics + license: MIT + license_file: LICENSE + summary: A multi core friendly rigid body physics and collision detection library. + description: A multi core friendly rigid body physics and collision detection library. Written in C++. Suitable for games and VR applications. Used by Horizon Forbidden West. + documentation: https://xtensor.readthedocs.iohttps://jrouwe.github.io/JoltPhysics/ + repository: https://github.com/jrouwe/JoltPhysics + +extra: + recipe-maintainers: + - baszalmstra From 70b6472bed60ee03dc5872f069fe4826ac267819 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 18 Jul 2024 09:37:50 +0200 Subject: [PATCH 02/14] modify build_all.py to use rattler-build --- .ci_support/build_all.py | 50 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 53014e054c101..d0c2ec9944846 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,3 +1,5 @@ +from itertools import chain +import json import conda.base.context import conda.core.index import conda.resolve @@ -44,17 +46,32 @@ def build_all(recipes_dir, arch): script_dir = os.path.dirname(os.path.realpath(__file__)) variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch))) + has_meta_yaml = False + has_recipe_yaml = False + found_cuda = False found_centos7 = False for folder in folders: meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml") if os.path.exists(meta_yaml): + has_meta_yaml = True with(open(meta_yaml, "r", encoding="utf-8")) as f: text = ''.join(f.readlines()) if 'cuda' in text: found_cuda = True if 'sysroot_linux-64' in text: found_centos7 = True + + recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml") + if os.path.exists(recipe_yaml): + has_recipe_yaml = True + with(open(recipe_yaml, "r", encoding="utf-8")) as f: + text = ''.join(f.readlines()) + if 'cuda' in text: + found_cuda = True + if 'sysroot_linux-64' in text: + found_centos7 = True + cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml") if os.path.exists(cbc): with open(cbc, "r") as f: @@ -72,6 +89,11 @@ def build_all(recipes_dir, arch): print(f"Found c_stdlib_version for linux: {version=}") found_centos7 |= version == (2, 17) + if has_meta_yaml and has_recipe_yaml: + raise ValueError('Mixing meta.yaml and recipe.yaml recipes is not supported') + if not has_meta_yaml and not has_recipe_yaml: + raise ValueError('Neither a meta.yaml or a recipe.yaml recipes was found') + if found_cuda: print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1') if found_centos7: @@ -147,8 +169,13 @@ def build_all(recipes_dir, arch): if 'conda-forge' not in channel_urls: raise ValueError('conda-forge needs to be part of channel_sources') - print("Building {} with {}".format(','.join(folders), ','.join(channel_urls))) - build_folders(recipes_dir, folders, arch, channel_urls) + + if has_meta_yaml: + print("Building {} with {} using conda-build".format(','.join(folders), ','.join(channel_urls))) + build_folders(recipes_dir, folders, arch, channel_urls) + elif has_recipe_yaml: + print("Building {} with {} using rattler-build".format(','.join(folders), ','.join(channel_urls))) + build_folders_rattler_build(recipes_dir, platform, arch, channel_urls) def get_config(arch, channel_urls): @@ -207,9 +234,26 @@ def build_folders(recipes_dir, folders, arch, channel_urls): conda_build.api.build([recipe], config=get_config(arch, channel_urls)) +def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: list[str]): + # Define the arguments for rattler-build + args = [ + "rattler-build", + "build", + "--recipe-dir", recipes_dir, + "--target-platform", f"{platform}-{arch}", + ] + for channel_url in channel_urls: + # Local is automatically added by rattler-build so we just remove it. + if channel_url != "local": + args.extend(["-c", channel_url]) + + # Execute rattler-build. + subprocess.run(args, check=True) + + def check_recipes_in_correct_dir(root_dir, correct_dir): from pathlib import Path - for path in Path(root_dir).rglob('meta.yaml'): + for path in chain(Path(root_dir).rglob('meta.yaml'), Path(root_dir).rglob('recipe.yaml')): path = path.absolute().relative_to(root_dir) if path.parts[0] == 'build_artifacts': # ignore pkg_cache in build_artifacts From 79ac2cae602519b757cdcf61f0237210544f80b3 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 18 Jul 2024 11:21:28 +0200 Subject: [PATCH 03/14] show variants --- .ci_support/build_all.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index d0c2ec9944846..ee1feda0d9111 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -178,7 +178,7 @@ def build_all(recipes_dir, arch): build_folders_rattler_build(recipes_dir, platform, arch, channel_urls) -def get_config(arch, channel_urls): +def get_config(platform, arch, channel_urls): exclusive_config_files = [os.path.join(conda.base.context.context.root_prefix, 'conda_build_config.yaml')] script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -191,10 +191,14 @@ def get_config(arch, channel_urls): if os.path.exists(exclusive_config_file): exclusive_config_files.append(exclusive_config_file) - config = conda_build.api.Config( - arch=arch, exclusive_config_files=exclusive_config_files, - channel_urls=channel_urls, error_overlinking=True, - ) + config = conda_build.config.get_or_merge_config( + None, + exclusive_config_file=exclusive_config_files, + platform=platform, + arch=arch, + channel_urls=channel_urls + ) + return config @@ -206,8 +210,8 @@ def build_folders(recipes_dir, folders, arch, channel_urls): index = conda.core.index.get_index(channel_urls=channel_urls) conda_resolve = conda.resolve.Resolve(index) - config = get_config(arch, channel_urls) platform = get_host_platform() + config = get_config(platform, arch, channel_urls) worker = {'platform': platform, 'arch': arch, 'label': '{}-{}'.format(platform, arch)} @@ -231,10 +235,23 @@ def build_folders(recipes_dir, folders, arch, channel_urls): d[G.nodes[node]['meta'].meta_path] = 1 for recipe in d.keys(): - conda_build.api.build([recipe], config=get_config(arch, channel_urls)) + conda_build.api.build([recipe], config=get_config(platform, arch, channel_urls)) def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: list[str]): + config = get_config(platform, arch, channel_urls) + + # Get the combined variants from normal variant locations prior to running migrations + ( + combined_variant_spec, + _, + ) = conda_build.variants.get_package_combined_spec( + None, config=config + ) + + variants = yaml.dump(sorted(combined_variant_spec)) + print(f"-- VARIANTS:\n\n{variants}\n\n") + # Define the arguments for rattler-build args = [ "rattler-build", From 78b28592e5538a46e69624e945798d28eac7528d Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 18 Jul 2024 11:49:12 +0200 Subject: [PATCH 04/14] show variants --- .ci_support/build_all.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index ee1feda0d9111..27238b91214bf 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -4,6 +4,7 @@ import conda.core.index import conda.resolve import conda_build.api +import conda_build.variants import conda_index.api import networkx as nx from compute_build_graph import construct_graph @@ -178,7 +179,7 @@ def build_all(recipes_dir, arch): build_folders_rattler_build(recipes_dir, platform, arch, channel_urls) -def get_config(platform, arch, channel_urls): +def get_config(arch, channel_urls): exclusive_config_files = [os.path.join(conda.base.context.context.root_prefix, 'conda_build_config.yaml')] script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -191,14 +192,10 @@ def get_config(platform, arch, channel_urls): if os.path.exists(exclusive_config_file): exclusive_config_files.append(exclusive_config_file) - config = conda_build.config.get_or_merge_config( - None, - exclusive_config_file=exclusive_config_files, - platform=platform, - arch=arch, - channel_urls=channel_urls - ) - + config = conda_build.api.Config( + arch=arch, exclusive_config_files=exclusive_config_files, + channel_urls=channel_urls, error_overlinking=True, + ) return config @@ -210,8 +207,8 @@ def build_folders(recipes_dir, folders, arch, channel_urls): index = conda.core.index.get_index(channel_urls=channel_urls) conda_resolve = conda.resolve.Resolve(index) + config = get_config(arch, channel_urls) platform = get_host_platform() - config = get_config(platform, arch, channel_urls) worker = {'platform': platform, 'arch': arch, 'label': '{}-{}'.format(platform, arch)} @@ -235,21 +232,20 @@ def build_folders(recipes_dir, folders, arch, channel_urls): d[G.nodes[node]['meta'].meta_path] = 1 for recipe in d.keys(): - conda_build.api.build([recipe], config=get_config(platform, arch, channel_urls)) + conda_build.api.build([recipe], config=get_config(arch, channel_urls)) def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: list[str]): - config = get_config(platform, arch, channel_urls) - - # Get the combined variants from normal variant locations prior to running migrations - ( - combined_variant_spec, - _, - ) = conda_build.variants.get_package_combined_spec( - None, config=config - ) - - variants = yaml.dump(sorted(combined_variant_spec)) + config = get_config(arch, channel_urls) + + # Determine the locations for the variant config files. + specs = OrderedDict() + for f in config.exclusive_config_files: + print(f"Using variants from {f}") + specs[f] = conda_build.variants.parse_config_file(os.path.abspath(os.path.expanduser(os.path.expandvars(f))), config) + combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose) + + variants = yaml.dump(sorted(combined_spec)) print(f"-- VARIANTS:\n\n{variants}\n\n") # Define the arguments for rattler-build From d47b63711d471c67628fab9f80930ed176332d99 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 18 Jul 2024 13:59:56 +0200 Subject: [PATCH 05/14] fix: combine variant configs --- .ci_support/build_all.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 27238b91214bf..5065e502278f7 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,5 +1,6 @@ from itertools import chain import json +import tempfile import conda.base.context import conda.core.index import conda.resolve @@ -241,12 +242,11 @@ def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: # Determine the locations for the variant config files. specs = OrderedDict() for f in config.exclusive_config_files: - print(f"Using variants from {f}") specs[f] = conda_build.variants.parse_config_file(os.path.abspath(os.path.expanduser(os.path.expandvars(f))), config) - combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose) - variants = yaml.dump(sorted(combined_spec)) - print(f"-- VARIANTS:\n\n{variants}\n\n") + # Combine all the variant config files together + combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose) + variant_config = yaml.dump(combined_spec) # Define the arguments for rattler-build args = [ @@ -260,8 +260,14 @@ def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: if channel_url != "local": args.extend(["-c", channel_url]) - # Execute rattler-build. - subprocess.run(args, check=True) + # Construct a temporary file where we write the combined variant config. We can then pass that + # to rattler-build. + with tempfile.NamedTemporaryFile(delete=False) as fp: + fp.write(variant_config.encode('utf-8')) + fp.close() + + # Execute rattler-build. + subprocess.run(args + ["--variant-config", fp.name], check=True) def check_recipes_in_correct_dir(root_dir, correct_dir): From f51478cf5594ffc256d404576f4da90c1cd09ea7 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 18 Jul 2024 15:02:14 +0200 Subject: [PATCH 06/14] small ux changes --- .ci_support/build_all.py | 10 ++- .github/workflows/correct_directory.yml | 3 +- .../workflows/scripts/create_feedstocks.py | 2 +- README.md | 2 +- recipes/example-new-recipe/recipe.yaml | 90 +++++++++++++++++++ 5 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 recipes/example-new-recipe/recipe.yaml diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 5065e502278f7..01d5d6c6209d6 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,5 +1,6 @@ from itertools import chain import json +from shutil import rmtree import tempfile import conda.base.context import conda.core.index @@ -23,6 +24,9 @@ from yaml import BaseLoader, load +EXAMPLE_RECIPE_FOLDERS = ["example", "example-new-recipe"] + + def get_host_platform(): from sys import platform if platform == "linux" or platform == "linux2": @@ -238,6 +242,10 @@ def build_folders(recipes_dir, folders, arch, channel_urls): def build_folders_rattler_build(recipes_dir: str, platform, arch, channel_urls: list[str]): config = get_config(arch, channel_urls) + + # Remove the example recipes to ensure that they are not also build. + for example_recipe in EXAMPLE_RECIPE_FOLDERS: + rmtree(os.path.join(recipes_dir, example_recipe), ignore_errors=True) # Determine the locations for the variant config files. specs = OrderedDict() @@ -291,7 +299,7 @@ def read_mambabuild(recipes_dir): folders = os.listdir(recipes_dir) conda_build_tools = [] for folder in folders: - if folder == "example": + if folder in EXAMPLE_RECIPE_FOLDERS: continue cf = os.path.join(recipes_dir, folder, "conda-forge.yml") if os.path.exists(cf): diff --git a/.github/workflows/correct_directory.yml b/.github/workflows/correct_directory.yml index 3eef316c92a78..9db47cbbf2d97 100644 --- a/.github/workflows/correct_directory.yml +++ b/.github/workflows/correct_directory.yml @@ -31,6 +31,7 @@ jobs: '\nUnfortunately, the recipe was added directly in the `recipes` folder without its own subfolder.\n' + 'Please move the recipe file into a folder with the name of the package you want to submit.\n\n' + 'For example: if your recipe is currently under `recipes/.yaml`, ' + - 'it should be moved to `recipes//meta.yaml`.\n' + + 'it should be moved to `recipes//meta.yaml` ' + + 'or, if you are using the new yaml based recipe format, `recipes//recipe.yaml`\n' + 'Thanks!' }) diff --git a/.github/workflows/scripts/create_feedstocks.py b/.github/workflows/scripts/create_feedstocks.py index ed9b5ffd7cf9c..e73d5f83d8d7a 100755 --- a/.github/workflows/scripts/create_feedstocks.py +++ b/.github/workflows/scripts/create_feedstocks.py @@ -61,7 +61,7 @@ def list_recipes() -> Iterator[tuple[str, str]]: # to be helpful. # .DS_Store is created by macOS to store custom attributes of its # containing folder. - if recipe_dir.name in ["example", ".DS_Store"]: + if recipe_dir.name in ["example", "example-new-recipe", ".DS_Store"]: continue # Try to look for a conda-build recipe. diff --git a/README.md b/README.md index 62cbd1a891352..7f7bcedc9e397 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If the issue persists, support can be found [on Gitter](https://gitter.im/conda- ## Getting started 1. Fork this repository. -2. Make a new folder in `recipes` for your package. Look at the example recipe, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. +2. Make a new folder in `recipes` for your package. Look at the example recipes, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. 3. Open a pull request. Building of your package will be tested on Windows, Mac and Linux. 4. When your pull request is merged a new repository, called a feedstock, will be created in the github conda-forge organization, and build/upload of your package will automatically be triggered. Once complete, the package is available on conda-forge. diff --git a/recipes/example-new-recipe/recipe.yaml b/recipes/example-new-recipe/recipe.yaml new file mode 100644 index 0000000000000..a9929b6e17e4d --- /dev/null +++ b/recipes/example-new-recipe/recipe.yaml @@ -0,0 +1,90 @@ +# This example shows how to define a recipe using the new YAML based recipe format introduced by +# CEP 13. + +# For more information about this format see: https://prefix-dev.github.io/rattler-build/latest/reference/recipe_file/ + +# Note: there are many handy hints in comments in this example -- remove them when you've finalized your recipe + +# Define variables in this section that you can use in other parts. +context: + name: simplejson + version: "3.8.2" + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name }}-${{ version }}.tar.gz + # If getting the source from GitHub, remove the line above, + # uncomment the line below, and modify as needed. Use releases if available: + # url: https://github.com/simplejson/simplejson/releases/download/${{ version }}/simplejson-${{ version }}.tar.gz + # and otherwise fall back to archive: + # url: https://github.com/simplejson/simplejson/archive/v${{ version }}.tar.gz + sha256: d58439c548433adcda98e695be53e526ba940a4b9c44fb9a05d92cd495cdd47f + # sha256 is the preferred checksum -- you can get it for a file with: + # `openssl sha256 `. + # You may need the openssl package, available on conda-forge: + # `conda install openssl -c conda-forge`` + +build: + # Uncomment the following line if the package is pure Python and the recipe is exactly the same for all platforms. + # It is okay if the dependencies are not built for all platforms/versions, although selectors are still not allowed. + # See https://conda-forge.org/docs/maintainer/knowledge_base.html#noarch-python for more details. + # noarch: python + # If the installation is complex, or different between Unix and Windows, use separate build.bat and build.sh files instead of this key. + # By default, the package will be built for the Python versions supported by conda-forge and for all major OSs. + # Add the line "skip: True # [py<35]" (for example) to limit to Python 3.5 and newer, or "skip: True # [not win]" to limit to Windows. + # More info about selectors can be found in the conda-build docs: + # https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#preprocessing-selectors + script: python -m pip install . -vv + number: 0 + +requirements: + build: + # If your project compiles code (such as a C extension) then add the required compilers as separate entries here. + # Compilers are named 'c', 'cxx' and 'fortran'. + - ${{ compiler('c') }} + host: + - python + - pip + run: + - python + +tests: + # Some packages might need a `test/commands` key to check CLI. + - python: + # A list of modules that the test will try to import + imports: + - simplejson + - simplejson.tests + # Also run `pip check` to verify the integrity + pip_check: true + +about: + homepage: https://github.com/simplejson/simplejson + summary: 'Simple, fast, extensible JSON encoder/decoder for Python' + description: | + simplejson is a simple, fast, complete, correct and extensible + JSON encoder and decoder for Python 2.5+ and + Python 3.3+. It is pure Python code with no dependencies, but includes + an optional C extension for a serious speed boost. + # Remember to specify the license variants for BSD, Apache, GPL, and LGPL. + # Use the SPDX identifier, e.g: GPL-2.0-only instead of GNU General Public License version 2.0 + # See https://spdx.org/licenses/ + license: MIT + # It is required to include a license file in the package, + # (even if the license doesn't require it) using the license_file entry. + # Please also note that some projects have multiple license files which all need to be added using a valid yaml list. + # See https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#license-file + license_file: LICENSE.txt + # The doc_url and dev_url are optional. + documentation: https://simplejson.readthedocs.io/ + repository: https://github.com/simplejson/simplejson + +extra: + recipe-maintainers: + # GitHub IDs for maintainers of the recipe. + # Always check with the people listed below if they are OK becoming maintainers of the recipe. (There will be spam!) + - LisaSimpson + - LandoCalrissian From f4b175161c26e124dbaacb78d20e3f5eaf23351b Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Fri, 19 Jul 2024 14:44:23 +0200 Subject: [PATCH 07/14] update requirements --- .ci_support/requirements.txt | 1 + .github/workflows/scripts/create_feedstocks | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci_support/requirements.txt b/.ci_support/requirements.txt index 91d2f5605e9d5..e32c072f4b417 100644 --- a/.ci_support/requirements.txt +++ b/.ci_support/requirements.txt @@ -6,3 +6,4 @@ conda-forge-ci-setup=4.* conda-forge-pinning frozendict networkx=2.4 +rattler-build-conda-compat>=0.2.1 diff --git a/.github/workflows/scripts/create_feedstocks b/.github/workflows/scripts/create_feedstocks index b754ac1d35370..4872aeb70b02f 100755 --- a/.github/workflows/scripts/create_feedstocks +++ b/.github/workflows/scripts/create_feedstocks @@ -46,7 +46,7 @@ conda install --yes --quiet \ requests \ ruamel.yaml \ "pygithub>=2.1.1" \ - "rattler-build-conda-compat>=0.0.6,<0.1" + "rattler-build-conda-compat>=0.2.1" conda info mamba info From 16fc2a5ae8433a2d2c9b13b532c1253185fc6557 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 25 Jul 2024 13:21:37 +0200 Subject: [PATCH 08/14] Apply suggestions from code review Co-authored-by: jaimergp --- README.md | 2 +- recipes/example-new-recipe/recipe.yaml | 5 +++-- recipes/jolt-physics/recipe.yaml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7f7bcedc9e397..52c6d4511427e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If the issue persists, support can be found [on Gitter](https://gitter.im/conda- ## Getting started 1. Fork this repository. -2. Make a new folder in `recipes` for your package. Look at the example recipes, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. +2. Make a new folder in `recipes` for your package. Look at the example recipes, our [documentation](http://conda-forge.org/docs/maintainer/adding_pkgs.html#) and the [FAQ](https://github.com/conda-forge/staged-recipes#faq) for help. 3. Open a pull request. Building of your package will be tested on Windows, Mac and Linux. 4. When your pull request is merged a new repository, called a feedstock, will be created in the github conda-forge organization, and build/upload of your package will automatically be triggered. Once complete, the package is available on conda-forge. diff --git a/recipes/example-new-recipe/recipe.yaml b/recipes/example-new-recipe/recipe.yaml index a9929b6e17e4d..07c01696dc3e0 100644 --- a/recipes/example-new-recipe/recipe.yaml +++ b/recipes/example-new-recipe/recipe.yaml @@ -43,8 +43,9 @@ build: requirements: build: # If your project compiles code (such as a C extension) then add the required compilers as separate entries here. - # Compilers are named 'c', 'cxx' and 'fortran'. + # Compiler names include 'c', 'cxx' and 'fortran', among others. - ${{ compiler('c') }} + - ${{ stdlib('c') }} # If you need a compiler, add the `stdlib` equivalent too host: - python - pip @@ -78,7 +79,7 @@ about: # Please also note that some projects have multiple license files which all need to be added using a valid yaml list. # See https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#license-file license_file: LICENSE.txt - # The doc_url and dev_url are optional. + # The documentation and repository URLs are optional. documentation: https://simplejson.readthedocs.io/ repository: https://github.com/simplejson/simplejson diff --git a/recipes/jolt-physics/recipe.yaml b/recipes/jolt-physics/recipe.yaml index 1f437594c3ba1..24c0ee1d5c383 100644 --- a/recipes/jolt-physics/recipe.yaml +++ b/recipes/jolt-physics/recipe.yaml @@ -12,6 +12,7 @@ source: requirements: build: - ${{ compiler("cxx") }} + - ${{ stdlib("c") }} - cmake - ninja From 2b8c99bde0311b0565ed61fab76cdb694a106410 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 25 Jul 2024 13:32:53 +0200 Subject: [PATCH 09/14] small reviews --- recipes/example-new-recipe/recipe.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/recipes/example-new-recipe/recipe.yaml b/recipes/example-new-recipe/recipe.yaml index 07c01696dc3e0..42c72833fc9e4 100644 --- a/recipes/example-new-recipe/recipe.yaml +++ b/recipes/example-new-recipe/recipe.yaml @@ -3,6 +3,11 @@ # For more information about this format see: https://prefix-dev.github.io/rattler-build/latest/reference/recipe_file/ +# The main differences with the old format is that no preprocessing is required for the file to be valid YAML. +# This means: +# - No "selectors", use YAML if-then-else expressions instead (https://prefix-dev.github.io/rattler-build/latest/selectors/) +# - Jinja expressions are formatted with `${{}}` + # Note: there are many handy hints in comments in this example -- remove them when you've finalized your recipe # Define variables in this section that you can use in other parts. @@ -34,9 +39,12 @@ build: # noarch: python # If the installation is complex, or different between Unix and Windows, use separate build.bat and build.sh files instead of this key. # By default, the package will be built for the Python versions supported by conda-forge and for all major OSs. - # Add the line "skip: True # [py<35]" (for example) to limit to Python 3.5 and newer, or "skip: True # [not win]" to limit to Windows. - # More info about selectors can be found in the conda-build docs: - # https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#preprocessing-selectors + # Uncomment the following lines to limit to Python 3.5 and newer (for example) + # skip: + # - match(python, "<3.5") + # or the following to limit to Windows. + # skip: + # - win script: python -m pip install . -vv number: 0 From fe0efb91bb1d2a212db00e67dc8bdf5397c5bab7 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 25 Jul 2024 13:38:04 +0200 Subject: [PATCH 10/14] ran isort --- .ci_support/build_all.py | 229 +++++++++++++++++++++++---------------- 1 file changed, 138 insertions(+), 91 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 01d5d6c6209d6..0ad363adfa186 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,7 +1,13 @@ +import argparse +import os +import re +import subprocess +import sys +import tempfile +from collections import OrderedDict from itertools import chain -import json from shutil import rmtree -import tempfile + import conda.base.context import conda.core.index import conda.resolve @@ -9,14 +15,8 @@ import conda_build.variants import conda_index.api import networkx as nx -from compute_build_graph import construct_graph -import argparse -import re -import os -from collections import OrderedDict -import sys -import subprocess import yaml +from compute_build_graph import construct_graph try: from ruamel_yaml import BaseLoader, load @@ -29,6 +29,7 @@ def get_host_platform(): from sys import platform + if platform == "linux" or platform == "linux2": return "linux" elif platform == "darwin": @@ -43,14 +44,21 @@ def get_config_name(arch): def build_all(recipes_dir, arch): - folders = list(filter(lambda d: os.path.isdir(os.path.join(recipes_dir, d)), os.listdir(recipes_dir))) + folders = list( + filter( + lambda d: os.path.isdir(os.path.join(recipes_dir, d)), + os.listdir(recipes_dir), + ) + ) if not folders: print("Found no recipes to build") return platform = get_host_platform() script_dir = os.path.dirname(os.path.realpath(__file__)) - variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch))) + variant_config_file = os.path.join( + script_dir, "{}.yaml".format(get_config_name(arch)) + ) has_meta_yaml = False has_recipe_yaml = False @@ -61,47 +69,49 @@ def build_all(recipes_dir, arch): meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml") if os.path.exists(meta_yaml): has_meta_yaml = True - with(open(meta_yaml, "r", encoding="utf-8")) as f: - text = ''.join(f.readlines()) - if 'cuda' in text: + with open(meta_yaml, "r", encoding="utf-8") as f: + text = "".join(f.readlines()) + if "cuda" in text: found_cuda = True - if 'sysroot_linux-64' in text: + if "sysroot_linux-64" in text: found_centos7 = True recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml") if os.path.exists(recipe_yaml): has_recipe_yaml = True - with(open(recipe_yaml, "r", encoding="utf-8")) as f: - text = ''.join(f.readlines()) - if 'cuda' in text: + with open(recipe_yaml, "r", encoding="utf-8") as f: + text = "".join(f.readlines()) + if "cuda" in text: found_cuda = True - if 'sysroot_linux-64' in text: + if "sysroot_linux-64" in text: found_centos7 = True cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml") if os.path.exists(cbc): with open(cbc, "r") as f: lines = f.readlines() - pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(? Date: Tue, 30 Jul 2024 15:42:53 +0200 Subject: [PATCH 11/14] use atexit --- .ci_support/build_all.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 0ad363adfa186..0f1344e1abbdf 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -1,4 +1,5 @@ import argparse +import atexit import os import re import subprocess @@ -315,11 +316,11 @@ def build_folders_rattler_build( # Construct a temporary file where we write the combined variant config. We can then pass that # to rattler-build. with tempfile.NamedTemporaryFile(delete=False) as fp: - fp.write(variant_config.encode("utf-8")) - fp.close() + fp.write(variant_config.encode("utf-8")) + atexit.register(os.unlink, fp.name) - # Execute rattler-build. - subprocess.run(args + ["--variant-config", fp.name], check=True) + # Execute rattler-build. + subprocess.run(args + ["--variant-config", fp.name], check=True) def check_recipes_in_correct_dir(root_dir, correct_dir): From 5f012c0c4c1cd5efabcebedd7fdd83f9445a93a5 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 30 Jul 2024 15:52:36 +0200 Subject: [PATCH 12/14] add comments --- .ci_support/build_all.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index 0f1344e1abbdf..c66205f3996c1 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -299,6 +299,11 @@ def build_folders_rattler_build( combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose) variant_config = yaml.dump(combined_spec) + # The `variants.yaml` file next to a recipe.yaml file is automatically + # picked up by rattler-build so there is no need to merge that in here + # as well. See: + # https://prefix-dev.github.io/rattler-build/latest/variants/#automatic-variantsyaml-discovery + # Define the arguments for rattler-build args = [ "rattler-build", From 4c61424e7227fecd9f73e943c984a56025bc03db Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 30 Jul 2024 15:53:30 +0200 Subject: [PATCH 13/14] recipe.yaml example --- recipes/example-new-recipe/recipe.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/example-new-recipe/recipe.yaml b/recipes/example-new-recipe/recipe.yaml index 42c72833fc9e4..f8d70852b9282 100644 --- a/recipes/example-new-recipe/recipe.yaml +++ b/recipes/example-new-recipe/recipe.yaml @@ -61,7 +61,8 @@ requirements: - python tests: - # Some packages might need a `test/commands` key to check CLI. + # More information about different tests that can be added can be found here: + # https://prefix-dev.github.io/rattler-build/latest/testing/ - python: # A list of modules that the test will try to import imports: From 06c464395d3f1e414b6705f604f74cb12b34a83e Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 30 Jul 2024 15:56:09 +0200 Subject: [PATCH 14/14] Update .ci_support/build_all.py Co-authored-by: jaimergp --- .ci_support/build_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/build_all.py b/.ci_support/build_all.py index c66205f3996c1..aa62eb1b4b0e2 100644 --- a/.ci_support/build_all.py +++ b/.ci_support/build_all.py @@ -98,7 +98,7 @@ def build_all(recipes_dir, arch): # "not linux", "not unix", "osx" or "win"; this also removes trailing newlines lines = [pat.sub("", x) for x in lines] text = "\n".join(lines) - if platform == "linux" and ("c_stdlib_version" in text): + if platform == "linux" and "c_stdlib_version" in text: config = load(text, Loader=BaseLoader) if "c_stdlib_version" in config: for version in config["c_stdlib_version"]: