Prototype hermetic Python.

PiperOrigin-RevId: 631907904

.bazelrc

@@ -145,9 +145,6 @@ build --config=v2
 build:cuda --action_env=TF_CUDA_VERSION="12"
 build:cuda --action_env=TF_CUDNN_VERSION="8"
 build:cuda --repo_env=TF_CUDA_COMPUTE_CAPABILITIES="sm_60,sm_70,sm_75,compute_80"
-build:cuda --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc"
-
-build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1
 
 # Default startup
 startup --host_jvm_args=-Dbazel.DigestFunction=SHA256

BUILD

@@ -1,4 +1,5 @@
 load("@rules_license//rules:license.bzl", "license")
+# load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
 package(
  default_applicable_licenses = [":package_license"],
@@ -14,3 +15,13 @@ license(
 licenses(["notice"])
 
 exports_files(["LICENSE"])
+
+# compile_pip_requirements(
+# name = "requirements",
+# src = "requirements.in",
+# extra_args = [
+# "--allow-unsafe",
+# "--resolver=backtracking",
+# ],
+# requirements_txt = "requirements.txt",
+# )

RELEASE.md

@@ -11,6 +11,7 @@
 * Moved aggregation from https://github.com/google-parfait/federated-compute
  to TFF to consolidate the federated language and remove circular
  dependencies.
+* Updated to use a hermetic Python environment.
 
 ## Breaking Changes
 

WORKSPACE

@@ -84,7 +84,6 @@ git_repository(
  # TODO: b/263201501 - Make DTensor C++ API public and remove this patch.
  "//third_party/tensorflow:dtensor_internal_visibility.patch",
  "//third_party/tensorflow:internal_visibility.patch",
- "//third_party/tensorflow:python_toolchain.patch",
  "//third_party/tensorflow:tf2xla_visibility.patch",
  ],
  remote = "https://github.com/tensorflow/tensorflow.git",
@@ -141,6 +140,39 @@ git_repository(
  commit = "e4635f223e7d36dfbea3b722a4ca4807a7e882e2",
 )
 
+#
+# Python toolchain.
+#
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+ name = "python",
+ ignore_root_user_error = True,
+ python_version = "3.11",
+)
+
+#
+# Python dependencies.
+#
+
+load("@python//:defs.bzl", "interpreter")
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+ name = "pypi",
+ python_interpreter_target = interpreter,
+ requirements_lock = "//:requirements.txt",
+)
+
+load("@pypi//:requirements.bzl", "install_deps")
+
+install_deps()
+
 #
 # Transitive dependencies, grouped by direct dependency.
 #

examples/datasets/BUILD

@@ -10,5 +10,8 @@ licenses(["notice"])
 py_binary(
  name = "load_flair",
  srcs = ["load_flair.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ ],
 )

examples/learning/federated_program/vizier/BUILD

@@ -3,13 +3,19 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 py_library(
  name = "data_sources",
  srcs = ["data_sources.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_library(
  name = "learning_process",
  srcs = ["learning_process.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_binary(
@@ -23,12 +29,15 @@ py_binary(
  ":learning_process",
  ":vizier_service",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_google_vizier//:pkg",
  ],
 )
 
 py_library(
  name = "vizier_service",
  srcs = ["vizier_service.py"],
+ deps = ["@pypi_tensorflow//:pkg"],
 )
 
 py_test(
@@ -37,5 +46,8 @@ py_test(
  data = [
  ":study_spec.textproto",
  ],
- deps = [":vizier_service"],
+ deps = [
+ ":vizier_service",
+ "@pypi_absl_py//:pkg",
+ ],
 )

examples/personalization/BUILD

@@ -10,7 +10,10 @@ licenses(["notice"])
 py_library(
  name = "p13n_utils",
  srcs = ["p13n_utils.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_test(
@@ -19,6 +22,8 @@ py_test(
  deps = [
  ":p13n_utils",
  "//tensorflow_federated",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
@@ -28,5 +33,8 @@ py_binary(
  deps = [
  ":p13n_utils",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )

examples/program/BUILD

@@ -10,7 +10,11 @@ licenses(["notice"])
 py_library(
  name = "computations",
  srcs = ["computations.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_library(
@@ -26,6 +30,8 @@ py_binary(
  ":computations",
  ":program_logic",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
@@ -36,5 +42,8 @@ py_test(
  ":computations",
  ":program_logic",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )

examples/simple_fedavg/BUILD

@@ -14,13 +14,21 @@ py_binary(
  deps = [
  ":simple_fedavg_tff",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
 py_library(
  name = "simple_fedavg_tf",
  srcs = ["simple_fedavg_tf.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_attrs//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_library(
@@ -29,6 +37,7 @@ py_library(
  deps = [
  ":simple_fedavg_tf",
  "//tensorflow_federated",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
@@ -42,5 +51,9 @@ py_cpu_gpu_test(
  ":simple_fedavg_tf",
  ":simple_fedavg_tff",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_attrs//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )

examples/stateful_clients/BUILD

@@ -14,13 +14,20 @@ py_binary(
  ":stateful_fedavg_tf",
  ":stateful_fedavg_tff",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
 py_library(
  name = "stateful_fedavg_tf",
  srcs = ["stateful_fedavg_tf.py"],
- deps = ["//tensorflow_federated"],
+ deps = [
+ "//tensorflow_federated",
+ "@pypi_attrs//:pkg",
+ "@pypi_tensorflow//:pkg",
+ ],
 )
 
 py_library(
@@ -29,6 +36,8 @@ py_library(
  deps = [
  ":stateful_fedavg_tf",
  "//tensorflow_federated",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )
 
@@ -40,5 +49,8 @@ py_test(
  ":stateful_fedavg_tf",
  ":stateful_fedavg_tff",
  "//tensorflow_federated",
+ "@pypi_absl_py//:pkg",
+ "@pypi_numpy//:pkg",
+ "@pypi_tensorflow//:pkg",
  ],
 )

requirements.in

@@ -0,0 +1,53 @@
+# Requirements for the TensorFlow Federated development environment.
+#
+# If you add a *new* dependency and it is required by the TensorFlow Federated
+# package, also add the dependency to
+# https://github.com/tensorflow/federated/blob/main/tensorflow_federated/tools/development/setup.py.
+#
+# If you update the version of an *existing* dependency and it is required by
+# the TensorFlow Federated package, also update the version of the dependency in
+# https://github.com/tensorflow/federated/blob/main/tensorflow_federated/tools/development/setup.py.
+#
+# * For packages that have a stable release, we use a version that is
+# compatible with that release (e.g. `~=x.y`). See
+# https://peps.python.org/pep-0440/#compatible-release for more information.
+# * For packages that do not have a stable release, we use a version that
+# matches a release that has been tested (e.g. `==x.y.z`). See
+# https://peps.python.org/pep-0440/#version-matching for more information.
+#
+# Note: There is bug in `pip` when multiple packages use the compatible release
+# operator `~=` to specify a version and one of those versions ends in `0`. See
+# https://github.com/pypa/pip/issues/9613 for more information. In this case,
+# use the equivalent clause `>=x.0,==x.*` instead of `~=x.0`.
+#
+# This assumes that the packages follows Semantic Versioning, see
+# https://semver.org/. If a package follows a different versioning scheme or
+# requires unique handling, we use a different version specifier and comment the
+# versioning scheme or reasoning.
+
+absl-py>=1.0,==1.*
+attrs~=23.1
+cachetools~=5.3
+dm-tree==0.1.8
+dp-accounting==0.4.3
+#farmhashpy==0.4.0
+google-vizier==0.1.11
+grpcio~=1.46
+jaxlib==0.4.14
+jax==0.4.14
+numpy~=1.25
+portpicker~=1.6
+scipy~=1.9.3
+semantic-version~=2.6
+tensorflow-model-optimization==0.7.5
+tensorflow-privacy==0.9.0
+tqdm~=4.64
+typing-extensions>=4.5.0,==4.5.*
+
+# The version of this dependency should match the version in
+# https://github.com/tensorflow/federated/blob/main/WORKSPACE.
+tensorflow>=2.14.0,==2.14.*
+
+# TODO: b/315515548 - Required because current dependencies are pulling in later
+# versions which are causing opaque failures.
+googleapis-common-protos==1.61.0