From 46ed115465f11464075eb085b527d067f014e731 Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 13:57:35 -0330 Subject: [PATCH 1/9] Extend python_extra_deps to depend on Python version --- .github/workflows/wheels.yml | 2 +- open_spiel/python/CMakeLists.txt | 36 +++++++++---------- open_spiel/scripts/ci_script.sh | 2 +- open_spiel/scripts/python_extra_deps.sh | 47 +++++++++++++++++++++++-- open_spiel/scripts/test_wheel.sh | 2 +- 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e90e6ad9e3..90417cfea2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -74,7 +74,7 @@ jobs: ${CI_PYBIN} -m pip --version ${CI_PYBIN} -m pip install --upgrade setuptools ${CI_PYBIN} -m pip install --upgrade -r requirements.txt -q - source ./open_spiel/scripts/python_extra_deps.sh + source ./open_spiel/scripts/python_extra_deps.sh ${CI_PYBIN} ${CI_PYBIN} -m pip install --no-cache-dir --upgrade $OPEN_SPIEL_PYTHON_JAX_DEPS ${CI_PYBIN} -m pip install --no-cache-dir --upgrade $OPEN_SPIEL_PYTHON_PYTORCH_DEPS ${CI_PYBIN} -m pip install --no-cache-dir --upgrade $OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS diff --git a/open_spiel/python/CMakeLists.txt b/open_spiel/python/CMakeLists.txt index ff1d58ad0d..85e5e65fea 100644 --- a/open_spiel/python/CMakeLists.txt +++ b/open_spiel/python/CMakeLists.txt @@ -265,14 +265,13 @@ set(PYTHON_TESTS ${PYTHON_TESTS} # Add Jax tests if it is enabled. if (OPEN_SPIEL_ENABLE_JAX) set (PYTHON_TESTS ${PYTHON_TESTS} - # Disable JAX tests temporarily - #jax/deep_cfr_jax_test.py - #jax/dqn_jax_test.py - #jax/nfsp_jax_test.py - #jax/opponent_shaping_jax_test.py - #jax/policy_gradient_jax_test.py - #algorithms/rnad/rnad_test.py - #mfg/algorithms/fictitious_play_test.py + jax/deep_cfr_jax_test.py + jax/dqn_jax_test.py + jax/nfsp_jax_test.py + jax/opponent_shaping_jax_test.py + jax/policy_gradient_jax_test.py + algorithms/rnad/rnad_test.py + mfg/algorithms/fictitious_play_test.py ) endif() @@ -367,15 +366,14 @@ endforeach(py_test_file) # Additional tests (running examples as tests) # We don't generate these automatically because we may want custom parameters. if (OPEN_SPIEL_ENABLE_JAX AND NOT OPEN_SPIEL_BUILDING_WHEEL) - # Disable JAX tests temporarily - #add_test(NAME python_examples_bridge_supervised_learning - # COMMAND ${Python3_EXECUTABLE} - # ${CMAKE_CURRENT_SOURCE_DIR}/examples/bridge_supervised_learning.py - # --iterations 10 - # --eval_every 5 - # --data_path ${CMAKE_CURRENT_SOURCE_DIR}/examples/data/bridge) - #set_property(TEST python_examples_bridge_supervised_learning - # PROPERTY ENVIRONMENT - # PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_CURRENT_SOURCE_DIR}/../..; - # TEST_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/../..) + add_test(NAME python_examples_bridge_supervised_learning + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/examples/bridge_supervised_learning.py + --iterations 10 + --eval_every 5 + --data_path ${CMAKE_CURRENT_SOURCE_DIR}/examples/data/bridge) + set_property(TEST python_examples_bridge_supervised_learning + PROPERTY ENVIRONMENT + PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_CURRENT_SOURCE_DIR}/../..; + TEST_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/../..) endif() diff --git a/open_spiel/scripts/ci_script.sh b/open_spiel/scripts/ci_script.sh index 0e8e537ff9..bb7c7b32ac 100755 --- a/open_spiel/scripts/ci_script.sh +++ b/open_spiel/scripts/ci_script.sh @@ -39,7 +39,7 @@ PYBIN=${PYBIN:-"python"} PYBIN=${PYBIN:-"python3"} PYBIN=`which $PYBIN` -source ./open_spiel/scripts/python_extra_deps.sh +source ./open_spiel/scripts/python_extra_deps.sh $PYBIN ${PYBIN} -m pip install --upgrade pip ${PYBIN} -m pip install --upgrade setuptools diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index 839e3b666b..bfe26d4415 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -24,7 +24,50 @@ # # To enable specific tests, please use the environment variables found in # scripts/global_variables.sh -export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.6 jaxlib==0.4.6 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.7 rlax==0.1.5 distrax==0.1.3" + +# This script depends on the Python version, which it gets from $PYBIN or +# $CI_PYBIN passed in as $1. If it's not defined, Python 3.9 is assumed. + +PY_VER="3.9" +if [ "$1" != "" ]; then + PY_VER=`$1 --version | awk '{print $2}'` + if [ "$PY_VER" = "" ]; then + PY_VER="3.9" + fi +fi + +verlte() { + stuff=`echo -e "$1\n$2" | sort -V | head -n1` + [ "$1" = "$stuff" ] +} + +verlt() { + [ "$1" = "$2" ] && return 1 || verlte $1 $2 +} + +# +# Python extra deps that work across all supported versions +# export OPEN_SPIEL_PYTHON_PYTORCH_DEPS="torch==1.13.1" -export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.10.1 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" + + +# +# Python-version dependent versions +# + +echo $PY_VER +verlt $PY_VER 3.11 +if [ "$?" == 0 ] +then + echo "Python < 3.11 detected" + export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.6 jaxlib==0.4.6 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.7 rlax==0.1.5 distrax==0.1.3" + export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" +else + echo "Python >= 3.11 detected" + export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.784 rlax==0.1.6 distrax==0.1.4" + export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" +fi + + + diff --git a/open_spiel/scripts/test_wheel.sh b/open_spiel/scripts/test_wheel.sh index 432415d2f2..eee2e40f68 100755 --- a/open_spiel/scripts/test_wheel.sh +++ b/open_spiel/scripts/test_wheel.sh @@ -54,7 +54,7 @@ $PYBIN -m pip install --upgrade -r $PROJDIR/requirements.txt -q if [[ "$MODE" = "full" ]]; then echo "Full mode. Installing Python extra deps libraries." - source $PROJDIR/open_spiel/scripts/python_extra_deps.sh + source $PROJDIR/open_spiel/scripts/python_extra_deps.sh $PYBIN $PYBIN -m pip install --upgrade $OPEN_SPIEL_PYTHON_JAX_DEPS $PYBIN -m pip install --upgrade $OPEN_SPIEL_PYTHON_PYTORCH_DEPS $PYBIN -m pip install --upgrade $OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS From 4125bedd84ba58000ec4a930240020f0bd204e5a Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 14:19:57 -0330 Subject: [PATCH 2/9] Trap the output of command rather than running it directly --- open_spiel/scripts/python_extra_deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index bfe26d4415..8275e5c439 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -56,8 +56,8 @@ export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5 # Python-version dependent versions # -echo $PY_VER -verlt $PY_VER 3.11 +echo "Set Python version: $PY_VER" +output=`verlt $PY_VER 3.12` if [ "$?" == 0 ] then echo "Python < 3.11 detected" From f76a94c21aa3974b3295f6864217dd147ce02187 Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 14:27:11 -0330 Subject: [PATCH 3/9] Move version threshold to 3.10 --- open_spiel/scripts/python_extra_deps.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index 8275e5c439..dea540e1bf 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -57,14 +57,14 @@ export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5 # echo "Set Python version: $PY_VER" -output=`verlt $PY_VER 3.12` +output=`verlt $PY_VER 3.10` if [ "$?" == 0 ] then - echo "Python < 3.11 detected" + echo "Python < 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.6 jaxlib==0.4.6 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.7 rlax==0.1.5 distrax==0.1.3" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" else - echo "Python >= 3.11 detected" + echo "Python >= 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.784 rlax==0.1.6 distrax==0.1.4" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" fi From c0651ccae75b260ee110582da604f321a4d2db59 Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 14:30:39 -0330 Subject: [PATCH 4/9] Change check to trap return value of function --- open_spiel/scripts/python_extra_deps.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index dea540e1bf..43371d669f 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -57,9 +57,7 @@ export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5 # echo "Set Python version: $PY_VER" -output=`verlt $PY_VER 3.10` -if [ "$?" == 0 ] -then +if verlt $PY_VER 3.10; then echo "Python < 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.6 jaxlib==0.4.6 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.7 rlax==0.1.5 distrax==0.1.3" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" From 0cf8362a2310f0c1b3cfd0004c9fab41b3502aa4 Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 14:34:27 -0330 Subject: [PATCH 5/9] Fix version of chex --- open_spiel/scripts/python_extra_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index 43371d669f..e8eb0640b3 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -63,7 +63,7 @@ if verlt $PY_VER 3.10; then export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" else echo "Python >= 3.10 detected" - export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.784 rlax==0.1.6 distrax==0.1.4" + export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.84 rlax==0.1.6 distrax==0.1.4" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" fi From 511040702a289dbb5780681658d303063bee5f9d Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 14:56:26 -0330 Subject: [PATCH 6/9] Upgrade version of networkx to match NumPy version --- open_spiel/scripts/python_extra_deps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index e8eb0640b3..c39d362397 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -49,7 +49,6 @@ verlt() { # Python extra deps that work across all supported versions # export OPEN_SPIEL_PYTHON_PYTORCH_DEPS="torch==1.13.1" -export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.10.1 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" # @@ -61,10 +60,12 @@ if verlt $PY_VER 3.10; then echo "Python < 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.6 jaxlib==0.4.6 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.7 rlax==0.1.5 distrax==0.1.3" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.23.5 tensorflow==2.13.1 tensorflow-probability==0.19.0 tensorflow_datasets==4.9.2 keras==2.13.1" + export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==2.4 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.10.1 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" else echo "Python >= 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.84 rlax==0.1.6 distrax==0.1.4" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" + export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==3.2 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.10.1 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" fi From cf947284157dac1ea18c6caa6eb450aaff7c177d Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 15:32:16 -0330 Subject: [PATCH 7/9] Try upgrading scipy version --- open_spiel/scripts/python_extra_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index c39d362397..b66fa53d35 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -65,7 +65,7 @@ else echo "Python >= 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.84 rlax==0.1.6 distrax==0.1.4" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" - export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==3.2 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.10.1 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" + export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==3.2 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.11.3 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" fi From 17bd38f99effc4ae6529d4891d24cdc9cdcebb14 Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 15:42:42 -0330 Subject: [PATCH 8/9] Add deadsnakes ppa to get newer version of Python 3.11 --- open_spiel/scripts/install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/open_spiel/scripts/install.sh b/open_spiel/scripts/install.sh index 8942700ec9..8ee38e1cd5 100755 --- a/open_spiel/scripts/install.sh +++ b/open_spiel/scripts/install.sh @@ -238,6 +238,9 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then PYTHON_PKGS="python3-dev python3-pip python3-setuptools python3-wheel python3-tk python3-venv" if [[ "$OS_PYTHON_VERSION" == "3.11" ]]; then # Need to special-case this until it's installed by default. + # https://vegastack.com/tutorials/how-to-install-python-3-11-on-ubuntu-22-04/ + echo "Adding Python 3.11 ppa repos" + sudo add-apt-repository ppa:deadsnakes/ppa PYTHON_PKGS="python3.11 python3.11-dev python3-pip python3-setuptools python3-wheel python3-tk python3.11-venv" fi EXT_DEPS="virtualenv clang cmake curl $PYTHON_PKGS" From 64559cbdc3f18f8ae2296899d20539f548b817dc Mon Sep 17 00:00:00 2001 From: Marc Lanctot Date: Thu, 9 Nov 2023 16:02:03 -0330 Subject: [PATCH 9/9] Upgrade version of cvxpy --- open_spiel/scripts/python_extra_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_spiel/scripts/python_extra_deps.sh b/open_spiel/scripts/python_extra_deps.sh index b66fa53d35..744922b736 100644 --- a/open_spiel/scripts/python_extra_deps.sh +++ b/open_spiel/scripts/python_extra_deps.sh @@ -65,7 +65,7 @@ else echo "Python >= 3.10 detected" export OPEN_SPIEL_PYTHON_JAX_DEPS="jax==0.4.20 jaxlib==0.4.20 dm-haiku==0.0.10 optax==0.1.7 chex==0.1.84 rlax==0.1.6 distrax==0.1.4" export OPEN_SPIEL_PYTHON_TENSORFLOW_DEPS="numpy==1.26.1 tensorflow==2.14.0 tensorflow-probability==0.22.1 tensorflow_datasets==4.9.2 keras==2.14.0" - export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==3.2 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.11.3 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.2.0 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" + export OPEN_SPIEL_PYTHON_MISC_DEPS="IPython==5.8.0 networkx==3.2 matplotlib==3.5.2 mock==4.0.2 nashpy==0.0.19 scipy==1.11.3 testresources==2.0.1 cvxopt==1.3.1 cvxpy==1.4.1 ecos==2.0.10 osqp==0.6.2.post5 clu==0.0.6 flax==0.5.3" fi