diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cbe6824..e406ccc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,97 +38,181 @@ jobs: cmake .. -DBUILD_SKETCHES=On make -j 4 - name: Test parser - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_parser + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_parser - name: Test DG integrator - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./dg_integ_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./dg_integ_mms - name: Test linalg_utils - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_linalg_utils + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_linalg_utils - name: Test nonlinear integrator gradient - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./nonlinear_integ_grad + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./nonlinear_integ_grad - name: Test nonlinear interface integrator gradient - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./interfaceinteg_grad + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./interfaceinteg_grad - name: Test ROM NonlinearForm - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_rom_nonlinearform + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_rom_nonlinearform - name: Test ROM InterfaceForm - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_rom_interfaceform + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_rom_interfaceform - name: Test Poisson DD solver - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./poisson_dd_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./poisson_dd_mms - name: Test Stokes DD solver - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./stokes_dd_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./stokes_dd_mms - name: Test SteadyNS DD solver - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./steady_ns_dd_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./steady_ns_dd_mms - name: Test UnsteadyNS DD solver - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./unsteady_ns_dd_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./unsteady_ns_dd_mms - name: Test LinearElastic DD solver - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./linelast_dd_mms + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./linelast_dd_mms - name: Test parametrized problem - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_param_prob + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_param_prob - name: Test workflow - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_workflow + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_workflow - name: Test hdf5 utils - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_hdf5 + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_hdf5 - name: Test component topology handler - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_topol + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_topol - name: Test block matrix preconditioner - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_block_smoother + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_block_smoother - name: Test multi-component domain decomposition - run: | - cd ${GITHUB_WORKSPACE}/build/test/gmsh - echo 'if [ -f "./multi_comp_dd_mms" ]; then ./multi_comp_dd_mms; else echo "multi_comp_dd_mms does not exist. Passing the test."; fi' >> command.sh - sh command.sh + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test/gmsh + echo 'if [ -f "./multi_comp_dd_mms" ]; then ./multi_comp_dd_mms; else echo "multi_comp_dd_mms does not exist. Passing the test."; fi' >> command.sh + sh command.sh - name: Test Stokes multi-component domain decomposition - run: | - cd ${GITHUB_WORKSPACE}/build/test/gmsh - echo 'if [ -f "./stokes_multi_comp_dd_mms" ]; then ./stokes_multi_comp_dd_mms; else echo "stokes_multi_comp_dd_mms does not exist. Passing the test."; fi' >> command.sh - sh command.sh + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test/gmsh + echo 'if [ -f "./stokes_multi_comp_dd_mms" ]; then ./stokes_multi_comp_dd_mms; else echo "stokes_multi_comp_dd_mms does not exist. Passing the test."; fi' >> command.sh + sh command.sh - name: Test multi-component workflow - run: | - cd ${GITHUB_WORKSPACE}/build/test/gmsh - echo 'if [ -f "./test_multi_comp_workflow" ]; then ./test_multi_comp_workflow; else echo "test_multi_comp_workflow does not exist. Passing the test."; fi' >> command.sh - sh command.sh + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test/gmsh + echo 'if [ -f "./test_multi_comp_workflow" ]; then ./test_multi_comp_workflow; else echo "test_multi_comp_workflow does not exist. Passing the test."; fi' >> command.sh + sh command.sh - name: Test parallel POD/EQP - run: | - cd ${GITHUB_WORKSPACE}/build/test - ./test_ns_parallel --gtest_filter=NSTensor.Sampling - mpirun -n 3 --oversubscribe ./test_ns_parallel --gtest_filter=NSTensor.Train - ./test_ns_parallel --gtest_filter=NSTensor.Build_SingleRun - ./test_ns_parallel --gtest_filter=NSEQP.Sampling - mpirun -n 3 --oversubscribe ./test_ns_parallel --gtest_filter=NSEQP.Train - ./test_ns_parallel --gtest_filter=NSEQP.Build_SingleRun + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 3 + command: | + cd ${GITHUB_WORKSPACE}/build/test + ./test_ns_parallel --gtest_filter=NSTensor.Sampling + mpirun -n 3 --oversubscribe ./test_ns_parallel --gtest_filter=NSTensor.Train + ./test_ns_parallel --gtest_filter=NSTensor.Build_SingleRun + ./test_ns_parallel --gtest_filter=NSEQP.Sampling + mpirun -n 3 --oversubscribe ./test_ns_parallel --gtest_filter=NSEQP.Train + ./test_ns_parallel --gtest_filter=NSEQP.Build_SingleRun # - name: Upload the compiled artifacts # uses: actions/upload-artifact@master # with: diff --git a/dependencies/FindMETIS.cmake b/dependencies/FindMETIS.cmake deleted file mode 100644 index dfae35a3..00000000 --- a/dependencies/FindMETIS.cmake +++ /dev/null @@ -1,79 +0,0 @@ -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -#[=======================================================================[.rst: -FindMETIS -------- -Michael Hirsch, Ph.D. - -Finds the METIS library. -NOTE: If libparmetis used, libmetis must also be linked. - -Imported Targets -^^^^^^^^^^^^^^^^ - -METIS::METIS - -Result Variables -^^^^^^^^^^^^^^^^ - -METIS_LIBRARIES - libraries to be linked - -METIS_INCLUDE_DIRS - dirs to be included - -#]=======================================================================] - - -if("parallel" IN_LIST METIS_FIND_COMPONENTS) - find_library(PARMETIS_LIBRARY - NAMES parmetis - PATH_SUFFIXES METIS libmetis - DOC "ParMETIS library" - ) - if(PARMETIS_LIBRARY) - set(METIS_parallel_FOUND true) - endif() -endif() - -find_library(METIS_LIBRARY - NAMES metis - PATH_SUFFIXES METIS libmetis - DOC "METIS library" - ) - -if(parallel IN_LIST METIS_FIND_COMPONENTS) - set(metis_inc parmetis.h) -else() - set(metis_inc metis.h) -endif() - -find_path(METIS_INCLUDE_DIR -NAMES ${metis_inc} -PATH_SUFFIXES METIS openmpi-x86_64 mpich-x86_64 -DOC "METIS include directory" -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(METIS -REQUIRED_VARS METIS_LIBRARY METIS_INCLUDE_DIR -HANDLE_COMPONENTS -) - -if(METIS_FOUND) - -set(METIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY}) -set(METIS_INCLUDE_DIRS ${METIS_INCLUDE_DIR}) - -message(VERBOSE "METIS libraries: ${METIS_LIBRARIES} -METIS include directories: ${METIS_INCLUDE_DIRS}") - -if(NOT TARGET METIS::METIS) - add_library(METIS::METIS INTERFACE IMPORTED) - set_property(TARGET METIS::METIS PROPERTY INTERFACE_LINK_LIBRARIES "${METIS_LIBRARIES}") - set_property(TARGET METIS::METIS PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${METIS_INCLUDE_DIR}") -endif() -endif(METIS_FOUND) - -mark_as_advanced(METIS_INCLUDE_DIR METIS_LIBRARY PARMETIS_LIBRARY) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7ee5033b..2ecee0ed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ WORKDIR $LIB_DIR RUN sudo git clone https://github.com/scivision/mumps.git WORKDIR ./mumps RUN sudo git checkout v5.6.2.1 -RUN sudo wget -O cmake/FindMETIS.cmake https://github.com/LLNL/scaleupROM/raw/install/dependencies/FindMETIS.cmake +RUN sudo wget -O cmake/FindMETIS.cmake "https://raw.githubusercontent.com/LLNL/scaleupROM/main/install-helper/mumps/FindMETIS.cmake" # RUN sudo sed -i 's/if(parallel IN_LIST METIS_FIND_COMPONENTS)/if("parallel" IN_LIST METIS_FIND_COMPONENTS)/g' cmake/FindMETIS.cmake # RUN sudo sed -i 's/option(CMAKE_TLS_VERIFY "Verify TLS certificates" ON)/option(CMAKE_TLS_VERIFY "Verify TLS certificates" OFF)/g' options.cmake @@ -52,9 +52,7 @@ ENV YAML_DIR=$LIB_DIR/yaml-cpp # install libROM for scaleupROM WORKDIR $LIB_DIR RUN sudo git clone https://github.com/LLNL/libROM.git -WORKDIR ./libROM -RUN sudo git pull && sudo git checkout mpi-io -WORKDIR ./build +WORKDIR ./libROM/build # libROM is using the MFEM without MUMPS right now. RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=OFF -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB} RUN sudo make -j 16 diff --git a/include/rom_interfaceform.hpp b/include/rom_interfaceform.hpp index cf0287e0..e091fbd7 100644 --- a/include/rom_interfaceform.hpp +++ b/include/rom_interfaceform.hpp @@ -9,6 +9,7 @@ #include "rom_handler.hpp" #include "hdf5_utils.hpp" #include "hyperreduction_integ.hpp" +#include "linalg/NNLS.h" namespace mfem { @@ -55,6 +56,9 @@ class ROMInterfaceForm : public InterfaceForm /// @brief Flag for precomputing necessary coefficients for fast computation. bool precompute = false; + /// @brief Energy norm criterion for NNLS. + CAROM::NNLS_termination nnls_criterion = CAROM::NNLS_termination::L2; + public: ROMInterfaceForm(Array &meshes_, Array &fes_, Array &comp_fes_, TopologyHandler *topol_); diff --git a/include/rom_nonlinearform.hpp b/include/rom_nonlinearform.hpp index 44dc2421..0ec1c360 100644 --- a/include/rom_nonlinearform.hpp +++ b/include/rom_nonlinearform.hpp @@ -44,6 +44,9 @@ class ROMNonlinearForm : public NonlinearForm /// @brief Flag for precomputing necessary coefficients for fast computation. bool precompute = false; + /// @brief Energy norm criterion for NNLS. + CAROM::NNLS_termination nnls_criterion = CAROM::NNLS_termination::L2; + /* Flag for being reference ROMNonlinearForm. If not reference, all EQPElement arrays are view arrays, not owning them. diff --git a/sketches/librom_qr.cpp b/sketches/librom_qr.cpp index d9b2d2df..93e42c2a 100644 --- a/sketches/librom_qr.cpp +++ b/sketches/librom_qr.cpp @@ -59,7 +59,7 @@ int main(int argc, char *argv[]) } printf("\n"); - CAROM::Matrix* Q = rand_proj->qr_factorize(); + CAROM::Matrix* Q = rand_proj->qr_factorize().get(); printf("Q (%d x %d) in RandomizedSVD::computeSVD.\n", Q->numRows(), Q->numColumns()); for (int i = 0; i < Q->numRows(); i++) { for (int j = 0; j < Q->numColumns(); j++) { @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) printf("\n"); rand_proj = snapshot_matrix->mult(rand_mat); - Q = rand_proj->qr_factorize(); + Q = rand_proj->qr_factorize().get(); printf("After perturbation, 2nd/3rd columns of Q totally changed.\n"); for (int i = 0; i < Q->numRows(); i++) { for (int j = 0; j < Q->numColumns(); j++) { diff --git a/src/rom_interfaceform.cpp b/src/rom_interfaceform.cpp index 1d83ea49..425bbfd8 100644 --- a/src/rom_interfaceform.cpp +++ b/src/rom_interfaceform.cpp @@ -5,7 +5,6 @@ #include "rom_interfaceform.hpp" #include "etc.hpp" #include "utils/mpi_utils.h" // this is from libROM/utils. -#include "linalg/NNLS.h" using namespace std; @@ -27,6 +26,12 @@ ROMInterfaceForm::ROMInterfaceForm( // block_offsets should be updated according to the number of basis vectors. block_offsets = -1; + + std::string nnls_str = config.GetOption("model_reduction/eqp/criterion", "l2"); + if (nnls_str == "l2") nnls_criterion = CAROM::NNLS_termination::L2; + else if (nnls_str == "linf") nnls_criterion = CAROM::NNLS_termination::LINF; + else + mfem_error("ROMNonlinearForm: unknown NNLS criterion!\n"); } ROMInterfaceForm::~ROMInterfaceForm() @@ -546,7 +551,8 @@ void ROMInterfaceForm::TrainEQPForIntegrator( CAROM::Vector eqpSol(Gt.numRows(), true); int nnz = 0; { - CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); + CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2, 1.0e-4, 1.0e-14, + 100000, 100000, nnls_criterion); CAROM::Vector rhs_ub(rhs_Gw); CAROM::Vector rhs_lb(rhs_Gw); diff --git a/src/rom_nonlinearform.cpp b/src/rom_nonlinearform.cpp index 2a89a4ac..61651773 100644 --- a/src/rom_nonlinearform.cpp +++ b/src/rom_nonlinearform.cpp @@ -16,6 +16,12 @@ ROMNonlinearForm::ROMNonlinearForm(const int num_basis, FiniteElementSpace *f, c { height = width = num_basis; + std::string nnls_str = config.GetOption("model_reduction/eqp/criterion", "l2"); + if (nnls_str == "l2") nnls_criterion = CAROM::NNLS_termination::L2; + else if (nnls_str == "linf") nnls_criterion = CAROM::NNLS_termination::LINF; + else + mfem_error("ROMNonlinearForm: unknown NNLS criterion!\n"); + for (int k = 0; k < Nt; k++) jac_timers[k] = new StopWatch; } @@ -769,7 +775,8 @@ void ROMNonlinearForm::TrainEQPForIntegrator( CAROM::Vector eqpSol(Gt.numRows(), true); int nnz = 0; { - CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); + CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2, 1.0e-4, 1.0e-14, + 100000, 100000, nnls_criterion); CAROM::Vector rhs_ub(rhs_Gw); CAROM::Vector rhs_lb(rhs_Gw);