Merge pull request #1274 from JCSDA-internal/release/1.1.0

Release/1.1.0
JCSDA · Jun 11, 2021 · 2af9b91 · 2af9b91
2 parents b1804f5 + be17f9d
commit 2af9b91
Show file tree

Hide file tree

Showing 1,123 changed files with 101,674 additions and 13,106 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 *.nc filter=lfs diff=lfs merge=lfs -text
 *.nc4 filter=lfs diff=lfs merge=lfs -text
+*.dat filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/ISSUE_TEMPLATE/basic-issue.md b/.github/ISSUE_TEMPLATE/basic-issue.md
@@ -0,0 +1,10 @@
+---
+name: Regular issue
+about: This is a blank issue.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+
diff --git a/.github/ISSUE_TEMPLATE/missing-docs.md b/.github/ISSUE_TEMPLATE/missing-docs.md
@@ -0,0 +1,30 @@
+---
+name: 'Missing Documentation'
+about: 'Use this issue to note missing documentation on ReadTheDocs and/or Doxygen'
+title: "[Documentation]: Give a descriptive title here"
+labels: documentation
+assignees: huishao-r, rhoneyager, mmiesch
+
+---
+
+## Description
+
+> Edit the text of this pull request to describe what is missing. You should include the file name or the class name.
+
+
+
+## ZenHub organization
+
+Please make sure that you have properly set:
+- Labels: documentation
+- Milestone: OBS - current month
+- Estimate: usually 0.5 - 1
+- Epics: Select a relevant epic. Here are a few hints.
+ - JEDI1.10: For **generic** UFO components and bias correction. Generic means a core UFO component, such as a base class
+ or a component of JEDI that more than one agency will use.
+ - OBS3.1.2: For **specific** ObsOperators, Filters, and ObsFunctions.
+- Assignees: Generally, the person who created the code is responsible for writing the documentation.
+ The "git blame" feature may be quite helpful in determining this. @huishao-r, @mmiesch, and @rhoneyager are
+ automatically added to help track these issues.
+
+
diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt
@@ -10,7 +10,7 @@
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-project( ufo-bundle LANGUAGES C CXX Fortran )
+project( ufo-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran )
 
 find_package(ecbuild)
 set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}")
@@ -19,20 +19,45 @@ include( ecbuild_bundle )
 set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 ecbuild_bundle_initialize()
-ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" )
+ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" )
+include( jedicmake/cmake/Functions/git_functions.cmake )
 
 option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON") #Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-ecbuild_bundle( PROJECT eckit GIT "https://github.com/JCSDA/eckit.git" )
-ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA/fckit.git" )
-ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA/atlas.git" )
+option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") #Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
+option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") #Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
 
-ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" )
-ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA/saber.git" )
-#ecbuild_bundle( PROJECT gsw GIT "https://github.com/JCSDA/GSW-Fortran.git" BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" )
-ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA/ioda.git" )
-ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA/ufo.git" )
+ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
+ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
+ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 )
 
+ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" )
+ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA-internal/saber.git" )
+#ecbuild_bundle( PROJECT gsw GIT "https://github.com/JCSDA-internal/GSW-Fortran.git" )
+ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA-internal/crtm.git" )
+ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA-internal/ioda.git" )
+ecbuild_bundle( PROJECT rttov GIT "https://github.com/JCSDA-internal/rttov.git" )
+ecbuild_bundle( PROJECT ropp-ufo GIT "https://github.com/JCSDA-internal/ropp-test.git" )
+ecbuild_bundle( PROJECT geos-aero GIT "https://github.com/JCSDA-internal/geos-aero.git" )
+ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" )
+
+# Don't need ioda-data in CI because we are only testing ufo
+
+# If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
+# If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
+# IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
+#find_branch_name(REPO_DIR_NAME ioda)
+# When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
+# in a local directory, ioda-data repo will not be cloned
+
+#if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
+# ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" )
+#endif()
+
+# same procedure for ufo-data
+find_branch_name(REPO_DIR_NAME ufo)
+if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED ${GIT_TAG_FUNC} )
+ ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" )
+endif()
 
 # Build Doxygen documentation
 option(BUILD_UFO_BUNDLE_DOC "Build documentation" OFF)
@@ -44,4 +69,3 @@ ecbuild_bundle_finalize()
 
 include(cmake/cdash-integration.cmake)
 include(CTest)
-
diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml
@@ -1,6 +1,7 @@
 version: 0.2
 
 env:
+ shell: bash
  parameter-store:
  GIT_USER: "/CodeBuild/Git_USER"
  GIT_PASS: "/CodeBuild/Git_PASS"
@@ -45,7 +46,7 @@ phases:
  - git lfs install # creates .gitconfig
 
  # Set crtm branch name
- - export CRTM_JEDI_BRANCH="v2.3-jedi"
+ - export CRTM_JEDI_BRANCH="release/crtm_jedi"
 
  - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ];
  then export CODEBUILD_GIT_BRANCH_FORK="release-stable";
@@ -55,29 +56,26 @@ phases:
  echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}";
  fi
 
+ # Upload branch name and commit sha as CodeBuild artifact to S3
+ - mkdir -p /jcsda/artifacts
+ - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
+ - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
+
  ## public repos
  # ufo
- - ./clone.sh $GIT_USER $GIT_PASS jcsda/ufo $CODEBUILD_GIT_BRANCH ufo /jcsda/ufo-bundle develop
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ufo $CODEBUILD_GIT_BRANCH ufo /jcsda/ufo-bundle develop
 
  # crtm
- - ./clone.sh $GIT_USER $GIT_PASS jcsda/crtm $CODEBUILD_GIT_BRANCH_CRTM crtm /jcsda/ufo-bundle $CRTM_JEDI_BRANCH
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/crtm $CODEBUILD_GIT_BRANCH_CRTM crtm /jcsda/ufo-bundle $CRTM_JEDI_BRANCH
 
  # ioda
- - ./clone.sh $GIT_USER $GIT_PASS jcsda/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ufo-bundle develop
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ufo-bundle develop
 
  # saber
- - ./clone.sh $GIT_USER $GIT_PASS jcsda/saber $CODEBUILD_GIT_BRANCH saber /jcsda/ufo-bundle develop
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/saber $CODEBUILD_GIT_BRANCH saber /jcsda/ufo-bundle develop
 
  # oops
- - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ufo-bundle develop
-
-
- ## ecmwf repos
- # atlas
- - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/ufo-bundle release-stable
-
- #fckit
- - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/ufo-bundle release-stable
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ufo-bundle develop
 
 
  ## internal repos
@@ -99,33 +97,33 @@ phases:
  #gsw
  - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/GSW-Fortran $CODEBUILD_GIT_BRANCH gsw /jcsda/ufo-bundle develop
 
+ # ufo-test
+ - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/ufo-data $CODEBUILD_GIT_BRANCH ufo-data /jcsda/ufo-bundle develop
+
  # move CMakeLists.txt
  - cp CMakeLists.txt /jcsda/ufo-bundle
  - cp -r cmake /jcsda/ufo-bundle/
- - cd /jcsda/ufo-bundle
+ - cp /jcsda/ufo-bundle/cmake/CTestConfig.cmake /jcsda/ufo-bundle
 
- - ls
  - cd /jcsda/ufo-bundle
  - ls
 
- - sed -i 's/BRANCH develop UPDATE//' CMakeLists.txt
- - sed -i 's/BRANCH develop //' CMakeLists.txt
- - sed -i 's/BRANCH release-stable UPDATE//' CMakeLists.txt
- - sed -i -e '/ecbuild_bundle( PROJECT ufo/i\
- ecbuild_bundle( PROJECT ropp-ufo GIT "https://github.com/JCSDA/ropp-test.git" )' CMakeLists.txt
- - sed -i -e '/ecbuild_bundle( PROJECT ufo/i\
- ecbuild_bundle( PROJECT geos-aero GIT "https://github.com/JCSDA/geos-aero.git" )' CMakeLists.txt
-
  build:
+ on-failure: CONTINUE
  commands:
  - echo Executing build phase
  - echo $CODEBUILD_BUILD_SUCCEEDING
  - export BUILD_STATUS="0"
  - echo $BUILD_STATUS
 
+ # configure and build
  - cd /build_container
- - ecbuild -DBUILD_RTTOV=1 /jcsda/ufo-bundle/
+ - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE /jcsda/ufo-bundle
  - cd ufo
+ - cp ../DartConfiguration.tcl .
+ - sed -i 's/ufo-bundle/ufo-bundle\/ufo/' DartConfiguration.tcl
+ - sed -i 's/build_container/build_container\/ufo/' DartConfiguration.tcl
+ - cat DartConfiguration.tcl
  - make -j4
 
  - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
@@ -134,16 +132,32 @@ phases:
  fi
  - echo $BUILD_STATUS
 
+ # run ctest
+ - cd /build_container/ufo
+ - ctest -C RelWithDebInfo -D ExperimentalTest
+
+ finally:
  - cd /build_container/ufo
- - ctest
+ - ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous
 
  post_build:
  commands:
  - echo Executing post_build phase
  - echo $CODEBUILD_BUILD_SUCCEEDING
- - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ];
- then echo "Build passed, rerunning failed tests";
- cd /build_container/ufo;
- ctest -VV --rerun-failed;
+ - echo $BUILD_STATUS
+
+ # upload find cdash url and upload it as CodeBuild artifact to S3
+ - if [ "$BUILD_STATUS" = "1" ];
+ then echo "Build & tests passed, find cdash url";
+ bash /jcsda/ufo-bundle/ufo/CI/cdash-url.sh /build_container/ufo/Testing;
+ url=$(bash /jcsda/ufo-bundle/ufo/CI/cdash-url.sh /build_container/ufo/Testing);
+ echo $url;
+ echo ${url} > /jcsda/artifacts/cdash-url.txt;
+ cat /jcsda/artifacts/cdash-url.txt;
  else echo "Build failed";
  fi
+
+artifacts:
+ files:
+ - '/jcsda/artifacts/*'
+ name: ufo-clang-url