diff --git a/.github/workflows/docker-image-build-publish.yml b/.github/workflows/docker-image-build-publish.yml deleted file mode 100644 index d22ab701e6..0000000000 --- a/.github/workflows/docker-image-build-publish.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Based on https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#publishing-a-package-using-an-action (last accessed 2025-05-09) -name: Build and publish ctsm-docs Docker image - -on: - # Run this whenever a change to certain files gets pushed to master - push: - branches: ['master'] - paths: - - 'doc/ctsm-docs_container/**' - - '!doc/ctsm-docs_container/README.md' - - # Run this whenever it's manually called - workflow_dispatch: - -jobs: - # This first job checks that the container can be built, and that we can use it to build the docs without error. - build-image-and-test-docs: - name: Build image and test docs - uses: ./.github/workflows/docker-image-common.yml - secrets: inherit - - # This second job actually builds and publishes the container. - push-and-attest: - name: Publish and attest - runs-on: ubuntu-latest - - # Wait to run this job until after build-image-and-test-docs. If that job fails, something might be wrong with the container, so don't try this one. (It might also have failed because of something wrong with doc-builder or the documentation.) - needs: build-image-and-test-docs - - # Variables output by the build-image-and-test-docs job. - env: - REGISTRY: ${{ needs.build-image-and-test-docs.outputs.REGISTRY }} - IMAGE_NAME: ${{ needs.build-image-and-test-docs.outputs.IMAGE_NAME }} - VERSION_TAG: ${{ needs.build-image-and-test-docs.outputs.version_tag }} - - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - attestations: write - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - # Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # This step sets up Docker Buildx, which is needed for the multi-platform build in the next step - # https://docs.docker.com/build/ci/github-actions/multi-platform/ - # v3.1.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 - - # This step uses the `docker/build-push-action` action to build the image, based on the ctsm-docs `Dockerfile`. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - # Note that we should avoid relying on the "latest" tag for anything, but it's good practice to have one. - # v6.15.0 - - name: Push Docker image - id: push - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 - with: - context: doc/ctsm-docs_container - platforms: linux/amd64,linux/arm64 - push: true - load: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.VERSION_TAG }} - labels: "" - - # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). - - name: Generate artifact attestation - uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true - diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml deleted file mode 100644 index 6d38e12c8b..0000000000 --- a/.github/workflows/docker-image-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Modified from https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#publishing-a-package-using-an-action (last accessed 2025-05-09) -name: Build and test ctsm-docs container - -# Configures this workflow to run every time a change in the Docker container setup is pushed or included in a PR -on: - push: - # Run when a change to these files is pushed to any branch. Without the "branches:" line, for some reason this will be run whenever a tag is pushed, even if the listed files aren't changed. - branches: ['*'] - paths: - - 'doc/ctsm-docs_container/**' - - '!doc/ctsm-docs_container/README.md' - - '.github/workflows/docker-image-common.yml' - - pull_request: - # Run on pull requests that change the listed files - paths: - - 'doc/ctsm-docs_container/**' - - '!doc/ctsm-docs_container/README.md' - - '.github/workflows/docker-image-common.yml' - - workflow_dispatch: - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-image-and-test-docs: - name: Build image and test docs - uses: ./.github/workflows/docker-image-common.yml - secrets: inherit diff --git a/.github/workflows/docker-image-common.yml b/.github/workflows/docker-image-common.yml deleted file mode 100644 index 86de4fc2f2..0000000000 --- a/.github/workflows/docker-image-common.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Workflow used by docker-image workflows - -on: - workflow_call: - outputs: - REGISTRY: - description: "Container registry" - value: ${{ jobs.build-image-and-test-docs.outputs.REGISTRY }} - IMAGE_NAME: - description: "Docker image name" - value: ${{ jobs.build-image-and-test-docs.outputs.IMAGE_NAME }} - version_tag: - description: "Version tag from Dockerfile" - value: ${{ jobs.check-version.outputs.VERSION_TAG }} - -# Defines custom environment variables for the workflow. -env: - REGISTRY: ghcr.io - IMAGE_BASENAME: ctsm-docs - REPO: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-image-and-test-docs: - runs-on: ubuntu-latest - # Variables that might be needed by the calling workflow - outputs: - REGISTRY: ${{ env.REGISTRY }} - IMAGE_NAME: ${{ steps.set-image-name.outputs.IMAGE_NAME }} - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - - steps: - - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - # Ensure that the repository part of IMAGE_NAME is lowercase. This is needed because Docker requires image names to be entirely lowercase. Note that the *image name* part, set as IMAGE_BASENAME in the env block above, is *not* converted. This will cause the check-version job to fail if the IMAGE_BASENAME contains capitals. We don't want to silently fix that here; rather, we require the user to specify a lowercase IMAGE_BASENAME. - - name: Get image name with lowercase repo - id: set-image-name - run: | - lowercase_repo=$(echo $REPO | tr '[:upper:]' '[:lower:]') - echo "IMAGE_NAME=${lowercase_repo}/${IMAGE_BASENAME}" >> $GITHUB_ENV - echo "IMAGE_NAME=${lowercase_repo}/${IMAGE_BASENAME}" >> $GITHUB_OUTPUT - - # Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ steps.set-image-name.outputs.IMAGE_NAME }} - - # This step uses the `docker/build-push-action` action to build the image, based on the ctsm-docs `Dockerfile`. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - # v6.15.0 - - name: Build Docker image - id: build-image - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 - with: - context: doc/ctsm-docs_container - push: false - load: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - # Check out all submodules because we might :literalinclude: something from one - - name: Checkout all submodules - run: | - bin/git-fleximod update -o - - - name: Set image tag for docs build - id: set-image-tag - run: | - echo "IMAGE_TAG=$(echo '${{ steps.meta.outputs.tags }}' | head -n 1 | cut -d',' -f1)" >> $GITHUB_ENV - - - name: Build docs using Docker (Podman has trouble on GitHub runners) - id: build-docs - run: | - cd doc && ./build_docs -b ${PWD}/_build -c -d -i $IMAGE_TAG - - - check-version: - needs: build-image-and-test-docs - uses: ./.github/workflows/docker-image-get-version.yml - with: - registry: ${{ needs.build-image-and-test-docs.outputs.REGISTRY }} - image_name: ${{ needs.build-image-and-test-docs.outputs.IMAGE_NAME }} diff --git a/.github/workflows/docker-image-get-version.yml b/.github/workflows/docker-image-get-version.yml deleted file mode 100644 index c405b861b5..0000000000 --- a/.github/workflows/docker-image-get-version.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Get and check version specified in a Dockerfile - -on: - workflow_call: - inputs: - registry: - required: true # Require any workflows calling this one to provide input - type: string - default: 'ghcr.io' # Provide default so this workflow works standalone too - image_name: - required: true # Require any workflows calling this one to provide input - type: string - default: 'escomp/ctsm/ctsm-docs' # Provide default so this workflow works standalone too - outputs: - VERSION_TAG: - description: "Tag to be pushed to container registry" - value: ${{ jobs.get-check-version.outputs.VERSION_TAG }} - workflow_dispatch: - inputs: - registry: - description: 'Container registry' - required: false - type: string - default: 'ghcr.io' - image_name: - description: 'Image name' - required: false - type: string - default: 'escomp/ctsm/ctsm-docs' - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - get-check-version: - name: Get version number from Dockerfile and check it - runs-on: ubuntu-latest - outputs: - VERSION_TAG: ${{ steps.get-check-version.outputs.version_tag }} - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - - steps: - - name: Check out repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get version number from Dockerfile and check it - id: get-check-version - run: | - set -e - set -o pipefail - set -u - VERSION="$(doc/ctsm-docs_container/get_version.sh)" - VERSION_TAG="${{ inputs.registry }}/${{ inputs.image_name }}:${VERSION}" - - # Store the manifest inspect result and output - set +e - INSPECT_RESULT="$(docker manifest inspect "$VERSION_TAG" 2>&1)" - INSPECT_STATUS=$? - set -e - - if [[ "${INSPECT_RESULT}" == *"schemaVersion"* ]]; then - echo "Tag $VERSION_TAG already exists!" >&2 - exit 123 - elif [[ "${INSPECT_RESULT}" != "manifest unknown" ]]; then - # "manifest unknown" means the tag doesn't exist, which is what we want - echo -e "Error checking manifest for $VERSION_TAG:\n${INSPECT_RESULT}" >&2 - exit $INSPECT_STATUS - fi - - echo "Setting version_tag to $VERSION_TAG" - echo "version_tag=$VERSION_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs-build-and-deploy.yml b/.github/workflows/docs-build-and-deploy.yml index 7951532848..cead6775ab 100644 --- a/.github/workflows/docs-build-and-deploy.yml +++ b/.github/workflows/docs-build-and-deploy.yml @@ -61,7 +61,7 @@ jobs: id: build-docs run: | cd doc - ./build_docs_to_publish -d --site-root https://escomp.github.io/CTSM + ./build_docs_to_publish -d --site-root https://escomp.github.io/CTSM --verbose - name: Upload artifact uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3 diff --git a/.github/workflows/docs-common.yml b/.github/workflows/docs-common.yml index a339b0b7f9..57f3141a2c 100644 --- a/.github/workflows/docs-common.yml +++ b/.github/workflows/docs-common.yml @@ -43,13 +43,13 @@ jobs: uses: actions/cache@2f8e54208210a422b2efd51efaa6bd6d7ca8920f # v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('doc/ctsm-docs_container/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('doc/doc-builder/doc-build-container/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - name: Install Python requirements if: ${{ ! inputs.use_conda }} run: | - pip install -r doc/ctsm-docs_container/requirements.txt + pip install -r doc/doc-builder/doc-build-container/requirements.txt # Do this if using conda - name: Set up conda environment @@ -73,7 +73,7 @@ jobs: if: success() || failure() run: | if ${{ inputs.use_conda }}; then - cd doc && conda run -n ${{ inputs.conda_env_name }} ./build_docs -b ${PWD}/_build -c + cd doc && conda run -n ${{ inputs.conda_env_name }} ./build_docs -b ${PWD}/_build -c --verbose else - cd doc && ./build_docs -b ${PWD}/_build -c + cd doc && ./build_docs -b ${PWD}/_build -c --verbose fi diff --git a/.github/workflows/docs-update-ctsm_pylib.yml b/.github/workflows/docs-update-ctsm_pylib.yml index 4ea5af396c..19e0bbfa2b 100644 --- a/.github/workflows/docs-update-ctsm_pylib.yml +++ b/.github/workflows/docs-update-ctsm_pylib.yml @@ -6,7 +6,7 @@ on: branches: ['*'] paths: - 'python/conda_env_ctsm_py.txt' - - 'doc/ctsm-docs_container/requirements.txt' + - 'doc/doc-builder' - '.github/workflows/docs-common.yml' - '.github/workflows/docs-update-dependency-common.yml' @@ -14,7 +14,7 @@ on: # Run on pull requests that change the listed files paths: - 'python/conda_env_ctsm_py.txt' - - 'doc/ctsm-docs_container/requirements.txt' + - 'doc/doc-builder' - '.github/workflows/docs-common.yml' - '.github/workflows/docs-update-dependency-common.yml' diff --git a/.github/workflows/docs-update-dependency-common.yml b/.github/workflows/docs-update-dependency-common.yml index 7bd0fd8839..dfe5692a81 100644 --- a/.github/workflows/docs-update-dependency-common.yml +++ b/.github/workflows/docs-update-dependency-common.yml @@ -22,7 +22,8 @@ jobs: # Don't run on forks, because test_container_eq_ctsm_pylib.sh uses # build_docs_to_publish, which will look for branch(es) that forks - # may not have + # may not have. + # To run locally with `act`, include `--env GITHUB_REPOSITORY=ESCOMP/CTSM` in the call. if: ${{ github.repository == 'ESCOMP/CTSM' }} runs-on: ubuntu-latest @@ -44,6 +45,7 @@ jobs: environment-file: ${{ inputs.conda_env_file }} channels: conda-forge auto-activate-base: false + miniconda-version: "latest" - name: Compare docs built with container vs. ctsm_pylib run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4832192fd9..77a91aeb87 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -63,7 +63,7 @@ jobs: run: | set -o pipefail mkdir -p build-logs - cd doc && PYTHONUNBUFFERED=1 ./build_docs -b ${PWD}/_build -c -d 2>&1 | tee >(sed -E $'s/\x1b\\[[0-9;]*[a-zA-Z]//g' > "${GITHUB_WORKSPACE}/build-logs/build.log") + cd doc && PYTHONUNBUFFERED=1 ./build_docs -b ${PWD}/_build -c -d --verbose 2>&1 | tee >(sed -E $'s/\x1b\\[[0-9;]*[a-zA-Z]//g' > "${GITHUB_WORKSPACE}/build-logs/build.log") # The tee writes build.log for the PR comment (posted by docs-pr-failure-post-comment.yml); the inner sed strips ANSI color codes that would otherwise render as garbage in the comment. # PYTHONUNBUFFERED=1 because otherwise the teed log will be out of order diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 79ad016da2..c5d9944ef3 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -24,6 +24,10 @@ jobs: # Checkout the code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Check out doc-builder, which has a requirements.txt we need + run: | + bin/git-fleximod update doc-builder + # Set up the conda environment - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3 with: @@ -44,6 +48,10 @@ jobs: # Checkout the code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Check out doc-builder, which has a requirements.txt we need + run: | + bin/git-fleximod update doc-builder + # Set up the conda environment - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3 with: diff --git a/.gitmodules b/.gitmodules index 87460be5f3..624d209db8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -124,7 +124,7 @@ fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial [submodule "doc-builder"] path = doc/doc-builder url = https://github.com/ESMCI/doc-builder -fxtag = v3.2.1 +fxtag = v3.5.1 fxrequired = ToplevelOptional # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed fxDONOTUSEurl = https://github.com/ESMCI/doc-builder diff --git a/doc/ctsm-docs_container/Dockerfile b/doc/ctsm-docs_container/Dockerfile deleted file mode 100644 index c38a384f80..0000000000 --- a/doc/ctsm-docs_container/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -################################################################### -# A base linux install + packages for building CTSM documentation # -################################################################### - -FROM python:3.13.2-alpine - -# Update the default packages and install some other necessary ones -RUN apk update -RUN apk add --no-cache git sudo -RUN apk add --no-cache git-lfs -RUN apk add --no-cache make -ADD requirements.txt requirements.txt -RUN pip3 install --break-system-packages -r requirements.txt - -# Add user etc. -RUN addgroup escomp -RUN adduser -h /home/user -G escomp -s /bin/bash -D user -RUN echo 'export USER=$(whoami)' >> /etc/profile.d/escomp.sh -RUN echo 'export PS1="[\u@escomp \W]\$ "' >> /etc/profile.d/escomp.sh -RUN echo 'user ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers.d/escomp - -ENV SHELL=/bin/bash \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 - -USER user -WORKDIR /home/user -CMD ["/bin/bash", "-l"] - -LABEL org.opencontainers.image.title="Container for building CTSM documentation" -LABEL org.opencontainers.image.source=https://github.com/ESCOMP/CTSM -LABEL org.opencontainers.image.version="v2.0.1a" diff --git a/doc/ctsm-docs_container/README.md b/doc/ctsm-docs_container/README.md deleted file mode 100644 index 2c8110275d..0000000000 --- a/doc/ctsm-docs_container/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# The ctsm-docs container -This directory and its Dockerfile are used to build a container for building the CTSM documentation. Unless you're a developer working on the container, you probably don't need to care about anything in here. - -## Introduction - -This Readme tells you how to update the ctsm-docs container if a need to do so arises—for example, adding a Python module that brings new functionality in the build. After you've followed all these instructions, you will probably want to push an update to [doc-builder](https://github.com/ESMCI/doc-builder) that updates `DEFAULT_IMAGE` in [build_commands.py](https://github.com/ESMCI/doc-builder/blob/master/doc_builder/build_commands.py) to point to the new tag. - -## Building - -If you actually want to build the container, you will need to have Podman installed. We previously used Docker for this, but had to move away from it due to licensing issues. Note that these issues have to do specifically with Docker Desktop, which is required to get the Docker Engine on some platforms. The Docker Engine itself is open-source, so our ctsm-docs container testing and publishing workflows are fine to continue using it. We are also fine to use the Docker Engine via Podman for local builds, which is what's described here. - -Once you have Podman installed, you can do: -```shell -podman build --no-cache -t ctsm-docs . -``` - -To use your new version for local testing, you'll need to tell doc-builder to use that image. Call `podman images`, which should return something like this: -```shell -REPOSITORY TAG IMAGE ID CREATED SIZE -localhost/ctsm-docs latest 6464f26339bc 22 seconds ago 241 MB -... -``` - -To test, you can tell `build_docs` to use your new version by adding `--container-image IMAGE_ID` to your call, where in the example above `IMAGE_ID` is `6464f26339bc`. - -## Publishing automatically - -The `docker-image-build-publish.yml` workflow makes it so that new versions of the workflow will be published to the GitHub Container Registry whenever changes to the container setup are merged to CTSM's `master` branch. This will fail (as will a similar, no-publish workflow that happens on PRs) unless you specify exactly one new version number in the Dockerfile. This version number will be used as a tag that can be referenced by, e.g., doc-builder. - -Lots of container instructions tell you to use the `latest` tag, and indeed the workflow will add that tag automatically. However, actually _using_ `latest` can lead to support headaches as users think they have the right version but actually don't. Instead, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format. - -Here's where you need to specify the version number in the Dockerfile: -```docker -LABEL org.opencontainers.image.version="vX.Y.Z" -``` -The string there can technically be anything as long as (a) it starts with a lowercase `v` and (b) it hasn't yet been used on a published version of the container. You may need to "bump" the version string in order for various tests to pass; if so, just add a lowercase letter at the end. - -You can check the results of the automatic publication on the [container's GitHub page](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs). - -### Updating doc-builder -After the new version of the container is published, you will probably want to tell [doc-builder](https://github.com/ESMCI/doc-builder) to use the new one. Open a PR where you change the tag (the part after the colon) in the definition of `DEFAULT_IMAGE` in `doc_builder/build_commands.py`. Remember, **use the version number**, not "latest". - -## Publishing manually - -It's vastly preferable to let GitHub build and publish the new repo using the `docker-image-build-publish.yml` workflow as described above. However, you may need to publish manually if, for instance, you introduce a change that breaks `doc-builder`. You could work around that by first merging a CTSM `master` PR that updates the container, then updating `doc-builder` to use it, then updating CTSM to use the new `doc-builder`. That's not always practical, though, so here's how to publish the container manually. - -### Building the multi-architecture version - -When publishing our container, we need to make sure it can run on either arm64 or amd64 processor architecture. This requires a special build process: -```shell -podman manifest rm ctsm-docs-manifest 2>/dev/null -podman manifest create ctsm-docs-manifest -podman build --platform linux/amd64,linux/arm64 --manifest ctsm-docs-manifest . -``` - -### Pushing to GitHub Container Registry -If you want to publish the container, you first need a [GitHub Personal Access Token (Classic)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic) with the `write:packages` permissions. You can see your existing PAT(C)s [here](https://github.com/settings/tokens). If you don't have one with the right permissions, [this link](https://github.com/settings/tokens/new?scopes=write:packages) should start the setup process for you. - -Once you have a PAT(C), you can authenticate in your shell session like so: - -```bash -# bash: Make it so that, in this session, commands with a leading space are not saved to terminal history -export HISTCONTROL=ignoreboth - -# Include leading spaces so that your secret PAT(C) isn't included in terminal history - echo YOUR_PERSONAL_ACCESS_TOKEN_CLASSIC | podman login ghcr.io -u YOUR_USERNAME --password-stdin -``` - -### Tagging -You'll next need to tag the image. Lots of container instructions tell you to use the `latest` tag, and Podman may actually add that for you. However, using `latest` in `doc-builder` can lead to support headaches as users think they have the right version but actually don't. So in addition to `latest`, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format. - -Tag the manifest with your version number like so: -```shell -podman tag ctsm-docs-manifest ghcr.io/escomp/ctsm/ctsm-docs:v2.0.1 -``` - -Push to the repo: -```shell -podman manifest push --all ctsm-docs-manifest ghcr.io/escomp/ctsm/ctsm-docs:vX.Y.Z -podman manifest push --all ctsm-docs-manifest ghcr.io/escomp/ctsm/ctsm-docs:latest -``` - -Then browse to the [container's GitHub page](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs) to make sure this all worked and the image is public. - -### Updating doc-builder -See "Updating doc-builder" in the "Publishing automatically" section above. - -## See also - -- [GitHub: Working with the container registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) -- [GitHub: Connecting a repository to a package](https://docs.github.com/en/packages/learn-github-packages/connecting-a-repository-to-a-package) \ No newline at end of file diff --git a/doc/ctsm-docs_container/get_version.sh b/doc/ctsm-docs_container/get_version.sh deleted file mode 100755 index 485e720eff..0000000000 --- a/doc/ctsm-docs_container/get_version.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -e -cd doc/ctsm-docs_container - -# Extract version from Dockerfile -version="$(grep "org.opencontainers.image.version" Dockerfile | cut -d'"' -f2 | sort |uniq)" -n_found=$(echo $version | wc -w) - -# Error if anything other than exactly one version tag was found -if [[ ${n_found} -gt 1 ]]; then - echo -e "Multiple version tags found:\n${version}" >&2 - exit 2 -elif [[ ${n_found} -lt 1 ]]; then - echo "Expected 1 but found 0 version tags" >&2 - exit -1 -fi - -# Error if version doesn't start with v -if [[ "${version}" != "v"* ]]; then - echo "Version '${version}' doesn't start with v" >&2 - exit 22 -fi - -echo ${version} - -exit 0 diff --git a/doc/ctsm-docs_container/requirements.txt b/doc/ctsm-docs_container/requirements.txt deleted file mode 100644 index 19872cb9c0..0000000000 --- a/doc/ctsm-docs_container/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Python packages used for building docs -rst2pdf == 0.103.1 -sphinx == 8.2.3 -sphinxcontrib_programoutput == 0.18 -myst-parser == 5.0.0 -sphinx_rtd_theme == 3.0.2 diff --git a/doc/doc-builder b/doc/doc-builder index 15e171dfcf..684c77782f 160000 --- a/doc/doc-builder +++ b/doc/doc-builder @@ -1 +1 @@ -Subproject commit 15e171dfcf77ca2bd85415a99a50ad3994c608c4 +Subproject commit 684c77782f6612be58b8f9c9adb494b99d01dbcc diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.md b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.md new file mode 100644 index 0000000000..262ef6f17d --- /dev/null +++ b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.md @@ -0,0 +1,989 @@ + +$\newcommand{\gddthreshgrain}{GDD_\textrm{*grain}}$ +$\newcommand{\gddthreshmat}{GDD_\textrm{*mat}}$ +$\newcommand{\gddthreshmatbl}{\gddthreshmat^\textrm{bl}}$ +$\newcommand{\gddaccsoil}{GDD_{T_\textrm{soi}}}$ +$\newcommand{\ttwom}{T_\textrm{2m}}$ +$\newcommand{\gddacctwom}{GDD_{\ttwom}}$ +$\newcommand{\gddzero}{GDD_0}$ +$\newcommand{\gddeight}{GDD_8}$ +$\newcommand{\gddten}{GDD_{10}}$ +$\newcommand{\gddx}{GDD_x}$ +$\newcommand{\gddzerorun}{\overline{\gddzero}^\textrm{20yr}}$ +$\newcommand{\gddeightrun}{\overline{\gddeight}^\textrm{20yr}}$ +$\newcommand{\gddtenrun}{\overline{\gddten}^\textrm{20yr}}$ +$\newcommand{\gddxrun}{\overline{\gddx}^\textrm{20yr}}$ +$\newcommand{\gddxrunbl}{\overline{\gddx}^\textrm{20-yr,bl}}$ +$\newcommand{\gddxdaymax}{\gddx^\textrm{daymax}}$ +$\newcommand{\huithreshlfemerg}{h_\textrm{*lfemerg}}$ +$\newcommand{\huithreshgrain}{h_\textrm{*grain}}$ +$\newcommand{\huithreshgrainactual}{h_\textrm{*grain}^{'}}$ +$\newcommand{\parambaset}{T_\textrm{base}}$ +$\newcommand{\paramztopmx}{z_\textrm{top}^\textrm{max}}$ +$\newcommand{\paramaleaff}{a_\textrm{leaf}^f}$ +$\newcommand{\paramfleafi}{a_\textrm{leaf}^i}$ +$\newcommand{\paramarooti}{a_\textrm{froot}^i}$ +$\newcommand{\paramarootf}{a_\textrm{froot}^f}$ +$\newcommand{\paramastemf}{a_\textrm{stem}^f}$ +$\newcommand{\paramaorganf}{a_\textrm{organ}^f}$ +$\newcommand{\paramlaimx}{L_\textrm{max}}$ +$\newcommand{\paramdeclfact}{d_L}$ +$\newcommand{\paramallconsl}{d_\textrm{alloc}^\textrm{leaf}}$ +$\newcommand{\paramallconss}{d_\textrm{alloc}^\textrm{stem}}$ +$\newcommand{\paramallconso}{d_\textrm{alloc}^\textrm{organ}}$ +$\newcommand{\paramleafcn}{CN_\textrm{leaf}}$ +$\newcommand{\paramfleafcn}{\paramleafcn^\textrm{f}}$ +$\newcommand{\paramlivewdcn}{CN_\textrm{stem}}$ +$\newcommand{\paramfstemcn}{\paramlivewdcn^\textrm{f}}$ +$\newcommand{\paramfrootcn}{CN_\textrm{froot}}$ +$\newcommand{\paramffrootcn}{\paramfrootcn^\textrm{f}}$ +$\newcommand{\paramgraincn}{CN_\textrm{grain}}$ +$\newcommand{\paramorgancn}{CN_\textrm{[organ]}}$ +$\newcommand{\paramforgancn}{\paramorgancn^\textrm{f}}$ +$\newcommand{\corgan}{C_\textrm{[organ]}}$ +$\newcommand{\norgan}{N_\textrm{[organ]}}$ +$\newcommand{\paramplantingtemp}{T_\textrm{p}}$ +$\newcommand{\paramminplantingtemp}{\paramplantingtemp^\textrm{min}}$ +$\newcommand{\paramgddmin}{GDD_\textrm{min}}$ +$\newcommand{\parambiofuelharvfrac}{\texttt{biofuel_harvfrac}}$ +$\newcommand{\paramcropresidueremovalfrac}{\texttt{crop_residue_removal_frac}}$ +$\newcommand{\fracthrugrnfill}{\frac{\gddacctwom - \gddthreshgrain}{\gddthreshmat \paramdeclfact - \gddthreshgrain}}$ + +(rst_crops and irrigation)= + +# Crops and Irrigation + +(the-crop-model)= + +## The crop model: cash and bioenergy crops + +### Introduction + +Groups developing Earth System Models generally account for the human footprint on the landscape in simulations of historical and future climates. For a long time, this was accomplished using natural vegetation types—particularly grasses—because they resemble many common crops, but now it is more common to explicitly represent land management such as crop type, planting, harvesting, fertilization, irrigation, and more. + +AgroIBIS is a state-of-the-art land surface model with options to simulate dynamic vegetation ({ref}`Kucharik et al. 2000 `) and interactive crop management ({ref}`Kucharik and Brye 2003 `). The interactive crop management parameterizations from AgroIBIS (March 2003 version) were coupled (not published) as a proof-of-concept in CLM3.0 ({ref}`Oleson et al. (2004) `), then CLM3.5 ({ref}`Levis et al. 2009 `) and later released to the community with CLM4CN ({ref}`Levis et al. 2012 `), and CLM4.5BGC. Additional updates after the release of CLM4.5 were available by request ({ref}`Levis et al. 2016 `), and beginning with CLM5 crops have been incorporated into the main codebase. + +With interactive crop management and, therefore, a more accurate representation of agricultural landscapes, we hope to improve CLM's simulated biogeophysics and biogeochemistry. These advances may improve fully coupled simulations with the Community Earth System Model (CESM), while helping human societies answer questions about changing food, energy, and water resources in response to climate, environmental, land use, and land management change (e.g., {ref}`Kucharik and Brye 2003 `; {ref}`Lobell et al. 2006 `). The CLM crop model uses the same physiology as the natural vegetation but with different crop-specific parameter values, phenology, allocation, as well as management (fertilizer, irrigation, tillage, and residue removal). + +(crop-plant-functional-types)= + +### Crop plant functional types + +To allow crops to coexist with natural vegetation in a grid cell, the vegetated land unit is separated into a naturally vegetated land unit and a managed crop land unit. Unlike the plant functional types (PFTs) in the naturally vegetated land unit, the managed crop PFTs in the managed crop land unit do not share soil columns and thus permit for differences in the land management between crops. Each crop type has a rainfed and an irrigated PFT that are on independent soil columns. Crop area distributions are defined as explained in Sects. {numref}`Surface Data` and {numref}`rst_Transient Landcover Change`; see Sect. {numref}`Surface Heterogeneity and Data Structure` for more information on land units and soil columns. + +CLM includes ten actively managed crop types (temperate soybean, tropical soybean, temperate corn, tropical corn, spring wheat, cotton, rice, sugarcane, _Miscanthus_, and switchgrass) that are chosen based on the availability of corresponding algorithms in AgroIBIS and as developed by {ref}`Badger and Dirmeyer (2015)` and described by {ref}`Levis et al. (2016)`, or from available observations as described by {ref}`Cheng et al. (2019)`. Sugarcane and tropical corn are both C{sub}`4` plants and are therefore represented using the temperate corn functional form—i.e., they differ in only a few parameters. Tropical soybean uses the temperate soybean functional form, while rice and cotton use the wheat functional form. In tropical regions, parameter values were originally developed for the Amazon Basin. Plantation areas of bioenergy crops are projected to expand throughout the 21st century as a major energy source to replace fossil fuels and mitigate climate change. _Miscanthus_ and switchgrass are perennial bioenergy crops and have quite different physiological traits and land management practices than annual crops, such as longer growing seasons, higher productivity, and lower demands for nutrients and water. About 70% of biofuel aboveground biomass (leaf and stem) is removed at harvest. Parameter values were developed by using observation data collected at the University of Illinois Energy Farm located in Central Midwestern United States ({ref}`Cheng et al., 2019`). + +In addition, CLM's default list of plant functional types (PFTs) includes an irrigated and unirrigated unmanaged C{sub}`3` crop ({numref}`Table Crop plant functional types`) treated as a second C{sub}`3` grass. The unmanaged C{sub}`3` crop is only used when the crop model is not active and has grid cell coverage assigned from satellite data, and the unmanaged C{sub}`3` irrigated crop type is currently not used since irrigation requires the crop model to be active. The default list of PFTs also includes twenty-one inactive crop PFTs that do not yet have associated parameters required for active management. Each of the inactive crop types is simulated using the parameters of the spatially closest associated crop type that is most similar to the functional type (e.g., C{sub}`3` or C{sub}`4`), which is required to maintain similar phenological parameters based on temperature thresholds. Information detailing which parameters are used for each crop type is included in {numref}`Table Crop plant functional types`; this information can also be found in the parameter file (variable `mergetoclmpft`; see [](#query-paramfile)). + +It should be noted that PFT-level history output merges all crop types into the actively managed crop type, so analysis of crop-specific output will require use of the land surface input dataset to remap the yields of each actively and inactively managed crop type. Otherwise, the actively managed crop type will include yields for that crop type and all inactively managed crop types that are using the same parameter set. Note that if a "pure" simulation of the actively-managed crop types is desired, the `mergetoclmpft` parameter of the other crop types should be set to something not usually simulated in runs with managed crops on (`BgcCrop` component sets), such as 15 (C{sub}`3` unmanaged crops; see [](#set-paramfile)). + +```{warning} +Enabling the inactive crop PFTs will cause the model to error due to certain logic in the code that does not yet handle those PFTs; see the related [GitHub issue](https://github.com/ESCOMP/CTSM/issues/3388). +``` + +(table crop plant functional types)= + +```{list-table} Crop plant functional types (PFTs) included in CLM. IVT is the integer used to refer to each vegetation type in the FORTRAN code. {{paramfile_disclaimer_md}} +:header-rows: 1 + +* - IVT + - Plant functional type (PFT) + - Simulated as (`mergetoclmpft`) +* - 15 + - C{sub}`3` unmanaged rainfed crop + - (itself) +* - 16 + - C{sub}`3` unmanaged irrigated crop + - (itself) +* - 17 + - rainfed temperate corn + - (itself) +* - 18 + - irrigated temperate corn + - (itself) +* - 19 + - rainfed spring wheat + - (itself) +* - 20 + - irrigated spring wheat + - (itself) +* - 21 + - rainfed winter wheat + - rainfed spring wheat +* - 22 + - irrigated winter wheat + - irrigated spring wheat +* - 23 + - rainfed temperate soybean + - (itself) +* - 24 + - irrigated temperate soybean + - (itself) +* - 25 + - rainfed barley + - rainfed spring wheat +* - 26 + - irrigated barley + - irrigated spring wheat +* - 27 + - rainfed winter barley + - rainfed spring wheat +* - 28 + - irrigated winter barley + - irrigated spring wheat +* - 29 + - rainfed rye + - rainfed spring wheat +* - 30 + - irrigated rye + - irrigated spring wheat +* - 31 + - rainfed winter rye + - rainfed spring wheat +* - 32 + - irrigated winter rye + - irrigated spring wheat +* - 33 + - rainfed cassava + - rainfed rice +* - 34 + - irrigated cassava + - irrigated rice +* - 35 + - rainfed citrus + - rainfed spring wheat +* - 36 + - irrigated citrus + - irrigated spring wheat +* - 37 + - rainfed cocoa + - rainfed rice +* - 38 + - irrigated cocoa + - irrigated rice +* - 39 + - rainfed coffee + - rainfed rice +* - 40 + - irrigated coffee + - irrigated rice +* - 41 + - rainfed cotton + - (itself) +* - 42 + - irrigated cotton + - (itself) +* - 43 + - rainfed datepalm + - rainfed cotton +* - 44 + - irrigated datepalm + - irrigated cotton +* - 45 + - rainfed foddergrass + - rainfed spring wheat +* - 46 + - irrigated foddergrass + - irrigated spring wheat +* - 47 + - rainfed grapes + - rainfed spring wheat +* - 48 + - irrigated grapes + - irrigated spring wheat +* - 49 + - rainfed groundnuts + - rainfed rice +* - 50 + - irrigated groundnuts + - irrigated rice +* - 51 + - rainfed millet + - rainfed tropical corn +* - 52 + - irrigated millet + - irrigated tropical corn +* - 53 + - rainfed oilpalm + - rainfed rice +* - 54 + - irrigated oilpalm + - irrigated rice +* - 55 + - rainfed potatoes + - rainfed spring wheat +* - 56 + - irrigated potatoes + - irrigated spring wheat +* - 57 + - rainfed pulses + - rainfed spring wheat +* - 58 + - irrigated pulses + - irrigated spring wheat +* - 59 + - rainfed rapeseed + - rainfed spring wheat +* - 60 + - irrigated rapeseed + - irrigated spring wheat +* - 61 + - rainfed rice + - (itself) +* - 62 + - irrigated rice + - (itself) +* - 63 + - rainfed sorghum + - rainfed tropical corn +* - 64 + - irrigated sorghum + - irrigated tropical corn +* - 65 + - rainfed sugarbeet + - rainfed spring wheat +* - 66 + - irrigated sugarbeet + - irrigated spring wheat +* - 67 + - rainfed sugarcane + - (itself) +* - 68 + - irrigated sugarcane + - (itself) +* - 69 + - rainfed sunflower + - rainfed spring wheat +* - 70 + - irrigated sunflower + - irrigated spring wheat +* - 71 + - rainfed _Miscanthus_ + - (itself) +* - 72 + - irrigated _Miscanthus_ + - (itself) +* - 73 + - rainfed switchgrass + - (itself) +* - 74 + - irrigated switchgrass + - (itself) +* - 75 + - rainfed tropical corn + - (itself) +* - 76 + - irrigated tropical corn + - (itself) +* - 77 + - rainfed tropical soybean + - (itself) +* - 78 + - irrigated tropical soybean + - (itself) +``` + +(phenology)= + +### Phenology + +CLM uses the AgroIBIS crop phenology algorithm, consisting of three distinct phases. + +Phase 1 starts at planting and ends with leaf emergence, phase 2 continues from leaf emergence to the beginning of grain fill, and phase 3 starts from the beginning of grain fill and ends with physiological maturity and harvest. + +(planting)= + +#### Planting + +Each crop can be planted in each gridcell once per year, in the "sowing window." This is, by default, defined for each crop in each gridcell, with the start and end of the window drawn from the input files `stream_fldFileName_swindow_start` and `stream_fldFileName_swindow_end`, respectively. These sowing windows are centered on the sowing dates developed for phase 3 of the Inter-Sectoral Impacts Model Intercomparison Project's Agriculture sector runs ({ref}`Jägermeyr et al., 2021 `, {ref}`Rabin et al., 2023 `). The widths of the sowing windows are based on the original sowing windows from the `min_NH_planting_date`, `max_NH_planting_date`, `min_SH_planting_date`, and `max_SH_planting_date` parameters, which remain on the parameter file but are ignored by default (i.e., unless you set `cropcals_rx = .false.` in `user_nl_clm`). See {ref}`running-with-custom-crop-calendars`. + +To be planted, a crop patch must meet the following requirements sometime within its sowing window: + +$$ +\begin{align} +T_{10d} &> \paramplantingtemp \\ +T_{10d}^{\min } &> \paramminplantingtemp \\ +\gddeightrun &\ge \paramgddmin +\end{align} +$$ (25.1) + +where ${T}_{10d}$ is the 10-day running mean of $\ttwom$, (the simulated 2-m air temperature during each model time step) and $T_{10d}^{\min}$ is the 10-day running mean of $\ttwom^{\min }$ (the daily minimum of $\ttwom$). $\paramplantingtemp$ and $\paramminplantingtemp$ are crop-specific coldest planting temperatures ({numref}`Table Crop phenology parameters`), $\gddeightrun$ is the 20-year running mean growing degree-days (units are °C day) tracked from April through September (NH) above 8°C with maximum daily increments of 30 degree-days (see equation {eq}`25.3`), and $\paramgddmin$is the minimum growing degree day requirement ({numref}`Table Crop phenology parameters`). $\gddeightrun$ does not change as quickly as ${T}_{10d}$ and $T_{10d}^{\min }$, so it determines whether it is warm enough for the crop to be planted in a grid cell, while the 2-m air temperature variables determine the day when the crop may be planted if the $\gddeightrun$ threshold is met. If the requirements in equation {eq}`25.1` are not met by the maximum planting date, crops are still planted on the maximum planting date as long as $\gddeightrun > 0$. + +At planting, each crop seed pool is assigned 3 gC m{sup}`-2` from its grain product pool. The seed carbon is transferred to the leaves upon leaf emergence. An equivalent amount of seed leaf N is assigned given the PFT's C to N ratio for leaves ($\paramleafcn$ in {numref}`Table Crop allocation parameters`; this differs from AgroIBIS, which uses a seed leaf area index instead of seed C). + +#### Maturity requirement +At planting, CLM determines how many growing degree-days will be needed for the crop to reach maturity and thus be harvested. By default (i.e., `cropcals_rx_adapt = .true.`), this is set according to two input files with PFT-specific maps: +- `stream_fldfilename_cultivar_gdds` ($\gddthreshmatbl$): the average growing-degree days to reach maturity in the "baseline" period. +- `stream_fldFileName_gdd20_baseline` ($\gddxrunbl$): the means over the baseline period of $\gddzerorun$, $\gddeightrun$, and $\gddtenrun$. + +Maturity requirement, $\gddthreshmat$, is then calculated as: + +$$ +\gddthreshmat = \max \left( 1,\ \gddthreshmatbl \times \frac{\gddxrun}{\gddxrunbl} \right), +$$ (gddmat-rx-adapt) + +where $x$ is 0 (wheat, cotton, and rice), 8 (corn, sugarcane, _Miscanthus_, and switchgrass), or 10 (soybean). This allows the maturity requirement to "adapt" over time, being lower in cool periods and higher in warm periods. The baseline period is the 1980-2009 growing seasons (i.e., seasons where planting occurred in those calendar years, inclusive); baseline values were calculated based on a half-degree, land-only run with CRU-JRA climate forcings (Rabin et al., in prep.). The minimum value of 1 avoids numeric issues when $\gddthreshmat$ is in the denominator of a calculation. + +- **Check baseline period** + +If `cropcals_rx_adapt` is false but `cropcals_rx` is true, the calculation is just $\gddthreshmat = \gddthreshmatbl$. + +If both `cropcals_rx_adapt` and `cropcals_rx` are false, or if $\gddthreshmatbl$ is negative, then CLM sets $\gddthreshmat$ according to various crop-specific rules based on $\gddx$, the PFT-specific parameter `hybgdd`, and hard-coded minimum and maximum values. + +Equation {eq}`25.3` shows how we calculate $\gddzero$, $\gddeight$, and $\gddten$ for each model timestep: + +$$ +\gddx = \gddx + \frac{\min \left( \gddxdaymax,\ \max \left[ 0,\ \ttwom - 273.15 - x \right] \right)}{48} +$$ (25.3) + +where $\ttwom$ is the 2-m air temperature (K), 273.15 K is the freezing temperature of water, and $GDD$ is in units of °C-days. $\gddxdaymax$, the maximum daily growing degree-day accumulation, is 26°C for $x=0$ and 30°C for $x=8$ and $x=10$. The division by 48 is to adjust for the number of timesteps in a day. + +By default, the $\gddx$ values are set to zero at the beginning of the "$\gddx$ season" and then accumulated through its end: from April 1 through September 30 in the Northern Hemisphere and from October 1 through March 31 in the Southern Hemisphere. (Setting `stream_gdd20_seasons = .true.` would instead take those start and end dates from PFT-specific maps in the input files `stream_fldFileName_gdd20_season_start` and `stream_fldFileName_gdd20_season_end`, respectively; however, this is not scientifically supported.) At the end of each $\gddx$ season, the final value of $\gddx$ is incorporated into $\gddxrun$ like so: + +$$ +\gddxrun = \frac{\gddxrun \times \min(n-1, 19) + \gddx}{20}, +$$ (update-gddxrun) + +where $n$ is the number of years that $\gddxrun$ gas been calculated for. Note that this is not a true rolling 20-year mean, which would come with a memory cost in the simulation as a 20-member array would need to be saved for each PFT. + +(leaf-emergence)= + +#### Leaf emergence + +The "leaf emergence" phase is the period of vegetative growth between when the leaves first emerge from the soil to when filling of the reproductive organ begins. + +According to AgroIBIS, leaves may emerge when the growing degree-days of soil temperature to 0.05 m depth ($\gddaccsoil$ ), which is tracked since planting, reaches 1 to 5% of $\gddthreshmat$ (see $\huithreshlfemerg$ in {numref}`Table Crop phenology parameters`). The base temperature threshold values for $\gddaccsoil$ are listed in {numref}`Table Crop phenology parameters` (the same base temperature threshold values are also used for $\gddacctwom$ in section {numref}`Grain Fill`), and leaf emergence (crop phenology phase 2) starts when this threshold is met. Leaf onset occurs in the first time step of phase 2, at which moment all seed C is transferred to leaf C. Subsequently, the leaf area index generally increases throughout phase 2 until it reaches a predetermined maximum value. Stem and root C also increase throughout phase 2 based on the carbon allocation algorithm in section {numref}`Leaf emergence to grain fill`. Note that, since all represented crops are herbaceous, their stem biomass is all is live stem. + +(grain fill)= + +#### Grain fill + +The grain fill phase (phase 3) begins in one of two ways. The first potential trigger is based on temperature, similar to phase 2. A variable tracked since planting, similar to $\gddaccsoil$ but for 2-m air temperature, $\gddacctwom$, must reach a heat unit threshold, $\huithreshgrain$, of 40 to 65% of $\gddthreshmat$ (see {numref}`Table Crop phenology parameters`). The second potential trigger for phase 3 is based on leaf area index. When the maximum value of leaf area index is reached in phase 2 ({numref}`Table Crop allocation parameters`), phase 3 begins. In phase 3, the leaf area index begins to decline in response to a background litterfall rate calculated as the inverse of leaf longevity for the PFT as done in the BGC part of the model. + +(harvest)= + +#### Harvest + +Harvest is assumed to occur as soon as the crop reaches maturity. When $\gddacctwom$ reaches 100% of $\gddthreshmat$ or the number of days past planting reaches a crop-specific maximum (`mxmat`; {numref}`Table Crop phenology parameters`), then the crop is harvested. Harvest occurs in one time step using the BGC leaf offset algorithm. + +(table crop phenology parameters)= + +```{list-table} Crop phenology and morphology parameters for the active crop plant functional types (PFTs) in CLM with managed crops on (BgcCrop component sets). Where there are two values in a cell, they refer to the rainfed and irrigated functional types, respectively. {{paramfile_disclaimer_md}} +:header-rows: 1 + +* - Parameter + - Variable + - temperate corn + - spring wheat + - temperate soybean + - cotton + - rice + - sugarcane + - tropical corn + - tropical soybean + - _Miscanthus_ + - switchgrass +* - IVT + - n/a + - 17, 18 + - 19, 20 + - 23, 24 + - 41, 42 + - 61, 62 + - 67, 68 + - 75, 76 + - 77, 78 + - 71, 72 + - 73, 74 +* - $\paramplantingtemp$ (K) + - `planting_temp` + - 283.15 + - 280.15 + - 286.15 + - 294.15 + - 294.15 + - 294.15 + - 294.15 + - 294.15 + - 283.15 + - 283.15 +* - $\paramminplantingtemp$ (K) + - `min_planting_temp` + - 279.15 + - 272.15 + - 279.15 + - 283.15 + - 283.15 + - 283.15 + - 283.15 + - 283.15 + - 279.15 + - 279.15 +* - $\paramgddmin$ (degree-days) + - `gddmin` + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 +* - $\parambaset$ (°C) + - `baset` + - 8 + - 0 + - 10 + - 10 + - 10 + - 10 + - 10 + - 10 + - 8 + - 8 +* - $\huithreshlfemerg$ (% $\gddthreshmat$) + - `lfemerg` + - 3% + - 5% + - 3% + - 3% + - 1% + - 3% + - 3% + - 3% + - 3% + - 3% +* - $\huithreshgrain$ (% $\gddthreshmat$) + - `grnfill` + - 65% + - 60% + - 50% + - 50% + - 40% + - 65% + - 50% + - 50% + - 40% + - 40% +* - Max. growing season length + - `mxmat` + - 165 + - 150 + - 180 + - 225 + - 180 + - 360 + - 160 + - 150 + - 210 + - 210 +* - $\paramztopmx$ (m) + - `ztopmx` + - 2.5 + - 1.2 + - 0.75 + - 1.5 + - 1.8 + - 4 + - 2.5 + - 1 + - 2.5 + - 2.5 +* - SLA (m {sup}`2` leaf g {sup}`-1` C) + - `slatop` + - 0.05 + - 0.035 + - 0.035 + - 0.035 + - 0.035 + - 0.05 + - 0.05 + - 0.035 + - 0.035 + - 0.035 +``` + +Notes: + +- $\paramplantingtemp$ and $\paramminplantingtemp$ are crop-specific average and coldest planting temperatures, respectively. (See Sect. {numref}`Planting`.) +- $\paramgddmin$ is a threshold describing the coolest historical climate a patch can have had in order for a crop to be sown there; see Sect. {numref}`Planting` for details. +- $\parambaset$ is the minimum temperature for accumulating growing degree-days. +- $\huithreshlfemerg$ and $\huithreshgrain$ are, respectively, the threshold fractions of $\gddthreshmat$ a crop must reach to enter the leaf-emergence phase (phase 2) and grain-filling phase (phase 3). However, note the adjustment applied to $\huithreshgrain$ for some crops (Eq. {eq}`corn-huigrain-adjustment`). +- `mxmat` is the maximum growing season length (days past planting), at which harvest occurs even if heat unit index has not reached $\gddthreshmat$. +- $\paramztopmx$ is the maximum top-of-canopy height of a crop (see Sect. {numref}`Vegetation Structure`). +- SLA is specific leaf area (see Chapter {numref}`rst_Photosynthetic Capacity`). + +(allocation)= + +### Allocation + +Allocation changes based on the crop phenology phase (section {numref}`Phenology`). Simulated C assimilation begins every year upon leaf emergence in phase 2 and ends with harvest at the end of phase 3; therefore, so does the allocation of such C to the crop's leaf, stem, fine root, and reproductive pools. + +Typically, C:N ratios in plant tissue vary throughout the growing season and tend to be lower during early growth stages and higher in later growth stages. In order to account for this seasonal change, two sets of C:N ratios are established in CLM for the leaf, stem, and fine root of crops: one during the leaf emergence phase (phenology phase 2), and a second during grain fill phase (phenology phase 3). This modified C:N ratio approach accounts for the nitrogen retranslocation that occurs during the grain fill phase (phase 3) of crop growth. Leaf, stem, and root C:N ratios for phase 2 are calculated using the standard CLM carbon and nitrogen allocation scheme (Chapter {numref}`rst_CN Allocation`), which provides a target C:N value ({numref}`Table Crop allocation parameters`) and allows C:N to vary through time. During grain fill (phase 3) of the crop growth cycle, a portion of the nitrogen in the plant tissues is moved to a storage pool to fulfill nitrogen demands of organ (reproductive pool) development, such that the resulting C:N ratio of the plant tissue is reflective of measurements at harvest. All C:N ratios were determined by calibration process, through comparisons of model output versus observations of plant carbon throughout the growing season. + +Carbon supply for excess maintenance respiration (Chapter {numref}`rst_Plant Respiration`) cannot continue to happen after harvest for annual crops, so at harvest the excess respiration pool is turned into a flux that extracts CO{sub}`2` directly from the atmosphere. This way any excess maintenance respiration remaining at harvest is eliminated as if such respiration had not taken place. + +(leaf emergence to grain fill)= + +#### Leaf emergence + +During phase 2, the allocation coefficients (fraction of available C) to +each C pool are defined as: + +$$ +\begin{aligned} +a_{repr} &= 0 \\ +a_{froot} &= \paramarooti -(\paramarooti - \paramarootf ) \times {\rm min}\left(\frac{\gddacctwom }{\gddthreshmat }, 1\right) \\ +a_{leaf} &= (1-a_{froot} ) \times \frac{\paramfleafi (e^{-b} -e^{-b\frac{\gddacctwom }{\gddthreshgrain} } )}{e^{-b} -1} {\rm \; \; \; where\; \; \; }b=0.1 \\ +a_{stem} &= 1-a_{repr} - a_{froot} - a_{leaf} +\end{aligned} +$$ (eq-lfemerg-allocations) + +where $\paramfleafi$, $\paramarooti$, and $\paramarootf$ are initial and final values of these coefficients (respectively parameters `fleafi`, `arooti`, and `arootf`), and $\gddthreshgrain$ is the growing degree-day threshold to enter the grain-filling phase. + +For most crops, $\gddthreshgrain$ is equal to $\gddthreshmat$ times the PFT parameter $\huithreshgrain$ (`grnfill`). However, for corn, sugarcane, _Miscanthus_, and switchgrass, an adjustment is applied ({ref}`Kucharik 2003 `; C.J. Kucharik, pers. comm.): + +$$ +\begin{aligned} +c &= \max \left( 73,\ \min \left[135, \frac{\gddthreshmat + 53.683}{13.882} \right] \right) \\ +\huithreshgrainactual &= \min \left( \max \left[ -0.002 * \left( c - 73 \right) + \huithreshgrain,\ \huithreshgrain-0.1 \right],\ \huithreshgrain \right) \\ +\gddthreshgrain &= \gddthreshmat \times \huithreshgrainactual +\end{aligned} +$$ (corn-huigrain-adjustment) + +At a crop-specific maximum leaf area index, $\paramlaimx$, carbon allocation is directed almost exclusively to fine roots, with only 0.001% of carbon going to leaves. See {numref}`Table Crop allocation parameters` for parameter values. + +(grain fill to harvest)= + +#### Grain fill + +The calculation of $a_{froot}$ remains the same from phase 2 (Eq. [](#eq-lfemerg-allocations)) to phase 3. During grain fill (phase 3), leaf and stem allocation coefficients change to: + +$$ +\begin{aligned} +x &= \min \left[1,\ \fracthrugrnfill \right] \\ +a_{organ} &= \min \left( a_{organ}^{i,3},\ \max \left[ \paramaorganf,\ a_{organ} \left( 1 - x \right)^{\paramallconso} \right] \right) +\end{aligned} +$$ (alloc-grnfill-leafstem) + +where $\textrm{organ}$ is either $\textrm{leaf}$ or $\textrm{stem}$, $a_{organ}^{i,3}$ (initial values) equals the last $a_{organ}$ calculated in phase 2, $\paramdeclfact$ and $\paramallconso$ are allocation decline factors, $\paramaorganf$ is the parameterized value of these allocation coefficients at maturity, and $\huithreshgrain$ is the heat unit threshold to enter the grain-filling phase. (See {numref}`Table Crop allocation parameters` for parameter values.) + +As in the leaf-emergence phase (Sect {numref}`leaf emergence to grain fill`), at a crop-specific maximum leaf area index, $\paramlaimx$, leaf allocation is reduced to 0.001%. + +After allocation to fine roots, leaves, and stem, the rest of the carbon goes to the reproductive pool: + +$$ +a_{repr} =1-a_{froot} -a_{stem} -a_{leaf} +$$ (alloc-grnfill-repr) + +(nitrogen-retranslocation-for-crops)= + +#### Nitrogen retranslocation for crops + +Nitrogen retranslocation in crops occurs when nitrogen that was used for tissue growth of leaves, stems, and fine roots during the early growth season is remobilized and used for grain development ({ref}`Pollmer et al. 1979 `, {ref}`Crawford et al. 1982 `, {ref}`Simpson et al. 1983 `, {ref}`Ta and Weiland 1992 `, {ref}`Barbottin et al. 2005 `, {ref}`Gallais et al. 2006 `, {ref}`Gallais et al. 2007 `). Nitrogen allocation for crops follows that of natural vegetation, is supplied in CLM by the soil mineral nitrogen pool, and depends on C:N ratios for leaves, stems, roots, and organs. Nitrogen demand during organ development is fulfilled through retranslocation from leaves, stems, and roots. Nitrogen retranslocation is initiated at the beginning of the grain fill stage for all crops except soybean, for which retranslocation is after LAI decline. Nitrogen stored in the leaf and stem is moved into a storage retranslocation pool for all crops, and for wheat and rice, nitrogen in roots is also released into the retranslocation storage pool. The quantity of nitrogen mobilized from an organ to the retranslocation pool is calculated as + +$$ +\norgan - \frac{\corgan}{\paramforgancn} +$$ (n-from-organ-to-retrans-pool) + +where $\corgan$ and $\norgan$ are the carbon and nitrogen in the organ, respectively, and $\paramforgancn$ is the C:N ratio of the organ ({numref}`Table Crop allocation parameters`). Since C:N measurements are often taken from mature crops, pre-grain development C:N ratios for leaves, stems, and roots in the model are optimized to allow maximum nitrogen accumulation for later use during organ development, and post-grain fill C:N ratios are assigned the same as crop residue. After nitrogen is moved into the retranslocated pool, the nitrogen in this pool is used to meet plant nitrogen demand by assigning the available nitrogen from the retranslocated pool equal to the plant nitrogen demand for each organ ($\paramforgancn$ in {numref}`Table Crop allocation parameters`). Once the retranslocation pool is depleted, soil mineral nitrogen pool is used to fulfill plant nitrogen demands. + +(harvest to food and seed)= + +#### Harvest + +CLM splits live crop grain C and N between "food" and "seed" pools. In the former, C and N decay to the atmosphere over approximately one year. Specifically, $E = P^k$, where $E$ is the emissions in one timestep, $P$ is the product pool before emissions, and $k$ is a decay constant set to 0.0001296. The seed pool is used in the subsequent year to account for the C and N required for crop seeding. + +Live leaf and stem biomass at harvest is transferred to biofuel, removed residue, and/or litter pools. The harvested biofuel and removed residue pools (together, the "crop product" pool) emit their C and N to the atmosphere with a turnover time of approximately one year (as for the food pool above). + +For the biofuel crops, _Miscanthus_ and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter `biofuel_harvfrac` (0-1). + +50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter `crop_residue_removal_frac` (0–1). The default 50% is derived from {ref}`Smerald et al. 2023 `, who found a global average of 50% of residues left on the field. This includes residues burned in the field, meaning that our implementation implictly assumes the CLM crop burning representation will handle those residues appropriately. + +The following equations illustrate how this works. Subscript $p$ refers to either the leaf or stem biomass pool. + +$$ +CF_{p,biofuel} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} + \right) \times \parambiofuelharvfrac +$$ (25.9) + +$$ +CF_{p,removed\_residue} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} + \right) \times (1 - \parambiofuelharvfrac) \times \paramcropresidueremovalfrac +$$ (harv_c_to_removed_residue) + +$$ +CF_{p,litter} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} + \right) \times \left( 1-\parambiofuelharvfrac \right) \times \left( 1-\paramcropresidueremovalfrac \right) +CF_{p,alloc} +$$ (25.11) + +with corresponding nitrogen fluxes: + +$$ +NF_{p,biofuel} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} + \right) \times \parambiofuelharvfrac +$$ (25.12) + +$$ +NF_{p,removed\_residue} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} + \right) \times \left( 1 - \parambiofuelharvfrac \right) \times \paramcropresidueremovalfrac +$$ (harv_n_to_removed_residue) + +$$ +NF_{p,litter} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} + \right) \times \left( 1-\parambiofuelharvfrac \right) \times \left( 1-\paramcropresidueremovalfrac \right) +$$ (25.14) + +where CF is the carbon flux, CS is stored carbon, NF is the nitrogen flux, NS is stored nitrogen, and `biofuel_harvfrac` is the harvested fraction of leaf/stem for biofuel feedstocks. + +Annual food crop yields (g dry matter m{sup}`-2`) can be calculated by saving the `GRAINC_TO_FOOD_ANN` variable once per year, then postprocessing with Equation {eq}`25.15`. This calculation assumes that grain C is 45% of the total dry weight. Additionally, harvest is not typically 100% efficient, so analysis needs to assume that harvest efficiency is less---we use 85%. + +$$ +\text{Grain yield} = \frac{\texttt{GRAINC_TO_FOOD_ANN} \times 0.85}{0.45} +$$ (25.15) + +(table crop allocation parameters)= + +```{list-table} Crop allocation parameters for the active crop plant functional types (PFTs) in CLM with managed crops on (BgcCrop component sets). {{paramfile_disclaimer_md}} +:header-rows: 1 + +* - Parameter + - Variable + - temperate corn + - spring wheat + - temperate soybean + - cotton + - rice + - sugarcane + - tropical corn + - tropical soybean + - _Miscanthus_ + - switchgrass +* - IVT + - n/a + - 17, 18 + - 19, 20 + - 23, 24 + - 41, 42 + - 61, 62 + - 67, 68 + - 75, 76 + - 77, 78 + - 71, 72 + - 73, 74 +* - $\paramfleafi$ + - `fleafi` + - 0.75 + - 0.9 + - 0.85 + - 0.85 + - 0.75 + - 0.6 + - 0.6 + - 0.85 + - 0.9 + - 0.7 +* - $\paramlaimx$ (m{sup}`2` m{sup}`-2`) + - `laimx` + - 6.2 + - 7 + - 6 + - 6 + - 7 + - 5 + - 5 + - 6 + - 10 + - 6.5 +* - $\paramarooti$ + - `arooti` + - 0.1 + - 0.3 + - 0.2 + - 0.2 + - 0.3 + - 0.3 + - 0.1 + - 0.2 + - 0.11 + - 0.14 +* - $\paramarootf$ + - `arootf` + - 0.05 + - 0 + - 0.2 + - 0.2 + - 0 + - 0.1 + - 0.05 + - 0.2 + - 0.09 + - 0.09 +* - $\paramastemf$ + - `astemf` + - 0 + - 0.05 + - 0.3 + - 0.3 + - 0.05 + - 0 + - 0 + - 0.3 + - 0 + - 0 +* - $\paramallconss$ + - `allconss` + - 2 + - 1 + - 2 + - 5 + - 1 + - 2 + - 2 + - 5 + - 2 + - 2 +* - $\paramallconsl$ + - `allconsl` + - 5 + - 3 + - 2 + - 2 + - 3 + - 5 + - 5 + - 2 + - 5 + - 5 +* - $\paramleafcn$ + - `leafcn` + - 25 + - 20 + - 20 + - 20 + - 20 + - 25 + - 25 + - 20 + - 20 + - 20 +* - $\paramlivewdcn$ + - `livewdcn` + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 +* - $\paramfrootcn$ + - `frootcn` + - 42 + - 42 + - 42 + - 42 + - 42 + - 42 + - 42 + - 42 + - 42 + - 42 +* - $\paramfleafcn$ + - `fleafcn` + - 65 + - 65 + - 65 + - 65 + - 65 + - 65 + - 65 + - 65 + - 65 + - 65 +* - $\paramfstemcn$ + - `fstemcn` + - 120 + - 100 + - 130 + - 130 + - 100 + - 120 + - 120 + - 130 + - 120 + - 120 +* - $\paramffrootcn$ + - `ffrootcn` + - 0 + - 40 + - 0 + - 0 + - 40 + - 0 + - 0 + - 0 + - 0 + - 0 +* - $\paramgraincn$ + - `graincn` + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 + - 50 +``` +- $\paramaleaff$ (parameter `aleaff`) is zero for all crops. +- $\paramdeclfact$ (parameter `declfact`) is 1.05 for all crops. + +(other-features)= + +### Other Features + +(physical-crop-characteristics)= + +#### Physical Crop Characteristics + +Leaf area index ($L$) is calculated as a function of specific leaf area (SLA, {numref}`Table Crop phenology parameters`) and leaf C. Stem area index ($S$) is equal to 0.1$L$ for temperate and tropical corn, sugarcane, switchgrass, and _Miscanthus_ and 0.2$L$ for other crops, as in AgroIBIS. All live C and N pools go to 0 after crop harvest, but the $S$ is kept at 0.25 to simulate a post-harvest "stubble" on the ground. + +Crop heights at the top and bottom of the canopy, $\paramztopmx$ and ${z}_{bot}$ (m), come from the AgroIBIS formulation: + +$$ +\begin{array}{l} +{z_{top} = \paramztopmx \left(\frac{L}{\paramlaimx -1} \right)^{2} \ge 0.05{\rm \; where\; }\frac{L}{\paramlaimx -1} \le 1} \\ +{z_{bot} =0.02{\rm m}} +\end{array} +$$ (25.16) + +where $\paramztopmx$ is the maximum top-of-canopy height of the crop ({numref}`Table Crop phenology parameters`) and $\paramlaimx$ is the maximum leaf area index ({numref}`Table Crop allocation parameters`). + +(interactive-fertilization)= + +#### Interactive Fertilization + +CLM simulates fertilization by adding nitrogen directly to the soil mineral nitrogen pool to meet crop nitrogen demands using both industrial fertilizer and manure application. CLM's separate crop land unit ensures that natural vegetation will not access the fertilizer applied to crops. Fertilizer in CLM is prescribed by crop functional types and varies spatially for each year based on the LUMIP land use and land cover change time series (LUH2 for historical and SSPs for future) ({ref}`Lawrence et al. 2016 `). One of two fields is used to prescribe industrial fertilizer based on the type of simulation. For non-transient simulations, annual fertilizer application in g N/m{sup}`2`/yr is specified on the land surface data set by the field `CONST_FERTNITRO_CFT`. In transient simulations, annual fertilizer application is specified on the land use time series file by the field `FERTNITRO_CFT`, which is also in g N/m{sup}`2`/yr. The values for both of these fields come from the LUMIP time series for each year. In addition to the industrial fertilizer, background manure fertilizer is specified on the parameter file by the field `manunitro`. For perennial bioenergy crops, little fertilizer (56kg/ha/yr) is applied to switchgrass and no fertilizer is applied to _Miscanthus_. Note these rates are only based on local land management practices at the University of Illinois Energy Farm located in Central Midwestern United States {ref}`(Cheng et al., 2019)` rather than the LUMIP timeseries. Manure N is applied at a rate of 0.002 kg N/m{sup}`2`/yr. Because previous versions of CLM (e.g., CLM4) had rapid denitrification rates, fertilizer is applied slowly to minimize N loss (primarily through denitrification) and maximize plant uptake. The current implementation of CLM inherits this legacy, although denitrification rates are slower in the current version of the model ({ref}`Koven et al. 2013 `). As such, fertilizer application begins during the leaf emergence phase of crop development (phase 2) and continues for 20 days, which helps reduce large losses of nitrogen from leaching and denitrification during the early stage of crop development. The 20-day period is chosen as an optimization to limit fertilizer application to the emergence stage. A fertilizer counter in seconds, $f$, is set as soon as the leaf emergence phase for crops initiates: + +$$ +f = n \times 86400 +$$ (25.17) + +where $n$ is set to 20 fertilizer application days and 86400 is the number of seconds per day. When the crop enters phase 2 (leaf emergence) of its growth cycle, fertilizer application begins by initializing fertilizer amount to the total fertilizer at each column within the grid cell divided by the initialized $f$. Fertilizer is applied and $f$ is decremented each time step until a zero balance on the counter is reached. + +(biological-nitrogen-fixation-for-soybeans)= + +#### Biological nitrogen fixation for soybeans + +Biological N fixation for soybeans is calculated by the fixation and uptake of nitrogen module (Chapter {numref}`rst_FUN`) and is the same as N fixation in natural vegetation. Unlike natural vegetation, where a fraction of each PFT are N fixers, all soybeans are treated as N fixers. + +(latitude-vary-base-temperature-for-growing-degree-days)= + +#### Latitudinal variation in base growth temperature + +For most crops, $\gddacctwom$ (growing degree days since planting) is the same in all locations. However, for spring wheat and sugarcane within 30° of the Equator, the calculation of $\gddacctwom$ allows for latitudinal variation: + +$$ +\parambaset^{\prime} = \parambaset + 12 - 0.4 \times \mid latitude \mid +$$ (25.18) + +where $\parambaset$ is the base temperature for GDD ({numref}`Table Crop phenology parameters`). Such latitudinal variation in base temperature could slow $\gddacctwom$ accumulation extend the growing season for regions within 30°S to 30°N for spring wheat and sugarcane. + +(separate-reproductive-pool)= + +#### Separate reproductive pool + +One notable difference between natural vegetation and crops is the presence of reproductive carbon and nitrogen pools. Accounting for the reproductive pools helps determine whether crops are performing reasonably through yield calculations. The reproductive pool is maintained similarly to the leaf, stem, and fine root pools, but allocation of carbon and nitrogen does not begin until the grain fill stage of crop development. Equation {eq}`alloc-grnfill-leafstem` describes the carbon and nitrogen allocation coefficients to the reproductive pool. In CLM, as allocation declines in stem, leaf, and root pools (see section {numref}`Grain fill to harvest`) during the grain fill stage of growth, increasing amounts of carbon and nitrogen are available for grain development. + +(tillage)= + +#### Tillage + +Tillage is represented as an enhancement of the decomposition rate coefficient; see section {numref}`decomp_mgmt_modifiers`. + +(the-irrigation-model)= + +## The irrigation model + +CLM includes the option to irrigate cropland areas that are equipped for irrigation. The application of irrigation responds dynamically to the simulated soil moisture conditions. This irrigation algorithm is based loosely on the implementation of {ref}`Ozdogan et al. (2010) `. + +When irrigation is enabled, the crop areas of each grid cell are divided into irrigated and rainfed fractions according to a dataset of areas equipped for irrigation ({ref}`Portmann et al. 2010 `). Irrigated and rainfed crops are placed on separate soil columns, so that irrigation is only applied to the soil beneath irrigated crops. + +In irrigated croplands, a check is made once per day to determine whether irrigation is required on that day. This check is made in the first time step after 6 AM local time. Irrigation is required if crop leaf area $>$ 0, and the available soil water is below a specified threshold. + +The soil moisture deficit $D_{irrig}$ is + +$$ +D_{irrig} = \left\{ +\begin{array}{lr} +w_{target} - w_{avail} &\qquad w_{thresh} > w_{avail} \\ +0 &\qquad w_{thresh} \le w_{avail} +\end{array} \right\} +$$ (25.61) + +where $w_{target}$ is the irrigation target soil moisture (mm) + +$$ +w_{target} = \sum_{j=1}^{N_{irr}} \theta_{target} \Delta z_{j} \ . +$$ (25.62) + +The irrigation moisture threshold (mm) is + +$$ +w_{thresh} = f_{thresh} \left(w_{target} - w_{wilt}\right) + w_{wilt} +$$ (25.63) + +where $w_{wilt}$ is the wilting point soil moisture (mm) + +$$ +w_{wilt} = \sum_{j=1}^{N_{irr}} \theta_{wilt} \Delta z_{j} \ , +$$ (25.64) + +and $f_{thresh}$ is a tuning parameter. The available moisture in the soil (mm) is + +$$ +w_{avail} = \sum_{j=1}^{N_{irr}} \theta_{j} \Delta z_{j} \ , +$$ (25.65) + +Note that $w_{target}$ is truly supposed to give the target soil moisture value that we're shooting for whenever irrigation happens; then the soil moisture deficit $D_{irrig}$ gives the difference between this target value and the current soil moisture. The irrigation moisture threshold $w_{thresh}$, on the other hand, gives a threshold at which we decide to do any irrigation at all. The way this is written allows for the possibility that one may not want to irrigate every time there becomes even a tiny soil moisture deficit. Instead, one may want to wait until the deficit is larger before initiating irrigation; at that point, one doesn't want to just irrigate up to the "threshold" but instead up to the higher "target". The target should always be greater than or equal to the threshold. + +$N_{irr}$ is the index of the soil layer corresponding to a specified depth $z_{irrig}$ ({numref}`Table Irrigation parameters`) and $\Delta z_{j}$ is the thickness of the soil layer in layer $j$ (section {numref}`Vertical Discretization`). $\theta_{j}$ is the volumetric soil moisture in layer $j$ (section {numref}`Soil Water`). $\theta_{target}$ and $\theta_{wilt}$ are the target and wilting point volumetric soil moisture values, respectively, and are determined by inverting {eq}`7.94` using soil matric potential parameters $\Psi_{target}$ and $\Psi_{wilt}$ ({numref}`Table Irrigation parameters`). After the soil moisture deficit $D_{irrig}$ is calculated, irrigation in an amount equal to $\frac{D_{irrig}}{T_{irrig}}$ (mm/s) is applied uniformly over the irrigation period $T_{irrig}$ (s). Irrigation water is applied directly to the ground surface, bypassing canopy interception (i.e., added to ${q}_{grnd,liq}$: section {numref}`Canopy Water`). + +To conserve mass, irrigation is removed from river water storage (Chapter {numref}`rst_MOSART`). When river water storage is inadequate to meet irrigation demand, there are two options: 1) the additional water can be removed from the ocean model, or 2) the irrigation demand can be reduced such that river water storage is maintained above a specified threshold. + +(table irrigation parameters)= + +```{list-table} Irrigation parameters. Values in the "Parameter" column are the associated namelist variable, if the value is not hard-coded in the model code. {{nonparamfile_disclaimer_md}} +:header-rows: 1 + +* - Symbol + - Parameter + - Units + - Value +* - $f_{thresh}$ + - `irrig_threshold_fraction` + - Unitless + - 1.0 +* - $z_{irrig}$ + - `irrig_depth` + - m + - 0.6 +* - $\Psi_{target}$ + - `irrig_target_smp` + - mm + - -3400 +* - $\Psi_{wilt}$ + - (Hard-coded) + - mm + - -150000 +``` + +% add a reference to surface data in chapter2 +% To accomplish this we downloaded data of percent irrigated and percent rainfed corn, soybean, and temperate cereals (wheat, barley, and rye) (:ref:`Portmann et al. 2010 `), available online from *ftp://ftp.rz.uni-frankfurt.de/pub/uni-frankfurt/physische\_geographie/hydrologie/public/data/MIRCA2000/harvested\_area\_grids.* diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst deleted file mode 100755 index ec43a50cd8..0000000000 --- a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst +++ /dev/null @@ -1,742 +0,0 @@ -.. _rst_Crops and Irrigation: - -Crops and Irrigation -==================== - -.. _Summary of CLM5.0 updates relative to the CLM4.5: - -Summary of CLM5.0 updates relative to the CLM4.5 ------------------------------------------------- - -We describe here the complete crop and irrigation parameterizations that appear in CLM5.0. Corresponding information for CLM4.5 appeared in the CLM4.5 Technical Note (:ref:`Oleson et al. 2013 `). - -CLM5.0 includes the following new updates to the CROP option, where CROP refers to the interactive crop management model and is included as an option with the BGC configuration: - -- New crop functional types - -- All crop areas are actively managed - -- Fertilization rates updated based on crop type and geographic region - -- New Irrigation triggers - -- Phenological triggers vary by latitude for some crop types - -- Ability to simulate transient crop management - -- Adjustments to allocation and phenological parameters - -- Crops reaching their maximum LAI triggers the grain fill phase - -- Grain C and N pools are included in a 1-year product pool - -- C for annual crop seeding comes from the grain C pool - -- Initial seed C for planting is increased from 1 to 3 g C/m^2 - -These updates appear in detail in the sections below. Many also appear in :ref:`Levis et al. (2016) `. - -Available new features since the CLM5 release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Addition of bioenergy crops -- Ability to customize crop calendars (sowing windows/dates, maturity requirements) using stream files -- Cropland soil tillage -- Crop residue removal - -.. _The crop model: - -The crop model: cash and bioenergy crops ----------------------------------------- - -Introduction -^^^^^^^^^^^^ - -Groups developing Earth System Models generally account for the human footprint on the landscape in simulations of historical and future climates. Traditionally we have represented this footprint with natural vegetation types and particularly grasses because they resemble many common crops. Most modeling efforts have not incorporated more explicit representations of land management such as crop type, planting, harvesting, tillage, fertilization, and irrigation, because global scale datasets of these factors have lagged behind vegetation mapping. As this begins to change, we increasingly find models that will simulate the biogeophysical and biogeochemical effects not only of natural but also human-managed land cover. - -AgroIBIS is a state-of-the-art land surface model with options to simulate dynamic vegetation (:ref:`Kucharik et al. 2000 `) and interactive crop management (:ref:`Kucharik and Brye 2003 `). The interactive crop management parameterizations from AgroIBIS (March 2003 version) were coupled as a proof-of-concept to the Community Land Model version 3 [CLM3.0, :ref:`Oleson et al. (2004) ` ] (not published), then coupled to the CLM3.5 (:ref:`Levis et al. 2009 `) and later released to the community with CLM4CN (:ref:`Levis et al. 2012 `), and CLM4.5BGC. Additional updates after the release of CLM4.5 were available by request (:ref:`Levis et al. 2016 `), and those are now incorporated into CLM5. - -With interactive crop management and, therefore, a more accurate representation of agricultural landscapes, we hope to improve the CLM's simulated biogeophysics and biogeochemistry. These advances may improve fully coupled simulations with the Community Earth System Model (CESM), while helping human societies answer questions about changing food, energy, and water resources in response to climate, environmental, land use, and land management change (e.g., :ref:`Kucharik and Brye 2003 `; :ref:`Lobell et al. 2006 `). As implemented here, the crop model uses the same physiology as the natural vegetation but with uses different crop-specific parameter values, phenology, and allocation, as well as fertilizer and irrigation management. - -.. _Crop plant functional types: - -Crop plant functional types -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To allow crops to coexist with natural vegetation in a grid cell, the vegetated land unit is separated into a naturally vegetated land unit and a managed crop land unit. Unlike the plant functional types (PFTs) in the naturally vegetated land unit, the managed crop PFTs in the managed crop land unit do not share soil columns and thus permit for differences in the land management between crops. Each crop type has a rainfed and an irrigated PFT that are on independent soil columns. Crop grid cell coverage is assigned from satellite data (similar to all natural PFTs), and the managed crop type proportions within the crop area is based on the dataset created by :ref:`Portmann et al. (2010)` for present day. New in CLM5, crop area is extrapolated through time using the dataset provided by Land Use Model Intercomparison Project (LUMIP), which is part of CMIP6 Land use timeseries (:ref:`Lawrence et al. 2016 `). For more details about how crop distributions are determined, see Chapter :numref:`rst_Transient Landcover Change`. - -CLM5 includes ten actively managed crop types (temperate soybean, tropical soybean, temperate corn, tropical corn, spring wheat, cotton, rice, sugarcane, miscanthus, and switchgrass) that are chosen based on the availability of corresponding algorithms in AgroIBIS and as developed by :ref:`Badger and Dirmeyer (2015)` and described by :ref:`Levis et al. (2016)`, or from available observations as described by :ref:`Cheng et al. (2019)`. The representations of sugarcane, rice, cotton, tropical corn, and tropical soy were new in CLM5; miscanthus and switchgrass were added after the CLM5 release. Sugarcane and tropical corn are both C4 plants and are therefore represented using the temperate corn functional form. Tropical soybean uses the temperate soybean functional form, while rice and cotton use the wheat functional form. In tropical regions, parameter values were developed for the Amazon Basin, and planting date window is shifted by six months relative to the Northern Hemisphere. Plantation areas of bioenergy crops are projected to expand throughout the 21st century as a major energy source to replace fossil fuels and mitigate climate change. Miscanthus and switchgrass are perennial bioenergy crops and have quite different physiological traits and land management practices than annual crops, such as longer growing seasons, higher productivity, and lower demands for nutrients and water. About 70% of biofuel aboveground biomass (leaf & livestem) is removed at harvest. Parameter values were developed by using observation data collected at the University of Illinois Energy Farm located in Central Midwestern United States (:ref:`Cheng et al., 2019`). - -In addition, CLM's default list of plant functional types (PFTs) includes an irrigated and unirrigated unmanaged C3 crop (:numref:`Table Crop plant functional types`) treated as a second C3 grass. The unmanaged C3 crop is only used when the crop model is not active and has grid cell coverage assigned from satellite data, and the unmanaged C3 irrigated crop type is currently not used since irrigation requires the crop model to be active. The default list of PFTs also includes twenty-one inactive crop PFTs that do not yet have associated parameters required for active management. Each of the inactive crop types is simulated using the parameters of the spatially closest associated crop type that is most similar to the functional type (e.g., C3 or C4), which is required to maintain similar phenological parameters based on temperature thresholds. Information detailing which parameters are used for each crop type is included in :numref:`Table Crop plant functional types`. It should be noted that PFT-level history output merges all crop types into the actively managed crop type, so analysis of crop-specific output will require use of the land surface dataset to remap the yields of each actively and inactively managed crop type. Otherwise, the actively managed crop type will include yields for that crop type and all inactively managed crop types that are using the same parameter set. - -.. _Table Crop plant functional types: - -.. table:: Crop plant functional types (PFTs) included in CLM5BGCCROP. - - === =========================== ================ =========================== - IVT Plant function types (PFTs) Management Class Crop Parameters Used - === =========================== ================ =========================== - 15 c3 unmanaged rainfed crop none not applicable - 16 c3 unmanaged irrigated crop none not applicable - 17 rainfed temperate corn active rainfed temperate corn - 18 irrigated temperate corn active irrigated temperate corn - 19 rainfed spring wheat active rainfed spring wheat - 20 irrigated spring wheat active irrigated spring wheat - 21 rainfed winter wheat inactive rainfed spring wheat - 22 irrigated winter wheat inactive irrigated spring wheat - 23 rainfed temperate soybean active rainfed temperate soybean - 24 irrigated temperate soybean active irrigated temperate soybean - 25 rainfed barley inactive rainfed spring wheat - 26 irrigated barley inactive irrigated spring wheat - 27 rainfed winter barley inactive rainfed spring wheat - 28 irrigated winter barley inactive irrigated spring wheat - 29 rainfed rye inactive rainfed spring wheat - 30 irrigated rye inactive irrigated spring wheat - 31 rainfed winter rye inactive rainfed spring wheat - 32 irrigated winter rye inactive irrigated spring wheat - 33 rainfed cassava inactive rainfed rice - 34 irrigated cassava inactive irrigated rice - 35 rainfed citrus inactive rainfed spring wheat - 36 irrigated citrus inactive irrigated spring wheat - 37 rainfed cocoa inactive rainfed rice - 38 irrigated cocoa inactive irrigated rice - 39 rainfed coffee inactive rainfed rice - 40 irrigated coffee inactive irrigated rice - 41 rainfed cotton active rainfed cotton - 42 irrigated cotton active irrigated cotton - 43 rainfed datepalm inactive rainfed cotton - 44 irrigated datepalm inactive irrigated cotton - 45 rainfed foddergrass inactive rainfed spring wheat - 46 irrigated foddergrass inactive irrigated spring wheat - 47 rainfed grapes inactive rainfed spring wheat - 48 irrigated grapes inactive irrigated spring wheat - 49 rainfed groundnuts inactive rainfed rice - 50 irrigated groundnuts inactive irrigated rice - 51 rainfed millet inactive rainfed tropical corn - 52 irrigated millet inactive irrigated tropical corn - 53 rainfed oilpalm inactive rainfed rice - 54 irrigated oilpalm inactive irrigated rice - 55 rainfed potatoes inactive rainfed spring wheat - 56 irrigated potatoes inactive irrigated spring wheat - 57 rainfed pulses inactive rainfed spring wheat - 58 irrigated pulses inactive irrigated spring wheat - 59 rainfed rapeseed inactive rainfed spring wheat - 60 irrigated rapeseed inactive irrigated spring wheat - 61 rainfed rice active rainfed rice - 62 irrigated rice active irrigated rice - 63 rainfed sorghum inactive rainfed tropical corn - 64 irrigated sorghum inactive irrigated tropical corn - 65 rainfed sugarbeet inactive rainfed spring wheat - 66 irrigated sugarbeet inactive irrigated spring wheat - 67 rainfed sugarcane active rainfed sugarcane - 68 irrigated sugarcane active irrigated sugarcane - 69 rainfed sunflower inactive rainfed spring wheat - 70 irrigated sunflower inactive irrigated spring wheat - 71 rainfed miscanthus active rainfed miscanthus - 72 irrigated miscanthus active irrigated miscanthus - 73 rainfed switchgrass active rainfed switchgrass - 74 irrigated switchgrass active irrigated switchgrass - 75 rainfed tropical corn active rainfed tropical corn - 76 irrigated tropical corn active irrigated tropical corn - 77 rainfed tropical soybean active rainfed tropical soybean - 78 irrigated tropical soybean active irrigated tropical soybean - === =========================== ================ =========================== - -.. _Phenology: - -Phenology -^^^^^^^^^ - -CLM5-BGC includes evergreen, seasonally deciduous (responding to changes in day length), and stress deciduous (responding to changes in temperature and/or soil moisture) phenology algorithms (Chapter :numref:`rst_Vegetation Phenology and Turnover`). CLM5-BGC-crop uses the AgroIBIS crop phenology algorithm, consisting of three distinct phases. - -Phase 1 starts at planting and ends with leaf emergence, phase 2 continues from leaf emergence to the beginning of grain fill, and phase 3 starts from the beginning of grain fill and ends with physiological maturity and harvest. - -.. _Planting: - -Planting -'''''''' - -All crops must meet the following requirements between the minimum planting date and the maximum planting date (for the northern hemisphere) in :numref:`Table Crop phenology parameters`: - -.. math:: - :label: 25.1 - - \begin{array}{c} - {T_{10d} >T_{p} } \\ - {T_{10d}^{\min } >T_{p}^{\min } } \\ - {GDD_{8} \ge GDD_{\min } } - \end{array} - -where :math:`{T}_{10d}` is the 10-day running mean of :math:`{T}_{2m}`, (the simulated 2-m air temperature during each model time step) and :math:`T_{10d}^{\min}` is the 10-day running mean of :math:`T_{2m}^{\min }` (the daily minimum of :math:`{T}_{2m}`). :math:`{T}_{p}` and :math:`T_{p}^{\min }` are crop-specific coldest planting temperatures (:numref:`Table Crop phenology parameters`), :math:`{GDD}_{8}` is the 20-year running mean growing degree-days (units are °C day) tracked from April through September (NH) above 8°C with maximum daily increments of 30 degree-days (see equation :eq:`25.3`), and :math:`{GDD}_{min }`\ is the minimum growing degree day requirement (:numref:`Table Crop phenology parameters`). :math:`{GDD}_{8}` does not change as quickly as :math:`{T}_{10d}` and :math:`T_{10d}^{\min }`, so it determines whether it is warm enough for the crop to be planted in a grid cell, while the 2-m air temperature variables determine the day when the crop may be planted if the :math:`{GDD}_{8}` threshold is met. If the requirements in equation :eq:`25.1` are not met by the maximum planting date, crops are still planted on the maximum planting date as long as :math:`{GDD}_{8} > 0`. In the southern hemisphere (SH) the NH requirements apply 6 months later. - -At planting, each crop seed pool is assigned 3 gC m\ :sup:`-2` from its grain product pool. The seed carbon is transferred to the leaves upon leaf emergence. An equivalent amount of seed leaf N is assigned given the PFT's C to N ratio for leaves (:math:`{CN}_{leaf}` in :numref:`Table Crop allocation parameters`; this differs from AgroIBIS, which uses a seed leaf area index instead of seed C). The model updates the average growing degree-days necessary for the crop to reach vegetative and physiological maturity, :math:`{GDD}_{mat}`, according to the following AgroIBIS rules: - -.. math:: - :label: 25.2 - - \begin{array}{lll} - GDD_{{\rm mat}}^{{\rm corn,sugarcane}} =0.85 GDD_{{\rm 8}} & {\rm \; \; \; and\; \; \; }& 950 `, :ref:`Crawford et al. 1982 `, :ref:`Simpson et al. 1983 `, :ref:`Ta and Weiland 1992 `, :ref:`Barbottin et al. 2005 `, :ref:`Gallais et al. 2006 `, :ref:`Gallais et al. 2007 `). Nitrogen allocation for crops follows that of natural vegetation, is supplied in CLM by the soil mineral nitrogen pool, and depends on C:N ratios for leaves, stems, roots, and organs. Nitrogen demand during organ development is fulfilled through retranslocation from leaves, stems, and roots. Nitrogen retranslocation is initiated at the beginning of the grain fill stage for all crops except soybean, for which retranslocation is after LAI decline. Nitrogen stored in the leaf and stem is moved into a storage retranslocation pool for all crops, and for wheat and rice, nitrogen in roots is also released into the retranslocation storage pool. The quantity of nitrogen mobilized depends on the C:N ratio of the plant tissue and is calculated as - -.. math:: - :label: 25.6 - - leaf\_ to\_ retransn=N_{leaf} -\frac{C_{leaf} }{CN_{leaf}^{f} } - -.. math:: - :label: 25.7 - - stemn\_ to\_ retransn=N_{stem} -\frac{C_{stem} }{CN_{stem}^{f} } - -.. math:: - :label: 25.8 - - frootn\_ to\_ retransn=N_{froot} -\frac{C_{froot} }{CN_{froot}^{f} } - -where :math:`{C}_{leaf}`, :math:`{C}_{stem}`, and :math:`{C}_{froot}` is the carbon in the plant leaf, stem, and fine root, respectively, :math:`{N}_{leaf}`, :math:`{N}_{stem}`, and :math:`{N}_{froot}` is the nitrogen in the plant leaf, stem, and fine root, respectively, and :math:`CN^f_{leaf}`, :math:`CN^f_{stem}`, and :math:`CN^f_{froot}` is the post-grain fill C:N ratio of the leaf, stem, and fine root respectively (:numref:`Table Crop allocation parameters`). Since C:N measurements are often taken from mature crops, pre-grain development C:N ratios for leaves, stems, and roots in the model are optimized to allow maximum nitrogen accumulation for later use during organ development, and post-grain fill C:N ratios are assigned the same as crop residue. After nitrogen is moved into the retranslocated pool, the nitrogen in this pool is used to meet plant nitrogen demand by assigning the available nitrogen from the retranslocated pool equal to the plant nitrogen demand for each organ (:math:`{CN_{[organ]}^{f} }` in :numref:`Table Crop allocation parameters`). Once the retranslocation pool is depleted, soil mineral nitrogen pool is used to fulfill plant nitrogen demands. - -.. _Harvest to food and seed: - -Harvest -''''''' - -Whereas live crop C and N in grain was formerly transferred to the litter pool upon harvest, CLM5 splits this between "food" and "seed" pools. In the former—more generally a "crop product" pool—C and N decay to the atmosphere over one year, similar to how the wood product pools work. The latter is used in the subsequent year to account for the C and N required for crop seeding. - -Live leaf and stem biomass at harvest is transferred to biofuel, removed residue, and/or litter pools. - -For the biofuel crops Miscanthus and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool as described for "food" harvest above. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter :math:`biofuel\_harvfrac` (0-1). - -50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter :math:`crop\_residue\_removal\_frac` (0–1). The default 50% is derived from :ref:`Smerald et al. 2023 `, who found a global average of 50% of residues left on the field. This includes residues burned in the field, meaning that our implementation implictly assumes the CLM crop burning representation will handle those residues appropriately. - -The following equations illustrate how this works. Subscript :math:`p` refers to either the leaf or live stem biomass pool. - -.. math:: - :label: 25.9 - - CF_{p,biofuel} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} - \right) * biofuel\_harvfrac - -.. math:: - :label: harv_c_to_removed_residue - - CF_{p,removed\_residue} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} - \right) * (1 - biofuel\_harvfrac) * crop\_residue\_removal\_frac - -.. math:: - :label: 25.11 - - CF_{p,litter} = \left({CS_{p} \mathord{\left/ {\vphantom {CS_{p} \Delta t}} \right.} \Delta t} - \right) * \left( 1-biofuel\_harvfrac \right) * \left( 1-crop\_residue\_removal\_frac \right) +CF_{p,alloc} - -with corresponding nitrogen fluxes: - -.. math:: - :label: 25.12 - - NF_{p,biofuel} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} - \right) * biofuel\_harvfrac - -.. math:: - :label: harv_n_to_removed_residue - - NF_{p,removed\_residue} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} - \right) * \left( 1 - biofuel\_harvfrac \right) * crop\_residue\_removal\_frac - -.. math:: - :label: 25.14 - - NF_{p,litter} = \left({NS_{p} \mathord{\left/ {\vphantom {NS_{p} \Delta t}} \right.} \Delta t} - \right) * \left( 1-biofuel\_harvfrac \right) * \left( 1-crop\_residue\_removal\_frac \right) - -where CF is the carbon flux, CS is stored carbon, NF is the nitrogen flux, NS is stored nitrogen, and :math:`biofuel\_harvfrac` is the harvested fraction of leaf/livestem for biofuel feedstocks. - -Annual food crop yields (g dry matter m\ :sup:`-2`) can be calculated by saving the GRAINC_TO_FOOD_ANN variable once per year, then postprocessing with Equation :eq:`25.15`. This calculation assumes that grain C is 45% of the total dry weight. Additionally, harvest is not typically 100% efficient, so analysis needs to assume that harvest efficiency is less---we use 85%. - -.. math:: - :label: 25.15 - - \text{Grain yield} = \frac{GRAINC\_TO\_FOOD\_ANN)*0.85}{0.45} - -.. _Table Crop allocation parameters: - -.. table:: Crop allocation parameters for the active crop plant functional types (PFTs) in CLM5BGCCROP. Numbers in the first row correspond to the list of PFTs in :numref:`Table Crop plant functional types`. - - =========================================== ============== ============ ================== ====== ====== ========= ============= ================ ================ ================ - \ temperate corn spring wheat temperate soybean cotton rice sugarcane tropical corn tropical soybean miscanthus switchgrass - =========================================== ============== ============ ================== ====== ====== ========= ============= ================ ================ ================ - IVT 17, 18 19, 20 23, 24 41, 42 61, 62 67, 68 75, 76 77, 78 71, 72 73, 74 - :math:`a_{leaf}^{i}` 0.6 0.9 0.85 0.85 0.75 0.6 0.6 0.85 0.9 0.7 - :math:`{L}_{max}` (m :sup:`2` m :sup:`-2`) 5 7 6 6 7 5 5 6 10 6.5 - :math:`a_{froot}^{i}` 0.1 0.05 0.2 0.2 0.1 0.1 0.1 0.2 0.11 0.14 - :math:`a_{froot}^{f}` 0.05 0 0.2 0.2 0 0.05 0.05 0.2 0.09 0.09 - :math:`a_{leaf}^{f}` 0 0 0 0 0 0 0 0 0 0 - :math:`a_{livestem}^{f}` 0 0.05 0.3 0.3 0.05 0 0 0.3 0 0 - :math:`d_{L}` 1.05 1.05 1.05 1.05 1.05 1.05 1.05 1.05 1.05 1.05 - :math:`d_{alloc}^{stem}` 2 1 5 5 1 2 2 5 2 2 - :math:`d_{alloc}^{leaf}` 5 3 2 2 3 5 5 2 5 5 - :math:`{CN}_{leaf}` 25 20 20 20 20 25 25 20 25 25 - :math:`{CN}_{stem}` 50 50 50 50 50 50 50 50 50 50 - :math:`{CN}_{froot}` 42 42 42 42 42 42 42 42 42 42 - :math:`CN^f_{leaf}` 65 65 65 65 65 65 65 65 65 65 - :math:`CN^f_{stem}` 120 100 130 130 100 120 120 130 120 120 - :math:`CN^f_{froot}` 0 40 0 0 40 0 0 0 0 0 - :math:`{CN}_{grain}` 50 50 50 50 50 50 50 50 50 50 - =========================================== ============== ============ ================== ====== ====== ========= ============= ================ ================ ================ - -Notes: Crop growth phases and corresponding variables are described throughout the text. :math:`{CN}_{leaf}`, :math:`{CN}_{stem}`, and :math:`{CN}_{froot}` are the target C:N ratios used during the leaf emergence phase (phase 2). - -.. _Other Features: - -Other Features -^^^^^^^^^^^^^^ - -.. _Physical Crop Characteristics: - -Physical Crop Characteristics -''''''''''''''''''''''''''''' -Leaf area index (*L*) is calculated as a function of specific leaf area (SLA, :numref:`Table Crop phenology parameters`) and leaf C. Stem area index (*S*) is equal to 0.1\ *L* for temperate and tropical corn, sugarcane, switchgrass, and miscanthus and 0.2\ *L* for other crops, as in AgroIBIS. All live C and N pools go to 0 after crop harvest, but the *S* is kept at 0.25 to simulate a post-harvest "stubble" on the ground. - -Crop heights at the top and bottom of the canopy, :math:`{z}_{top}` and :math:`{z}_{bot}` (m), come from the AgroIBIS formulation: - -.. math:: - :label: 25.16 - - \begin{array}{l} - {z_{top} =z_{top}^{\max } \left(\frac{L}{L_{\max } -1} \right)^{2} \ge 0.05{\rm \; where\; }\frac{L}{L_{\max } -1} \le 1} \\ - {z_{bot} =0.02{\rm m}} - \end{array} - -where :math:`z_{top}^{\max }` is the maximum top-of-canopy height of the crop (:numref:`Table Crop phenology parameters`) and :math:`L_{\max }` is the maximum leaf area index (:numref:`Table Crop allocation parameters`). - -.. _Interactive fertilization: - -Interactive Fertilization -''''''''''''''''''''''''' -CLM simulates fertilization by adding nitrogen directly to the soil mineral nitrogen pool to meet crop nitrogen demands using both industrial fertilizer and manure application. CLM's separate crop land unit ensures that natural vegetation will not access the fertilizer applied to crops. Fertilizer in CLM5BGCCROP is prescribed by crop functional types and varies spatially for each year based on the LUMIP land use and land cover change time series (LUH2 for historical and SSPs for future) (:ref:`Lawrence et al. 2016 `). One of two fields is used to prescribe industrial fertilizer based on the type of simulation. For non-transient simulations, annual fertilizer application in g N/m\ :sup:`2`/yr is specified on the land surface data set by the field CONST_FERTNITRO_CFT. In transient simulations, annual fertilizer application is specified on the land use time series file by the field FERTNITRO_CFT, which is also in g N/m\ :sup:`2`/yr. The values for both of these fields come from the LUMIP time series for each year. In addition to the industrial fertilizer, background manure fertilizer is specified on the parameter file by the field ``manunitro``. For perennial bioenergy crops, little fertilizer (56kg/ha/yr) is applied to switchgrass and no fertilizer is applied to Miscanthus. Note these rates are only based on local land management practices at the University of Illinois Energy Farm located in Central Midwestern United States :ref:`(Cheng et al., 2019)` rather than the LUMIP timeseries. For the current CLM5BGCCROP, manure N is applied at a rate of 0.002 kg N/m\ :sup:`2`/yr. Because previous versions of CLM (e.g., CLM4) had rapid denitrification rates, fertilizer is applied slowly to minimize N loss (primarily through denitrification) and maximize plant uptake. The current implementation of CLM5 inherits this legacy, although denitrification rates are slower in the current version of the model (:ref:`Koven et al. 2013 `). As such, fertilizer application begins during the leaf emergence phase of crop development (phase 2) and continues for 20 days, which helps reduce large losses of nitrogen from leaching and denitrification during the early stage of crop development. The 20-day period is chosen as an optimization to limit fertilizer application to the emergence stage. A fertilizer counter in seconds, *f*, is set as soon as the leaf emergence phase for crops initiates: - -.. math:: - :label: 25.17 - - f = n \times 86400 - -where *n* is set to 20 fertilizer application days and 86400 is the number of seconds per day. When the crop enters phase 2 (leaf emergence) of its growth cycle, fertilizer application begins by initializing fertilizer amount to the total fertilizer at each column within the grid cell divided by the initialized *f*. Fertilizer is applied and *f* is decremented each time step until a zero balance on the counter is reached. - -.. _Biological nitrogen fixation for soybeans: - -Biological nitrogen fixation for soybeans -''''''''''''''''''''''''''''''''''''''''' -Biological N fixation for soybeans is calculated by the fixation and uptake of nitrogen module (Chapter :numref:`rst_FUN`) and is the same as N fixation in natural vegetation. Unlike natural vegetation, where a fraction of each PFT are N fixers, all soybeans are treated as N fixers. - -.. _Latitude vary base tempereature for growing degree days: - -Latitudinal variation in base growth tempereature -''''''''''''''''''''''''''''''''''''''''''''''''' -For most crops, :math:`GDD_{T_{{\rm 2m}} }` (growing degree days since planting) is the same in all locations. However, for both rainfed and irrigated spring wheat and sugarcane, the calculation of :math:`GDD_{T_{{\rm 2m}} }` allows for latitudinal variation: - -.. math:: - :label: 25.18 - - latitudinal\ variation\ in\ base\ T = \left\{ - \begin{array}{lr} - baset +12 - 0.4 \times latitude &\qquad 0 \le latitude \le 30 \\ - baset +12 + 0.4 \times latitude &\qquad -30 \le latitude \le 0 - \end{array} \right\} - -where :math:`baset` is the *base temperature for GDD* (7\ :sup:`th` row) in :numref:`Table Crop phenology parameters`. Such latitudinal variation in base temperature could slow :math:`GDD_{T_{{\rm 2m}} }` accumulation extend the growing season for regions within 30°S to 30°N for spring wheat and sugarcane. - -.. _Separate reproductive pool: - -Separate reproductive pool -'''''''''''''''''''''''''' -One notable difference between natural vegetation and crops is the presence of reproductive carbon and nitrogen pools. Accounting for the reproductive pools helps determine whether crops are performing reasonably through yield calculations. The reproductive pool is maintained similarly to the leaf, stem, and fine root pools, but allocation of carbon and nitrogen does not begin until the grain fill stage of crop development. Equation :eq:`25.5` describes the carbon and nitrogen allocation coefficients to the reproductive pool. In CLM5BGCCROP, as allocation declines in stem, leaf, and root pools (see section :numref:`Grain fill to harvest`) during the grain fill stage of growth, increasing amounts of carbon and nitrogen are available for grain development. - -.. _Tillage: - -Tillage -''''''' -Tillage is represented as an enhancement of the decomposition rate coefficient; see section :numref:`decomp_mgmt_modifiers`. - -.. _The irrigation model: - -The irrigation model --------------------- - -The CLM includes the option to irrigate cropland areas that are equipped for irrigation. The application of irrigation responds dynamically to the soil moisture conditions simulated by the CLM. This irrigation algorithm is based loosely on the implementation of :ref:`Ozdogan et al. (2010) `. - -When irrigation is enabled, the crop areas of each grid cell are divided into irrigated and rainfed fractions according to a dataset of areas equipped for irrigation (:ref:`Portmann et al. 2010 `). Irrigated and rainfed crops are placed on separate soil columns, so that irrigation is only applied to the soil beneath irrigated crops. - -In irrigated croplands, a check is made once per day to determine whether irrigation is required on that day. This check is made in the first time step after 6 AM local time. Irrigation is required if crop leaf area :math:`>` 0, and the available soil water is below a specified threshold. - -The soil moisture deficit :math:`D_{irrig}` is - -.. math:: - :label: 25.61 - - D_{irrig} = \left\{ - \begin{array}{lr} - w_{target} - w_{avail} &\qquad w_{thresh} > w_{avail} \\ - 0 &\qquad w_{thresh} \le w_{avail} - \end{array} \right\} - -where :math:`w_{target}` is the irrigation target soil moisture (mm) - -.. math:: - :label: 25.62 - - w_{target} = \sum_{j=1}^{N_{irr}} \theta_{target} \Delta z_{j} \ . - -The irrigation moisture threshold (mm) is - -.. math:: - :label: 25.63 - - w_{thresh} = f_{thresh} \left(w_{target} - w_{wilt}\right) + w_{wilt} - -where :math:`w_{wilt}` is the wilting point soil moisture (mm) - -.. math:: - :label: 25.64 - - w_{wilt} = \sum_{j=1}^{N_{irr}} \theta_{wilt} \Delta z_{j} \ , - -and :math:`f_{thresh}` is a tuning parameter. The available moisture in the soil (mm) is - -.. math:: - :label: 25.65 - - w_{avail} = \sum_{j=1}^{N_{irr}} \theta_{j} \Delta z_{j} \ , - -Note that :math:`w_{target}` is truly supposed to give the target soil moisture value that we're shooting for whenever irrigation happens; then the soil moisture deficit :math:`D_{irrig}` gives the difference between this target value and the current soil moisture. The irrigation moisture threshold :math:`w_{thresh}`, on the other hand, gives a threshold at which we decide to do any irrigation at all. The way this is written allows for the possibility that one may not want to irrigate every time there becomes even a tiny soil moisture deficit. Instead, one may want to wait until the deficit is larger before initiating irrigation; at that point, one doesn't want to just irrigate up to the "threshold" but instead up to the higher "target". The target should always be greater than or equal to the threshold. - -:math:`N_{irr}` is the index of the soil layer corresponding to a specified depth :math:`z_{irrig}` (:numref:`Table Irrigation parameters`) and :math:`\Delta z_{j}` is the thickness of the soil layer in layer :math:`j` (section :numref:`Vertical Discretization`). :math:`\theta_{j}` is the volumetric soil moisture in layer :math:`j` (section :numref:`Soil Water`). :math:`\theta_{target}` and :math:`\theta_{wilt}` are the target and wilting point volumetric soil moisture values, respectively, and are determined by inverting :eq:`7.94` using soil matric potential parameters :math:`\Psi_{target}` and :math:`\Psi_{wilt}` (:numref:`Table Irrigation parameters`). After the soil moisture deficit :math:`D_{irrig}` is calculated, irrigation in an amount equal to :math:`\frac{D_{irrig}}{T_{irrig}}` (mm/s) is applied uniformly over the irrigation period :math:`T_{irrig}` (s). Irrigation water is applied directly to the ground surface, bypassing canopy interception (i.e., added to :math:`{q}_{grnd,liq}`: section :numref:`Canopy Water`). - -To conserve mass, irrigation is removed from river water storage (Chapter :numref:`rst_MOSART`). When river water storage is inadequate to meet irrigation demand, there are two options: 1) the additional water can be removed from the ocean model, or 2) the irrigation demand can be reduced such that river water storage is maintained above a specified threshold. - -.. _Table Irrigation parameters: - -.. table:: Irrigation parameters - - +--------------------------------------+-------------+ - | Parameter | | - +======================================+=============+ - | :math:`f_{thresh}` | 1.0 | - +--------------------------------------+-------------+ - | :math:`z_{irrig}` (m) | 0.6 | - +--------------------------------------+-------------+ - | :math:`\Psi_{target}` (mm) | -3400 | - +--------------------------------------+-------------+ - | :math:`\Psi_{wilt}` (mm) | -150000 | - +--------------------------------------+-------------+ - -.. add a reference to surface data in chapter2 - To accomplish this we downloaded data of percent irrigated and percent rainfed corn, soybean, and temperate cereals (wheat, barley, and rye) (:ref:`Portmann et al. 2010 `), available online from *ftp://ftp.rz.uni-frankfurt.de/pub/uni-frankfurt/physische\_geographie/hydrologie/public/data/MIRCA2000/harvested\_area\_grids.* diff --git a/doc/source/tech_note/References/CLM50_Tech_Note_References.rst b/doc/source/tech_note/References/CLM50_Tech_Note_References.rst index b8daafe939..f3721c05f3 100644 --- a/doc/source/tech_note/References/CLM50_Tech_Note_References.rst +++ b/doc/source/tech_note/References/CLM50_Tech_Note_References.rst @@ -684,6 +684,10 @@ Jacksonetal1996: E., and Schulze, E. D. 1996. A global analysis of root distribu Jackson, T.L., Feddema, J.J., Oleson, K.W., Bonan, G.B., and Bauer, J.T. 2010. Parameterization of urban characteristics for global climate modeling. Annals of the Association of American Geographers. 100:848-865. +.. _Jägermeyretal2021: + +Jägermeyr, J., Müller, C., Minoli, S., Ray, D., & Siebert, S. (2021). GGCMI Phase 3 crop calendar [Data set]. Zenodo. https://doi.org/10.5281/zenodo.5062513. See also: https://www.isimip.org/gettingstarted/input-data-bias-adjustment/details/115/ + .. _JenkinsonColeman2008: Jenkinson, D. and Coleman, K. 2008. The turnover of organic carbon in subsoils. Part 2. Modelling carbon turnover. European Journal of Soil Science 59:400-413. @@ -772,6 +776,10 @@ Koven, C.D., G. Hugelius, D.M. Lawrence, and W.R. Wieder, 2017: Higher climatolo Kucharik, C.J., J.M. Norman, and S.T. Gower, 1998. Measurements of branch area and adjusting leaf area index indirect measurements. Agricultural and Forest Meteorology 91.1, pp. 69-88. +.. _kucharik2003: + +Kucharik, C.J., 2003. Evaluation of a Process-Based Agro-Ecosystem Model (Agro-IBIS) across the U.S. Corn Belt: Simulations of the Interannual Variability in Maize Yield. Earth Interactions 7, paper 14. DOI: 10.1175/1087-3562(2003)007<0001:EOAPAM>2.0.CO;2. + .. _Kuchariketal2000: Kucharik, C.J., Foley, J.A., Delire, C., Fisher, V.A., Coe, M.T., Lenters, J.D., Young-Molling, C., and Ramankutty, N. 2000. Testing the performance of a dynamic global ecosystem model: water balance, carbon balance, and vegetation structure. Global Biogeochem. Cycles 14: 795–825. @@ -1297,6 +1305,10 @@ Purves, D.W. et al., 2008. Predicting and understanding forest dynamics using a Qian, T et al., 2006. Simulation of global land surface conditions from 1948 to 2004: Part I: Forcing data and evaluations. J. Hydrometeorology 7, pp. 953-975. +.. _Rabinetal2023: + +Rabin, S., Sacks, W., Lombardozzi, D., Xia, L. & Robock, A., 2023. Observation-based sowing dates and cultivars significantly affect yield and irrigation for some crops in the Community Land Model (CLM5). Geosci. Model Dev. 16, 7253–7273 (2023). DOI 10.5194/gmd-16-7253-2023. https://gmd.copernicus.org/articles/16/7253/2023/ + .. _RamankuttyFoley1998: Ramankutty, N., and Foley, J. A., 1998. Characterizing patterns of global land use: An analysis of global croplands data. Global Biogeochemical Cycles, 12, 667-685. diff --git a/doc/source/tech_note/index.rst b/doc/source/tech_note/index.rst index 61ccae4361..56cd7d3334 100644 --- a/doc/source/tech_note/index.rst +++ b/doc/source/tech_note/index.rst @@ -42,7 +42,7 @@ CLM Technical Note Plant_Mortality/CLM50_Tech_Note_Plant_Mortality.rst Fire/CLM50_Tech_Note_Fire.rst Methane/CLM50_Tech_Note_Methane.rst - Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst + Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.md Transient_Landcover/CLM50_Tech_Note_Transient_Landcover.rst DGVM/CLM50_Tech_Note_DGVM.rst BVOCs/CLM50_Tech_Note_BVOCs.rst diff --git a/doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst b/doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst index 5e2b1998cc..34271d0027 100644 --- a/doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst +++ b/doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst @@ -1,7 +1,7 @@ -.. running-with-custom-crop-calendars: - .. include:: ../substitutions.rst +.. _running-with-custom-crop-calendars: + ======================================= Running with custom crop calendars ======================================= diff --git a/doc/source/users_guide/using-clm-tools/paramfile-tools.md b/doc/source/users_guide/using-clm-tools/paramfile-tools.md index 84b4d496eb..52bff402d7 100644 --- a/doc/source/users_guide/using-clm-tools/paramfile-tools.md +++ b/doc/source/users_guide/using-clm-tools/paramfile-tools.md @@ -5,6 +5,7 @@ This guide describes the features and usage of the `query_paramfile` and `set_pa Note that you need to have the `ctsm_pylib` conda environment activated to use these tools. See Sect. {numref}`using-ctsm-pylib` for more information. +(query-paramfile)= ## `query_paramfile` **Purpose:** Print the values of one or more parameters from a CTSM parameter file (netCDF format). @@ -32,6 +33,7 @@ Print values for specific PFTs: tools/param_utils/query_paramfile -i paramfile.nc -p needleleaf_evergreen_temperate_tree,c4_grass medlynintercept medlynslope ``` +(set-paramfile)= ## `set_paramfile` **Purpose:** Change values of one or more parameters in a CTSM parameter file (netCDF format). diff --git a/doc/source/users_guide/working-with-documentation/vscode-doc-editing-setup.md b/doc/source/users_guide/working-with-documentation/vscode-doc-editing-setup.md index 4a531b7647..a8e7a7c4d4 100644 --- a/doc/source/users_guide/working-with-documentation/vscode-doc-editing-setup.md +++ b/doc/source/users_guide/working-with-documentation/vscode-doc-editing-setup.md @@ -3,7 +3,7 @@ # Recommended setup for editing docs in VS Code ## Set up Python -The Python packages needed for this setup are listed at `doc/ctsm-docs_container/requirements.txt`. If you recently installed [the `ctsm_pylib` conda environment](https://escomp.github.io/CTSM/users_guide/using-clm-tools/using-ctsm-pylib.html), they will already be included there. Check like so: +The Python packages needed for this setup are listed at `doc/doc-builder/doc-build-container/requirements.txt`. If you recently installed [the `ctsm_pylib` conda environment](https://escomp.github.io/CTSM/users_guide/using-clm-tools/using-ctsm-pylib.html), they will already be included there. Check like so: ```shell conda activate ctsm_pylib pip show myst-parser @@ -11,7 +11,8 @@ pip show myst-parser If you see `WARNING: Package(s) not found: myst-parser`, you'll need to install the doc-building Python modules. With your conda environment activated (or in whatever other environment you want to use), from the top level of a CTSM checkout, do: ```shell -pip install -r doc/ctsm-docs_container/requirements.txt +bin/git-fleximod update doc-builder +pip install -r doc/doc-builder/doc-build-container/requirements.txt ``` You will need to provide the path to your Python binary in various VS Code settings detailed below. You can get that with `which python` (after loading the relevant conda environment, if needed). diff --git a/doc/substitutions.py b/doc/substitutions.py index 29b6fb54de..1fae4915b0 100644 --- a/doc/substitutions.py +++ b/doc/substitutions.py @@ -60,4 +60,28 @@ "dirmenu_entry": "clmdoc", "description": "One line description of project.", "category": tex_category, -} \ No newline at end of file +} + +############################### +### Purely custom variables ### +############################### + +nonparamfile_disclaimer_md = ( + "**Note:** The values here should be up-to-date with those used in {{version_label}}," + " but there may be mistakes." +) +nonparamfile_disclaimer_rst = ( + "**Note:** The values here should be up-to-date with those used in |version_label|," + " but there may be mistakes." +) + +paramfile_disclaimer_md = ( + nonparamfile_disclaimer_md + + " If you want to check the values for your version, use" + " [](query-paramfile) on the `paramfile` for your case." +) +paramfile_disclaimer_rst = ( + nonparamfile_disclaimer_rst + + " If you want to check the values for your version, use" + " :ref:`query-paramfile` on the ``paramfile`` for your case." +) diff --git a/doc/test/test_build_docs_-b.sh b/doc/test/test_build_docs_-b.sh index 8b49e2f7aa..17454c982e 100755 --- a/doc/test/test_build_docs_-b.sh +++ b/doc/test/test_build_docs_-b.sh @@ -9,7 +9,7 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd "${SCRIPT_DIR}/.." msg="~~~~~ Check that -b works" -cmd="./build_docs -b _build -c" +cmd="./build_docs -b _build -c --verbose" . test/compose_test_cmd.sh set -x diff --git a/doc/test/test_build_docs_-r-v.sh b/doc/test/test_build_docs_-r-v.sh index 6f9415b563..8d64991c51 100755 --- a/doc/test/test_build_docs_-r-v.sh +++ b/doc/test/test_build_docs_-r-v.sh @@ -9,7 +9,7 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd "${SCRIPT_DIR}/.." msg="~~~~~ Check that -r -v works" -cmd="./build_docs -r _build -v latest -c --conf-py-path doc-builder/test/conf.py --static-path ../_static --templates-path ../_templates" +cmd="./build_docs -r _build -v latest -c --verbose" . test/compose_test_cmd.sh set -x diff --git a/doc/test/test_container_eq_ctsm_pylib.sh b/doc/test/test_container_eq_ctsm_pylib.sh index 729f1b723e..3a1375d3b7 100755 --- a/doc/test/test_container_eq_ctsm_pylib.sh +++ b/doc/test/test_container_eq_ctsm_pylib.sh @@ -17,15 +17,16 @@ echo "~~~~~ Build all docs using container" # Also do a custom --conf-py-path rm -rf _build _publish d1="$PWD/_publish_container" -./build_docs_to_publish -r _build -d --site-root "$PWD/_publish" +./build_docs_to_publish -r _build -d --site-root "$PWD/_publish" --verbose # VERSION LINKS WILL NOT RESOLVE IN _publish_container cp -a _publish "${d1}" # Build all docs using ctsm_pylib echo "~~~~~ Build all docs using ctsm_pylib" rm -rf _build _publish +rm -rf "${d2}" d2="$PWD/_publish_nocontainer" -conda run -n ctsm_pylib --no-capture-output ./build_docs_to_publish -r _build --site-root "$PWD/_publish" --conf-py-path doc-builder/test/conf.py --static-path ../_static --templates-path ../_templates +conda run -n ctsm_pylib --no-capture-output ./build_docs_to_publish -r _build --site-root "$PWD/_publish" --verbose # VERSION LINKS WILL NOT RESOLVE IN _publish_nocontainer cp -a _publish "${d2}" diff --git a/py_env_create b/py_env_create index cf34131e41..f5a2d5c598 100755 --- a/py_env_create +++ b/py_env_create @@ -237,6 +237,10 @@ if [[ ${dry_run} -ne 0 ]]; then exit 0 fi +# Make sure doc-builder is up-to-date, since it contains a requirements.txt file +echo "Making sure doc-builder is up-to-date..." +bin/git-fleximod update doc-builder 1>/dev/null + # Rename or overwrite existing if [[ ${conda_env_does_exist} -gt 0 ]]; then if [[ "${new_name_for_existing}" != "" ]]; then diff --git a/python/conda_env_ctsm_py.yml b/python/conda_env_ctsm_py.yml index 96656696db..9cabb8d283 100644 --- a/python/conda_env_ctsm_py.yml +++ b/python/conda_env_ctsm_py.yml @@ -25,7 +25,7 @@ dependencies: # For building docs - pip: - - -r ../doc/ctsm-docs_container/requirements.txt + - -r ../doc/doc-builder/doc-build-container/requirements.txt # Used in ctsm_postprocessing repo - pytest=8.3.5 diff --git a/tools/contrib/preview_docs_pr b/tools/contrib/preview_docs_pr deleted file mode 100755 index 038951293b..0000000000 --- a/tools/contrib/preview_docs_pr +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python -"""Given a GitHub PR URL, download the code as if it had been merged and then build the docs""" - -import urllib.request -import urllib.error -import json -import re -import os -import sys -import argparse -import subprocess -import time -import zipfile -import stat -import shutil - -# Get default location in which to clone the code -SCRATCH = os.getenv("SCRATCH") -DEFAULT_EXTRACTION_DIR_BASENAME = ( - "preview_docs_pr.{}.{}.pr-{}" # repo owner, repo name, PR number -) -DEFAULT_EXTRACTION_DIR = os.path.join( - SCRATCH if SCRATCH else "", - DEFAULT_EXTRACTION_DIR_BASENAME, -) - -INDENT = 4 * " " - - -def parse_pr_url(url): - """Extract owner, repo, and PR number from a GitHub PR URL.""" - pattern = r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)" - match = re.match(pattern, url.strip()) - if not match: - raise ValueError(f"Invalid GitHub PR URL: {url}") - owner, repo, pr_number = match.groups() - return owner, repo, int(pr_number) - - -def make_api_call(url, token): - """Helper function to make a GitHub API call""" - headers = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - if token: - headers["Authorization"] = f"Bearer {token}" - - req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read().decode()) - - -def fetch_pr_info(owner, repo, pr_number, token=None): - """Fetch PR metadata from the GitHub API.""" - - # Get overall PR info - pr_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" - pr_info = make_api_call(pr_url, token) - - # Get files touched by PR - files_url = pr_url + "/files" - pr_files = make_api_call(files_url, token) - - return pr_info, pr_files - - -def fetch_pr_info_with_mergeability(owner, repo, pr_number, token=None, retries=5): - """Fetch the PR info, retrying as needed if mergeability hasn't yet been computed""" - wait_time = 10 # seconds - for attempt in range(retries): - pr, files = fetch_pr_info(owner, repo, pr_number, token) - if pr["state"] != "open" or pr.get("mergeable") is not None: - return pr, files - print( - f" Mergeability not yet computed, waiting {wait_time} seconds and retrying" - f"({attempt + 1}/{retries})..." - ) - time.sleep(wait_time) - raise RuntimeError(f"Mergeability still unknown after {retries} retries.") - - -def pick_ref(pr): - """ - Choose the best ref to download, with explanation. - """ - state = pr["state"] - merge_commit_sha = pr.get("merge_commit_sha") - mergeable = pr.get("mergeable") - - # If the PR can't be merged, we can't preview what it'd look like after merge. - # Note that "mergeable" only refers to whether there are Git conflicts; it doesn't "know" - # anything about PR tests that are passing or failing. - if not mergeable: - print( - "PR branch has merge conflicts, so merged docs can't be previewed. Exiting." - ) - sys.exit(1) - - # If PR was closed without merging, GitHub supposedly nulls out merge_commit_sha, so that - # situation would require special handling. - if state == "closed" and not pr.get("merged_at"): - raise NotImplementedError("PR closed without merge") - - if not merge_commit_sha: - raise RuntimeError("How is this possible?") - - return merge_commit_sha - - -def download_zip(extraction_dir, token, pr_info, merge_commit_sha): - """Download the code as of the merge commit from GitHub""" - print("Downloading zip from GitHub...") - owner, repo, pr_number = pr_info - zip_url = f"https://api.github.com/repos/{owner}/{repo}/zipball/{merge_commit_sha}" - headers = { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - if token: - headers["Authorization"] = f"Bearer {token}" - req = urllib.request.Request(zip_url, headers=headers) - pr_number = pr_info[2] - zip_path = os.path.join(extraction_dir, f"pr-{pr_number}.zip") - try: - with urllib.request.urlopen(req) as resp, open(zip_path, "wb") as f: - f.write(resp.read()) - except urllib.error.HTTPError as e: - raise RuntimeError(f"Failed to download zip from {zip_url}: {e}") from e - return zip_path - - -def extract_zip(extraction_dir, zip_path, overwrite): - """Unzip the code""" - try: - with zipfile.ZipFile(zip_path, "r") as zf: - # GitHub zips have a single top-level folder like "owner-repo-/" - top_level = zf.namelist()[0].split("/")[0] - extracted_path = os.path.join(extraction_dir, top_level) - - # If it already exists, it would be nice to skip re-extracting. But we would want to do - # a git reset to make sure it's actually at the right commit, and unfortunately the - # downloaded ZIP file doesn't include the git history. So instead, we delete the - # existing one first. - if os.path.exists(extracted_path): - if not overwrite: - raise FileExistsError( - "Clone directory exists; add -o/--overwrite to overwrite:" - f"'{extracted_path}'" - ) - print(f"Deleting existing clone dir: '{extracted_path}'") - shutil.rmtree(extracted_path) - - # Unzip - print("Unzipping...") - zf.extractall(extraction_dir) - os.remove(zip_path) - except: - os.remove(zip_path) - raise - - print(f"Code downloaded to: '{extracted_path}'") - return extracted_path - - -def download_pr_code(pr_url, extraction_dir, overwrite, token=None): - """Download code as if the pull request had been merged""" - # Fetch PR information via GitHub API - pr_info = parse_pr_url(pr_url) - pr, files = fetch_pr_info_with_mergeability(*pr_info, token) - - # Get the ref to download - merge_commit_sha = pick_ref(pr) - - if not os.path.exists(extraction_dir): - os.makedirs(extraction_dir) - - # Download the zip (follows redirects automatically) - zip_path = download_zip(extraction_dir, token, pr_info, merge_commit_sha) - - # Unzip - code_dir = extract_zip(extraction_dir, zip_path, overwrite) - - return code_dir, files, pr_info - - -def build_docs(code_dir, files, verbose, pr_info): - """Build the documentation""" - os.chdir(code_dir) - - # Handle verbosity for subprocess calls - if verbose: - stdout = None - else: - stdout = subprocess.DEVNULL - - # Set up so that git lfs works - set_up_git(pr_info, stdout) - - print("Getting submodules...") - os.chmod(path := "bin/git-fleximod", os.stat(path).st_mode | stat.S_IXUSR) - subprocess.check_call([path, "update"], stdout=stdout) - - print("Building docs...") - os.chdir("doc") - os.chmod(path := "./build_docs", os.stat(path).st_mode | stat.S_IXUSR) - output = [] - cmd = [path, "-b", "_build", "-d", "-c"] - if verbose: - cmd.append("--verbose") - with subprocess.Popen( - cmd, - cwd=os.path.join(code_dir, "doc"), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Merge stderr into stdout - text=True, - bufsize=1, # Line buffering - env={**os.environ, "PYTHONUNBUFFERED": "1"}, # Force unbuffered output - ) as process: - for line in process.stdout: - print(line, end="") # Print to screen - output.append(line) # Save for later - output = "".join(output) - if output and os.path.exists(os.path.join("_build", "html", "index.html")): - print_files_msg(code_dir, files) - - -def set_up_git(pr_info, stdout): - """Initialize a git repo in the downloaded code so that git lfs works right""" - - owner, repo, _ = pr_info - remote_url = f"https://github.com/{owner}/{repo}.git" - subprocess.check_call(["git", "init", "-b", "preview-docs"], stdout=stdout) - subprocess.check_call(["git", "remote", "add", "origin", remote_url], stdout=stdout) - subprocess.check_call(["git", "add", "-A"], stdout=stdout) - subprocess.check_call( - [ - "git", - "commit", - "-m", - "PR snapshot for docs preview", - ], - stdout=stdout, - ) - - -def print_files_msg(code_dir, files): - """Print a message about the directly-affected files""" - build_dir = os.path.join(code_dir, "doc", "_build") - html_dir = os.path.join(build_dir, "html") - print(f"\nThe updated files are in {html_dir}") - print("Doc source files directly touched (not deleted) by PR:") - for f_dict in files: - f = f_dict["filename"] - # Slashes here are platform-independent, because f is returned from GitHub API call - # Skip deleted or otherwise nonexistent files - full_path = os.path.join(code_dir, f) - if f_dict["status"] == "removed" or not os.path.exists(full_path): - continue - - # Skip files not in doc source - if not f.startswith("doc/source/"): - continue - - # Get string to print - f_print = "/".join(f.split("/")[2:]) # Remove leading doc/source/ - root, extension = os.path.splitext(f_print) - basename = f.split("/")[-1] # pylint: disable=use-maxsplit-arg - if extension in [".rst", ".md"]: - # These types get converted to HTML - f_print = root + ".html" - assert os.path.exists(os.path.join(html_dir, f_print)) - elif os.path.exists(os.path.join(html_dir, "_images", basename)): - # Image files get put in _build/html/_images/ - f_print = os.path.join("_images", basename) - else: - f_print = "[NOT SURE WHERE THIS IS BUILT TO] " + f_print - print(INDENT + f_print) - print( - "Note that changes to these or other files may indirectly affect other doc files!" - " For example, if one of these files is a new/changed image, or a new/changed text" - " file that's `include`d somewhere, or a text file with an updated label that's cross-" - "referenced elsewhere. Or if a file outside doc/source/ that's `include`d in a doc file" - " got changed." - ) - - -def parse_args(): - """Parse arguments""" - parser = argparse.ArgumentParser( - description="Given a GitHub PR URL, download the merged version and build the docs." - ) - - parser.add_argument( - "pr_url", - help="GitHub pull request URL", - type=str, - ) - - extraction_dir_help_default = DEFAULT_EXTRACTION_DIR.format( - "REPO_OWNER", "REPO", "PR_NUM" - ) - parser.add_argument( - "--dir", - "--extraction-dir", - dest="extraction_dir", - help=( - "Directory to download and extract code versions to." - f" Default: {extraction_dir_help_default}" - ), - type=str, - default=None, - ) - - parser.add_argument( - "-o", - "--overwrite", - help="Overwrite existing clone dir, if any.", - action="store_true", - ) - - parser.add_argument( - "-v", - "--verbose", - help="Verbose output, including for build_docs command.", - action="store_true", - ) - - args = parser.parse_args(sys.argv[1:]) - - # Get clone dir, if not provided - pr_info = parse_pr_url(args.pr_url) - if not args.extraction_dir: - args.extraction_dir = DEFAULT_EXTRACTION_DIR.format(*pr_info) - elif os.path.abspath(args.extraction_dir) == os.getcwd(): - args.extraction_dir = os.path.join( - os.getcwd(), DEFAULT_EXTRACTION_DIR_BASENAME.format(*pr_info) - ) - print(f"Will clone to '{args.extraction_dir}'") - - # Check that clone dir parent exists - args.extraction_dir = os.path.abspath(args.extraction_dir) - clone_parent = os.path.dirname(args.extraction_dir) - if not os.path.exists(clone_parent): - raise NotImplementedError( - f"Clone parent directory does not exist: '{clone_parent}'" - ) - - # Check that clone dir parent is not (in) a git repo - result = subprocess.run( - ["git", "-C", clone_parent, "rev-parse", "--is-inside-work-tree"], - check=False, - capture_output=True, - ) - if not result.returncode: - raise RuntimeError( - "Clone parent directory is (in) a git repo, which would cause git-fleximod problems: " - f"'{clone_parent}'" - ) - - return args - - -def main(): - """Main function""" - args = parse_args() - # GitHub personal access tokens are currently not supported, but the code is all there. Just - # need to think about how to implement them in a way that will prevent people from putting their - # secrets into their shell history. - code_dir, files, pr_info = download_pr_code( - args.pr_url, args.extraction_dir, args.overwrite, token=None - ) - build_docs(code_dir, files, args.verbose, pr_info) - - -if __name__ == "__main__": - main()