Merge branch 'main' into sb/constructors

Author: glwagner

.buildkite/JuliaProject.toml

@@ -1,20 +1,14 @@
 agents:
-  queue: central
+  queue: new-central
   slurm_mem: 8G
-  modules: julia/1.10.0 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.2.1
+  modules: climacommon/2024_03_18
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
-  # JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   JULIA_CPU_TARGET: 'broadwell;skylake'
   JULIA_NVTX_CALLBACKS: gc
-  JULIA_CUDA_MEMORY_POOL: none
-  JULIA_MPI_HAS_CUDA: "true"
   OPENBLAS_NUM_THREADS: 1
-  OMPI_MCA_opal_warn_on_missing_libcuda: 0
-  MPITRAMPOLINE_LIB: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so
-  MPITRAMPOLINE_MPIEXEC: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec
 
 steps:
   - label: "initialize"
@@ -729,13 +723,12 @@ steps:
 
   # TODO: improve performance label: [inference, allocs, flops, latency, benchmark, boundscheck]
   # TODO: use perf env for perf jobs, or merge test/perf envs
-  # https://github.com/triscale-innov/GFlops.jl/issues/49
-  # - group: "Perf: Geometry"
-  #   steps:
+  - group: "Perf: Geometry"
+    steps:
 
-  #     - label: "Perf: Axis tensor conversion benchmarks"
-  #       key: "cpu_axis_tensor_conversion_perf_bm"
-  #       command: "julia --color=yes --project=test test/Geometry/axistensor_conversion_benchmarks.jl"
+      - label: "Perf: Axis tensor conversion benchmarks"
+        key: "cpu_axis_tensor_conversion_perf_bm"
+        command: "julia --color=yes --project=test test/Geometry/axistensor_conversion_benchmarks.jl"
 
   - group: "Perf: Fields"
     steps:
@@ -760,6 +753,14 @@ steps:
         key: perf_FD_ops_examples
         command: "julia --color=yes --project=test test/Operators/finitedifference/opt_examples.jl"
 
+      - label: "Perf: FD operators from the wild (gpu)"
+        key: perf_FD_ops_examples_gpu
+        command:
+          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --project=test test/Operators/finitedifference/opt_examples.jl"
+        agents:
+          slurm_gpus: 1
+
       - label: "Perf: dss"
         key: perf_dss
         command: "julia --color=yes --project=test test/Operators/hybrid/dss_opt.jl"

.buildkite/pipeline.yml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.9.3"
+julia_version = "1.10.2"
 manifest_format = "2.0"
 project_hash = "30b405be1c677184b7703a9bfb3d2100029ccad0"
 
@@ -16,9 +16,9 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[deps.CSTParser]]
 deps = ["Tokenize"]
-git-tree-sha1 = "3ddd48d200eb8ddf9cb3e0189fc059fd49b97c1f"
+git-tree-sha1 = "b544d62417a99d091c569b95109bc9d8c223e9e3"
 uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
-version = "3.3.6"
+version = "3.4.2"
 
 [[deps.CommonMark]]
 deps = ["Crayons", "JSON", "PrecompileTools", "URIs"]
@@ -27,10 +27,10 @@ uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6"
 version = "0.8.12"
 
 [[deps.Compat]]
-deps = ["UUIDs"]
-git-tree-sha1 = "8a62af3e248a8c4bad6b32cbbe663ae02275e32c"
+deps = ["TOML", "UUIDs"]
+git-tree-sha1 = "c955881e3c981181362ae4088b35995446298b80"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.10.0"
+version = "4.14.0"
 
     [deps.Compat.extensions]
     CompatLinearAlgebraExt = "LinearAlgebra"
@@ -46,9 +46,9 @@ version = "4.1.1"
 
 [[deps.DataStructures]]
 deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d"
+git-tree-sha1 = "0f4b5d62a88d8f59003e43c25a8a90de9eb76317"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.15"
+version = "0.18.18"
 
 [[deps.Dates]]
 deps = ["Printf"]
@@ -79,28 +79,33 @@ version = "0.21.4"
 
 [[deps.JuliaFormatter]]
 deps = ["CSTParser", "CommonMark", "DataStructures", "Glob", "Pkg", "PrecompileTools", "Tokenize"]
-git-tree-sha1 = "3d5b5b539e4606dcca0e6a467b98a64c8da4850b"
+git-tree-sha1 = "e07d6fd7db543b11cd90ed764efec53f39851f09"
 uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
-version = "1.0.42"
+version = "1.0.54"
 
 [[deps.LibCURL]]
 deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.6.3"
+version = "0.6.4"
 
 [[deps.LibCURL_jll]]
 deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
-version = "7.84.0+0"
+version = "8.4.0+0"
 
 [[deps.LibGit2]]
-deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
+deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
+[[deps.LibGit2_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
+uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.6.4+0"
+
 [[deps.LibSSH2_jll]]
 deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
 uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
-version = "1.10.2+0"
+version = "1.11.0+1"
 
 [[deps.Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
@@ -115,46 +120,46 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[deps.MbedTLS_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.2+0"
+version = "2.28.2+1"
 
 [[deps.Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [[deps.MozillaCACerts_jll]]
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2022.10.11"
+version = "2023.1.10"
 
 [[deps.NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 version = "1.2.0"
 
 [[deps.OrderedCollections]]
-git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3"
+git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.6.2"
+version = "1.6.3"
 
 [[deps.Parsers]]
 deps = ["Dates", "PrecompileTools", "UUIDs"]
-git-tree-sha1 = "a935806434c9d4c506ba941871b327b96d41f2bf"
+git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.8.0"
+version = "2.8.1"
 
 [[deps.Pkg]]
 deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.9.2"
+version = "1.10.0"
 
 [[deps.PrecompileTools]]
 deps = ["Preferences"]
-git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f"
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
 uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
-version = "1.2.0"
+version = "1.2.1"
 
 [[deps.Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
+git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.1"
+version = "1.4.3"
 
 [[deps.Printf]]
 deps = ["Unicode"]
@@ -165,7 +170,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
 uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [[deps.Random]]
-deps = ["SHA", "Serialization"]
+deps = ["SHA"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [[deps.SHA]]
@@ -189,9 +194,9 @@ uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 version = "1.10.0"
 
 [[deps.Tokenize]]
-git-tree-sha1 = "0454d9a9bad2400c7ccad19ca832a2ef5a8bc3a1"
+git-tree-sha1 = "5b5a892ba7704c0977013bd0f9c30f5d962181e0"
 uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
-version = "0.5.26"
+version = "0.5.28"
 
 [[deps.URIs]]
 git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b"
@@ -208,14 +213,14 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 [[deps.Zlib_jll]]
 deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.13+0"
+version = "1.2.13+1"
 
 [[deps.nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
-version = "1.48.0+0"
+version = "1.52.0+1"
 
 [[deps.p7zip_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "17.4.0+0"
+version = "17.4.0+2"

.dev/Manifest.toml

@@ -0,0 +1,57 @@
+ClimaCore.jl Release Notes
+========================
+
+v0.13.3
+-------
+
+- ![][badge-🚀performance] We now always inline for all ClimaCore kernels. PR [#1647](https://github.com/CliMA/ClimaCore.jl/pull/1647). This can result in more brittle inference (due to compiler heuristics). Technically, this is not a breaking change, but some code changes may be needed in practice.
+
+v0.13.2
+-------
+
+- ![][badge-🐛bugfix] fixed array allocation for interpolation on CPU.
+  PR [#1643](https://github.com/CliMA/ClimaCore.jl/pull/1643).
+
+v0.13.1
+-------
+
+- ![][badge-🐛bugfix] fixed edge case in interpolation that led to incorrect
+  vertical interpolation.
+  PR [#1640](https://github.com/CliMA/ClimaCore.jl/pull/1640).
+- ![][badge-🐛bugfix] fixed `interpolate!` for MPI runs.
+  PR [#1642](https://github.com/CliMA/ClimaCore.jl/pull/1642).
+
+
+v0.13.0
+-------
+- ![][badge-💥breaking] support for many deprecated methods have been dropped PR [#1632](https://github.com/CliMA/ClimaCore.jl/pull/1632).
+- ![][badge-🤖precisionΔ]![][badge-🚀performance] Slight performance improvement by replacing `rdiv` with `rmul`. PR ([#1496](https://github.com/CliMA/ClimaCore.jl/pull/1496)) Machine-precision differences are expected.
+- ![][badge-💥breaking]![][badge-🚀performance] Rewritten `distributed_remapping`. New `distributed_remapping` is non-allocating and up to 1000x faster (on GPUs). New `distributed_remapping` no longer supports the `physical_z` argument (this option is still available in `Remapping.interpolate_column`). New `interpolate!` function is available for remapping in-place. The new preferred way to define a `Rampper` is `Remapper(space, target_hcoords, target_zcoords)` (instead of `Remapper(target_hcoords, target_zcoords, space)`).
+  PR ([#1630](https://github.com/CliMA/ClimaCore.jl/pull/1630))
+
+v0.12.1
+-------
+- Started changelog
+- Fixed matrix field iterative solver tests.
+- ![][badge-🚀performance] Specialize on diagonal fieldvector broadcasts to skip uninferred `check_broadcast_axes` PR [#1615](https://github.com/CliMA/ClimaCore.jl/pull/1615), Issue [#1465](https://github.com/CliMA/ClimaCore.jl/issues/1465).
+- ![][badge-🚀performance] Fixed inference errors when not debugging PR [#1617](https://github.com/CliMA/ClimaCore.jl/pull/1617), Issue [#2597](https://github.com/CliMA/ClimaAtmos.jl/issues/2597).
+
+<!--
+
+Contributors are welcome to begin the description of changelog items with badge(s) below. Here is a brief description of when to use badges for a particular pull request / set of changes:
+
+ - 🔥behavioralΔ - behavioral changes. For example: a new model is used, yielding more accurate results.
+ - 🤖precisionΔ - machine-precision changes. For example, swapping the order of summed arguments can result in machine-precision changes.
+ - 💥breaking - breaking changes. For example: removing deprecated functions/types, removing support for functionality, API changes.
+ - 🚀performance - performance improvements. For example: improving type inference, reducing allocations, or code hoisting.
+ - ✨feature - new feature added. For example: adding support for a cubed-sphere grid
+ - 🐛bugfix - bugfix. For example: fixing incorrect logic, resulting in incorrect results, or fixing code that otherwise might give a `MethodError`.
+
+-->
+
+[badge-🔥behavioralΔ]: https://img.shields.io/badge/🔥behavioralΔ-orange.svg
+[badge-🤖precisionΔ]: https://img.shields.io/badge/🤖precisionΔ-black.svg
+[badge-💥breaking]: https://img.shields.io/badge/💥BREAKING-red.svg
+[badge-🚀performance]: https://img.shields.io/badge/🚀performance-green.svg
+[badge-✨feature/enhancement]: https://img.shields.io/badge/feature/enhancement-blue.svg
+[badge-🐛bugfix]: https://img.shields.io/badge/🐛bugfix-purple.svg

NEWS.md

@@ -1,7 +1,7 @@
 name = "ClimaCore"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
 authors = ["CliMA Contributors <clima-software@caltech.edu>"]
-version = "0.12.0"
+version = "0.13.3"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
@@ -39,8 +39,8 @@ KrylovExt = "Krylov"
 Adapt = "3, 4"
 BandedMatrices = "0.17, 1"
 BlockArrays = "0.16"
-CUDA = "4.2, 5"
-ClimaComms = "0.5.6"
+CUDA = "5"
+ClimaComms = "0.5.7"
 CubedSphere = "0.2"
 DataStructures = "0.18"
 DocStringExtensions = "0.8, 0.9"

Project.toml

@@ -81,7 +81,7 @@ function initialize_mwe(device, ::Type{FT}) where {FT}
     z_domain = Domains.IntervalDomain(
         Geometry.ZPoint(zero(z_max)),
         Geometry.ZPoint(z_max);
-        boundary_tags = (:bottom, :top),
+        boundary_names = (:bottom, :top),
     )
     z_mesh = Meshes.IntervalMesh(z_domain, nelems = z_elem)
     z_topology = Topologies.IntervalTopology(context, z_mesh)

benchmarks/3d/se_kernels.jl

@@ -11,21 +11,19 @@ using ClimaCore:
     Quadratures
 using CUDA, BenchmarkTools
 
+context = ClimaComms.context()
 hdomain = Domains.SphereDomain(6.37122e6)
 hmesh = Meshes.EquiangularCubedSphere(hdomain, 30)
-htopology = Topologies.Topology2D(hmesh)
+htopology = Topologies.Topology2D(context, hmesh)
 hspace = Spaces.SpectralElementSpace2D(htopology, Quadratures.GLL{4}())
 
 vdomain = Domains.IntervalDomain(
     Geometry.ZPoint(0.0),
     Geometry.ZPoint(10e3);
-    boundary_tags = (:bottom, :top),
+    boundary_names = (:bottom, :top),
 )
 vmesh = Meshes.IntervalMesh(vdomain; nelems = 45)
-vtopology = Topologies.IntervalTopology(
-    ClimaComms.SingletonCommsContext(ClimaComms.device()),
-    vmesh,
-)
+vtopology = Topologies.IntervalTopology(context, vmesh)
 vspace = Spaces.CenterFiniteDifferenceSpace(vtopology)
 
 cspace = Spaces.ExtrudedFiniteDifferenceSpace(hspace, vspace)