diff --git a/Project.toml b/Project.toml index 60368e38..fd3d58b8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Kinematic1D" uuid = "a8a3bb05-6cc5-4342-abf6-5cc636bd2b35" authors = ["Climate Modeling Alliance"] -version = "0.3.0" +version = "0.3.1" [deps] ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884" @@ -30,7 +30,7 @@ ClimaCorePlots = "0.2" CloudMicrophysics = "0.10" DiffEqBase = "6.75" Distributions = "0.25" -EnsembleKalmanProcesses = "0.13" +EnsembleKalmanProcesses = "1.1" Interpolations = "0.14" JLD2 = "0.4" NCDatasets = "0.12" diff --git a/src/calibrateKiD/Diagnostics.jl b/src/calibrateKiD/Diagnostics.jl index a938fba5..a7a87ade 100644 --- a/src/calibrateKiD/Diagnostics.jl +++ b/src/calibrateKiD/Diagnostics.jl @@ -10,21 +10,21 @@ Returns liquid water path [kg/m^2] """ function lwp(vec::Vector{FT}, config) where {FT <: Real} - @assert issubset(Set(["rho", "rl"]), Set(config["observations"]["data_names"])) + @assert issubset(Set(["rho", "ql"]), Set(config["observations"]["data_names"])) (n_c, n_v, n_z, n_t) = get_numbers_from_config(config) n_vht = n_v * n_z * n_t dz = (config["model"]["z_max"] - config["model"]["z_min"]) / n_z lwp = zeros(n_t, n_c) ind_ρ = findall(x -> x == "rho", config["observations"]["data_names"])[1] - ind_rl = findall(x -> x == "rl", config["observations"]["data_names"])[1] + ind_ql = findall(x -> x == "ql", config["observations"]["data_names"])[1] for i in 1:n_c v_ = vec[((i - 1) * n_vht + 1):(i * n_vht)] m_ = reshape(v_, n_v * n_z, n_t) ρ = m_[((ind_ρ - 1) * n_z + 1):(ind_ρ * n_z), :] - rl = m_[((ind_rl - 1) * n_z + 1):(ind_rl * n_z), :] - lwp[:, i] = sum(rl .* ρ, dims = 1) .* dz + ql = m_[((ind_ql - 1) * n_z + 1):(ind_ql * n_z), :] + lwp[:, i] = sum(ql .* ρ, dims = 1) .* dz end return lwp end @@ -42,21 +42,21 @@ Returns rain water path [kg/m^2] """ function rwp(vec::Vector{FT}, config) where {FT <: Real} - @assert issubset(Set(["rho", "rr"]), Set(config["observations"]["data_names"])) + @assert issubset(Set(["rho", "qr"]), Set(config["observations"]["data_names"])) (n_c, n_v, n_z, n_t) = get_numbers_from_config(config) n_vht = n_v * n_z * n_t dz = (config["model"]["z_max"] - config["model"]["z_min"]) / n_z rwp = zeros(n_t, n_c) ind_ρ = findall(x -> x == "rho", config["observations"]["data_names"])[1] - ind_rr = findall(x -> x == "rr", config["observations"]["data_names"])[1] + ind_qr = findall(x -> x == "qr", config["observations"]["data_names"])[1] for i in 1:n_c v_ = vec[((i - 1) * n_vht + 1):(i * n_vht)] m_ = reshape(v_, n_v * n_z, n_t) ρ = m_[((ind_ρ - 1) * n_z + 1):(ind_ρ * n_z), :] - rr = m_[((ind_rr - 1) * n_z + 1):(ind_rr * n_z), :] - rwp[:, i] = sum(rr .* ρ, dims = 1) .* dz + qr = m_[((ind_qr - 1) * n_z + 1):(ind_qr * n_z), :] + rwp[:, i] = sum(qr .* ρ, dims = 1) .* dz end return rwp end @@ -77,37 +77,44 @@ Returns rain rate [mm/hr] """ function rainrate(vec::Vector{FT}, config; height::FT = FT(0), isref = false) where {FT <: Real} - @assert issubset(Set(["rho", "rr", "rain averaged terminal velocity"]), Set(config["observations"]["data_names"])) + @assert issubset(Set(["rho", "qr", "rain averaged terminal velocity"]), Set(config["observations"]["data_names"])) (n_c, n_v, n_z, n_t) = get_numbers_from_config(config) n_vht = n_v * n_z * n_t dz = (config["model"]["z_max"] - config["model"]["z_min"]) / n_z rainrate = zeros(n_t, n_c) ind_ρ = findall(x -> x == "rho", config["observations"]["data_names"])[1] - ind_rr = findall(x -> x == "rr", config["observations"]["data_names"])[1] + ind_qr = findall(x -> x == "qr", config["observations"]["data_names"])[1] ind_vt = findall(x -> x == "rain averaged terminal velocity", config["observations"]["data_names"])[1] - ind_z = min(n_z, max(1, ceil(Int, (height - config["model"]["z_min"]) / dz))) + ind_z1 = min(n_z - 1, max(1, ceil(Int, (height - config["model"]["z_min"]) / dz - FT(0.5)))) + ind_z2 = ind_z1 + 1 + + a1 = ((ind_z2 - 0.5) * dz - height) / dz + a2 = (height - (ind_z1 - 0.5) * dz) / dz for i in 1:n_c v_ = vec[((i - 1) * n_vht + 1):(i * n_vht)] m_ = reshape(v_, n_v * n_z, n_t) ρ = m_[((ind_ρ - 1) * n_z + 1):(ind_ρ * n_z), :] - rr = m_[((ind_rr - 1) * n_z + 1):(ind_rr * n_z), :] + qr = m_[((ind_qr - 1) * n_z + 1):(ind_qr * n_z), :] - ρ_z = ρ[ind_z, :] - rr_z = rr[ind_z, :] + ρ_z = a1 .* ρ[ind_z1, :] + a2 .* ρ[ind_z2, :] + ρ_z[findall(x -> x < 0, ρ_z)] .= FT(0) + qr_z = a1 .* qr[ind_z1, :] + a2 .* qr[ind_z2, :] + qr_z[findall(x -> x < 0, qr_z)] .= FT(0) if isref vt = m_[((ind_vt - 1) * n_z + 1):(ind_vt * n_z), :] - vt_z = vt[ind_z, :] + vt_z = a1 .* vt[ind_z1, :] + a2 .* vt[ind_z2, :] + vt_z[findall(x -> x < 0, vt_z)] .= FT(0) else ϕ_names = collect(keys(config["prior"]["parameters"])) ϕ_values = collect([v.mean for v in values(config["prior"]["parameters"])]) params_calib = create_param_dict(ϕ_values, ϕ_names) params = create_parameter_set(Float64, config["model"], params_calib) microphys_params = KP.microphysics_params(params) - vt_z = CM1.terminal_velocity.(microphys_params, CMT.RainType(), ρ_z, rr_z) + vt_z = CM1.terminal_velocity.(microphys_params, CMT.RainType(), ρ_z, qr_z) end - rainrate[:, i] = rr_z .* ρ_z .* vt_z .* 3600 + rainrate[:, i] = qr_z .* ρ_z .* vt_z .* 3600 end return rainrate end diff --git a/src/calibrateKiD/DistributionUtils.jl b/src/calibrateKiD/DistributionUtils.jl index 58673a6c..e5405eeb 100644 --- a/src/calibrateKiD/DistributionUtils.jl +++ b/src/calibrateKiD/DistributionUtils.jl @@ -1,6 +1,6 @@ function initial_parameter_ensemble(priors, N_ensemble; rng_seed = 10) Random.seed!(rng_seed) - initial_ensemble = construct_initial_ensemble(priors, N_ensemble; rng_seed = rng_seed) + initial_ensemble = construct_initial_ensemble(priors, N_ensemble) return initial_ensemble end diff --git a/src/calibrateKiD/KiDUtils.jl b/src/calibrateKiD/KiDUtils.jl index 06a5d413..3eeaa600 100644 --- a/src/calibrateKiD/KiDUtils.jl +++ b/src/calibrateKiD/KiDUtils.jl @@ -279,17 +279,42 @@ function create_parameter_set(FT, model_settings::Dict, params_cal::Dict) CM.Parameters.CloudMicrophysicsParameters{FT, TP}(; fixed_microphys_pairs..., pairs..., thermo_params) MP = typeof(microphys_params) + precip_sources = if ("precip_sources" in keys(model_settings)) + model_settings["precip_sources"] + else + 1 + end + precip_sinks = if ("precip_sinks" in keys(model_settings)) + model_settings["precip_sinks"] + else + 1 + end + r_dry = if ("r_dry" in keys(model_settings)) + model_settings["r_dry"] + else + 0.04 * 1e-6 + end + std_dry = if ("std_dry" in keys(model_settings)) + model_settings["std_dry"] + else + 1.4 + end + κ = if ("κ" in keys(model_settings)) + model_settings["κ"] + else + 0.9 + end param_set = KP.KinematicParameters{FT, MP}(; w1 = model_settings["w1"], t1 = model_settings["t1"], p0 = model_settings["p0"], - precip_sources = 1, - precip_sinks = 1, + precip_sources = precip_sources, + precip_sinks = precip_sinks, prescribed_Nd = model_settings["Nd"], qtot_flux_correction = Int(model_settings["qtot_flux_correction"]), - r_dry = model_settings["r_dry"], - std_dry = model_settings["std_dry"], - κ = model_settings["κ"], + r_dry = r_dry, + std_dry = std_dry, + κ = κ, microphys_params, ) return param_set diff --git a/src/calibrateKiD/OptimizationUtils.jl b/src/calibrateKiD/OptimizationUtils.jl index 2e3deb4e..6edf4cf1 100644 --- a/src/calibrateKiD/OptimizationUtils.jl +++ b/src/calibrateKiD/OptimizationUtils.jl @@ -105,7 +105,13 @@ end function generate_ekp(param_ensemble::Matrix{FT}, RS::ReferenceStatistics, process_settings) where {FT <: Real} if process_settings["EKP_method"] == "EKI" - ekpobj = EnsembleKalmanProcess(param_ensemble, RS.y, RS.Γ, Inversion(), Δt = process_settings["Δt"]) + ekpobj = EnsembleKalmanProcess( + param_ensemble, + RS.y, + RS.Γ, + Inversion(), + scheduler = DefaultScheduler(process_settings["Δt"]), + ) elseif process_settings["EKP_method"] == "UKI" process = Unscented( mean(param_ensemble, dims = 2)[:], @@ -113,7 +119,7 @@ function generate_ekp(param_ensemble::Matrix{FT}, RS::ReferenceStatistics, proce α_reg = process_settings["α_reg"], update_freq = process_settings["update_freq"], ) - ekpobj = EnsembleKalmanProcess(RS.y, RS.Γ, process, Δt = process_settings["Δt"]) + ekpobj = EnsembleKalmanProcess(RS.y, RS.Γ, process, scheduler = DefaultScheduler(process_settings["Δt"])) end return ekpobj end diff --git a/test/unit_tests/calibration_pipeline/config.jl b/test/unit_tests/calibration_pipeline/config.jl index 1d2fde6e..2cc5b3d9 100644 --- a/test/unit_tests/calibration_pipeline/config.jl +++ b/test/unit_tests/calibration_pipeline/config.jl @@ -26,11 +26,11 @@ end function get_process_config() config = Dict() config["batch_size"] = 1 - config["n_iter"] = 10 + config["n_iter"] = 5 config["n_ens"] = 10 config["Δt"] = 1.0 config["EKP_method"] = "EKI" - config["α_reg"] = 0.5 + config["α_reg"] = 1.0 config["update_freq"] = 1 config["tol"] = 1e-8 config["maxiter"] = 1000 diff --git a/test/unit_tests/calibration_pipeline/test_diagnostics.jl b/test/unit_tests/calibration_pipeline/test_diagnostics.jl index 387fb0b3..b76906c2 100644 --- a/test/unit_tests/calibration_pipeline/test_diagnostics.jl +++ b/test/unit_tests/calibration_pipeline/test_diagnostics.jl @@ -1,7 +1,7 @@ @testset "lwp rwp and rainrate" begin #setup config = get_config() - config["observations"]["data_names"] = ["rho", "rl", "rr", "rain averaged terminal velocity"] + config["observations"]["data_names"] = ["rho", "ql", "qr", "rain averaged terminal velocity"] (n_c, n_v, n_z, n_t) = KID.get_numbers_from_config(config) n_vht = n_v * n_z * n_t vec = rand(n_c * n_vht) @@ -14,14 +14,17 @@ v_ = vec[((i - 1) * n_vht + 1):(i * n_vht)] m_ = reshape(v_, n_v * n_z, n_t) ρ = m_[1:n_z, :] - rl = m_[(n_z + 1):(2 * n_z), :] - rr = m_[(2 * n_z + 1):(3 * n_z), :] - _lwp[:, i] = sum(rl .* ρ, dims = 1) .* dz - _rwp[:, i] = sum(rr .* ρ, dims = 1) .* dz - ρ0 = ρ[1, :] - rr0 = rr[1, :] - vt0 = m_[3 * n_z + 1, :] - _rainrate[:, i] = rr0 .* ρ0 .* vt0 .* 3600 + ql = m_[(n_z + 1):(2 * n_z), :] + qr = m_[(2 * n_z + 1):(3 * n_z), :] + _lwp[:, i] = sum(ql .* ρ, dims = 1) .* dz + _rwp[:, i] = sum(qr .* ρ, dims = 1) .* dz + ρ0 = 1.5 * ρ[1, :] - 0.5 * ρ[2, :] + qr0 = 1.5 * qr[1, :] - 0.5 * qr[2, :] + vt0 = 1.5 * m_[3 * n_z + 1, :] - 0.5 * m_[3 * n_z + 2, :] + ρ0[findall(x -> x < 0, ρ0)] .= Float64(0) + qr0[findall(x -> x < 0, qr0)] .= Float64(0) + vt0[findall(x -> x < 0, vt0)] .= Float64(0) + _rainrate[:, i] = qr0 .* ρ0 .* vt0 .* 3600 end #action diff --git a/test/unit_tests/calibration_pipeline/test_optimization_utils.jl b/test/unit_tests/calibration_pipeline/test_optimization_utils.jl index 7d14f663..df5dbd3a 100644 --- a/test/unit_tests/calibration_pipeline/test_optimization_utils.jl +++ b/test/unit_tests/calibration_pipeline/test_optimization_utils.jl @@ -23,7 +23,7 @@ @test size(res[1]) == (n_vars, n_ensemble) @test u.ϕ_optim isa Vector @test u.cov_optim isa Matrix - @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.05 + @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.1 #setup config["process"]["EKP_method"] = "UKI" @@ -39,7 +39,7 @@ @test size(res[1]) == (n_vars, 2 * n_vars + 1) @test u.ϕ_optim isa Vector @test u.cov_optim isa Matrix - @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.05 + @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.1 #setup config["process"]["EKP_method"] = "UKP_" @@ -81,12 +81,10 @@ end u = KID.get_results(res, priors) #test - @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.05 + @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.1 #setup config["process"]["EKP_method"] = "UKI" - config["process"]["batch_size"] = 1 - config["process"]["α_reg"] = 0.1 config["process"]["Δt"] = 10.0 #action @@ -94,7 +92,7 @@ end u = KID.get_results(res, priors) #test - @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.05 + @test norm((u.ϕ_optim .- u_true) ./ u_true) < 0.1 end @testset "Calibrate by Optim and Get results" begin