Merge branch 'jd/dev' into data

src/SimulationService.jl

@@ -109,7 +109,7 @@ RABBITMQ_PORT        = parse(Int, get(ENV, "SIMSERVICE_RABBITMQ_PORT", "5672"))
 JSON_HEADER = ["Content-Type" => "application/json"]
 
 # Get Config object from JSON at `url`
-get_json(url::String)::Config = JSON3.read(HTTP.get(url, JSON_HEADER).body, Config)
+get_json(url::String, T=Config) = JSON3.read(HTTP.get(url, JSON_HEADER).body, T)
 
 
 #-----------------------------------------------------------------------------# amr_get
@@ -181,6 +181,7 @@ end
 
 # data
 function amr_get(df::DataFrame, sys::ODESystem, ::Val{:data})
+
     statelist = states(sys)
     statenames = string.(statelist)
     statenames = map(statenames) do n; n[1:end-3]; end # there's a better way to do this
@@ -269,17 +270,19 @@ end
 #-----------------------------------------------------------------------------# DataServiceModel
 # https://raw.githubusercontent.com/DARPA-ASKEM/simulation-api-spec/main/schemas/simulation.json
 # How a model/simulation is represented in TDS
+# you can HTTP.get("$TDS_URL/simulation") and the JSON3.read(body, DataServiceModel)
 Base.@kwdef mutable struct DataServiceModel
     # Required
     id::String = "UNINITIALIZED_ID"         # matches with our `id`
-    engine::Symbol = :sciml                 # (ignore) TDS supports multiple engine.  We are the `sciml` engine.
-    type::Symbol = :uninitialized           # :simulate, :calibrate, etc.
+    engine::String = "sciml"                # (ignore) TDS supports multiple engine.  We are the `sciml` engine.
+    type::String = "uninitialized"          # :calibration, :calibration_simulation, :ensemble, :simulation
     execution_payload::Config = Config()    # untouched JSON from request sent by HMI
     workflow_id::String = "IGNORED"         # (ignore)
     # Optional
-    timestamp::Union{Nothing, DateTime} = nothing           # (ignore?)
+    description::String = ""                                # (ignore)
+    timestamp::Union{Nothing, DateTime} = nothing             # (ignore?)
     result_files::Union{Nothing, Vector{String}} = nothing  # URLs of result files in S3
-    status::Union{Nothing, Symbol} = :queued                # queued|running|complete|error|cancelled|failed"
+    status::Union{Nothing, String} = "queued"                # queued|running|complete|error|cancelled|failed"
     reason::Union{Nothing, String} = nothing                # why simulation failed (returned junk results)
     start_time::Union{Nothing, DateTime} = nothing          # when job started
     completed_time::Union{Nothing, DateTime} = nothing      # when job completed
@@ -291,12 +294,17 @@ end
 StructTypes.StructType(::Type{DataServiceModel}) = StructTypes.Mutable()
 
 # Initialize a DataServiceModel
+operation_to_dsm_type = Dict(
+    :simulate => "simulation",
+    :calibrate => "calibration_simulation",
+    :ensemble => "ensemble"
+)
+
 function DataServiceModel(o::OperationRequest)
     m = DataServiceModel()
     m.id = o.id
-    m.type = o.operation
+    m.type = operation_to_dsm_type[o.operation]
     m.execution_payload = o.obj
-    m.timestamp = now(UTC)
     return m
 end
 
@@ -312,8 +320,8 @@ function DataServiceModel(id::String)
     delays = fill(1, TDS_RETRIES)
 
     try
-        m = retry(() -> JSON3.read(download("$TDS_URL/simulations/$id")); delays, check)()
-        return m
+        res = retry(() -> HTTP.get("$TDS_URL/simulations/$id"); delays, check)()
+        return JSON3.read(res.body, DataServiceModel)
     catch
         return error("No simulation found in TDS with id=$id.")
     end
@@ -359,20 +367,19 @@ function create(o::OperationRequest)
          @warn "TDS disabled - create with JSON $body"
          return body
     end
-    HTTP.post("$TDS_URL/simulations/$(o.id)", JSON_HEADER; body)
+    HTTP.post("$TDS_URL/simulations/", JSON_HEADER; body)
 end
 
 # update the DataServiceModel in TDS: PUT /simulations/{id}
+# kw args and their types much match field::fieldtype in DataServiceModel
 function update(o::OperationRequest; kw...)
     if !ENABLE_TDS
         @warn "TDS disabled - update OperationRequest with id=$(o.id), $kw"
         return kw
     end
     m = DataServiceModel(o.id)
     for (k,v) in kw
-        hasfield(m, k) ?
-            setproperty!(o, k, v) :
-            @warn "Skipping `update` of unrecognized field: $k = $v."
+        setproperty!(m, k, v)
     end
     HTTP.put("$TDS_URL/simulations/$(o.id)", JSON_HEADER; body=JSON3.write(m))
 end
@@ -401,7 +408,7 @@ function complete(o::OperationRequest)
     tds_url = "$TDS_URL/simulations/sciml-$(o.id)/upload-url?filename=$filename)"
     s3_url = get_json(tds_url).url
     HTTP.put(s3_url, header; body=body)
-    update(o; status = :complete, completed_time = Dates.now(UTC), result_files = [s3_url])
+    update(o; status = "complete", completed_time = Dates.now(), result_files = [s3_url])
 end
 
 
@@ -429,7 +436,7 @@ function operation(request::HTTP.Request, operation_name::String)
         @task begin
             # try
                 @info "Updating job (id=$(o.id))...)"
-                update(o; status = :running, start_time = Dates.now(UTC)) # 4
+                update(o; status = "running", start_time = Dates.now()) # 4
                 @info "Solving job (id=$(o.id))..."
                 solve(o) # 5
                 @info "Completing job (id=$(o.id))...)"

test/runtests.jl

@@ -57,10 +57,10 @@ end
 #-----------------------------------------------------------# DataServiceModel and OperationRequest
 @testset "DataServiceModel and OperationRequest" begin
     m = DataServiceModel()
-    @test m.engine == :sciml
+    @test m.engine == "sciml"
     @test m.id == "UNINITIALIZED_ID"
 
-    o = OperationRequest()
+    o = OperationRequest(operation = :simulate)
     m2 = DataServiceModel(o)
 
     # OperationRequest constructor with dummy HTTP.Request
@@ -171,11 +171,3 @@ end
 
     stop!()
 end
-
-#-----------------------------------------------------------------------------# TDS Enable
-@testset "TDS Enabled" begin
-
-    if SimulationService.ENABLE_TDS
-
-    end
-end

test/tds.jl

@@ -1,8 +1,92 @@
-using SimulationService
-using Test
+# This only works if you are using the TDS instance set up by Brandon Rose.
+# Talk to Brandon or Josh Day for information.
+
+# Docs: $TDS_URL/#/Simulation/simulation_delete_simulations__simulation_id__delete
+
+using SimulationService: get_model, get_dataset, create, update, complete, get_json, solve,
+    DataServiceModel, OperationRequest, TDS_URL, TDS_RETRIES, ENABLE_TDS
+
+using HTTP, JSON3, DataFrames, EasyConfig, Test
 
 SimulationService.ENABLE_TDS = true
 
-json = @config(model_config_id = "2b08c681-ee0e-4ad1-81d5-e0e3e203ffbe", timespan.start=0.0, timespan.end=100.0)
-body = JSON3.write(json)
-res = HTTP.post("$url/simulate",  ["Content-Type" => "application/json"]; body)
+#-----------------------------------------------------------------------------# Check that TDS is running
+get_simulations() = get_json("$TDS_URL/simulations", Vector{Config})
+simulations = get_simulations()
+for sim in simulations
+    res = HTTP.delete("$TDS_URL/simulations/$(sim.id)")
+    @test res.status == 200
+end
+sleep(2)
+simulations = get_simulations()
+@test isempty(simulations)
+
+#-----------------------------------------------------------------------------# get_model ✓
+model_config_id = "2b08c681-ee0e-4ad1-81d5-e0e3e203ffbe"
+obj = get_model(model_config_id)
+@test obj.id == model_config_id
+
+#-----------------------------------------------------------------------------# get_dataset ???
+# TODO
+
+#-----------------------------------------------------------------------------# create ✓
+# mock request to SimulationService
+payload = @config(model_config_id=model_config_id, timespan.start=1, timespan.end=100, engine="sciml")
+req = HTTP.Request("POST", "", ["Content-Type" => "application/json"], JSON3.write(payload))
+o = OperationRequest(req, "simulate")
+id = o.id
+
+res = create(o)  # Create TDS representation of model
+@test JSON3.read(res.body).id == id
+
+# Check that simulation with `id` now exists in TDS
+sleep(2)
+@test HTTP.get("$TDS_URL/simulations/$id").status == 200
+
+#-----------------------------------------------------------------------------# DataServiceModel ✓
+m = DataServiceModel(id)
+@test m.id == id
+
+#-----------------------------------------------------------------------------# update ✓
+res = update(o; status = "running")
+@test res.status == 200
+@test JSON3.read(res.body).id == id
+
+#-----------------------------------------------------------------------------# complete ✓
+@test_throws "solve(" complete(o)
+
+o.result = DataFrame(fake=1:10, results=randn(10))
+
+res = complete(o)
+@test res.status == 200
+@test JSON3.read(res.body).id == id
+
+
+#-----------------------------------------------------------------------------# ALL TOGETHER NOW ???
+SimulationService.operation(req, "simulate")
+
+#-----------------------------------------------------------------------------# with server ???
+start!()
+
+url = SimulationService.server_url[]
+
+sleep(3) # Give server a chance to start
+
+@testset "/" begin
+    res = HTTP.get(url)
+    @test res.status == 200
+    @test JSON3.read(res.body).status == "ok"
+end
+
+
+res = HTTP.post("$url/simulate", ["Content-Type" => "application/json"]; body)
+@test res.status == 201
+id = JSON3.read(res.body).simulation_id
+done_or_failed = false
+while !done_or_failed
+    st = JSON3.read(HTTP.get("$url/status/$id").body).status
+    st in ["queued", "complete", "running"] ? @test(true) : @test(false)
+    done_or_failed = st in ["complete", "error"]
+    sleep(1)
+end
+@test SimulationService.last_operation[].result isa DataFrame