Skip to content

Commit

Permalink
Dataset timestamp col mapping (#149)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jami159 authored Dec 5, 2023
1 parent 7400ab4 commit 7ddfed9
Showing 1 changed file with 13 additions and 2 deletions.
15 changes: 13 additions & 2 deletions src/SimulationService.jl
Original file line number Diff line number Diff line change
Expand Up @@ -412,14 +412,25 @@ end
function get_dataset(obj::JSON3.Object)
@assert ENABLE_TDS[]
@info "get_dataset with obj = $(JSON3.write(obj))"

tds_url = "$(TDS_URL[])/datasets/$(obj.id)/download-url?filename=$(obj.filename)"
s3_url = get_json(tds_url).url
df = CSV.read(download(s3_url), DataFrame)

# there should always be a mapping from the dataset timestamp column name to 'timestamp'
if !any(v -> v == "timestamp", values(obj.mappings))
@error "Expected mapping from <dataset timestamp column> to 'timestamp' not provided"
end

for (k,v) in get(obj, :mappings, Dict())
@info "`get_dataset` (dataset id=$(repr(obj.id))) rename! $k => $v"
rename!(df, k => v)
if k != "tstep"
rename!(df, k => v)
else
rename!(df, v => "timestamp") # hack to get df in our "schema"
end
end
obj.mappings["tstep"] in names(df) && rename!(df, obj.mappings["tstep"] => "timestamp") # hack to get df in our "schema"

@info "get_dataset (id=$(repr(obj.id))) with names: $(names(df))"
return df
end
Expand Down

0 comments on commit 7ddfed9

Please sign in to comment.