Fixes for EarthDataLab tests (#282)

* Allow some niceties when subsetting * Dont wrap yaxarrays into yaxarrays * fix zarr output * Changes to cubetable --------- Co-authored-by: Felix Cremer <[email protected]> Co-authored-by: Fabian Gans <[email protected]>
JuliaDataCubes · Jul 6, 2023 · a0f5246 · a0f5246 · felixcremer · Jul 6, 2023
1 parent 1d1e1ab
commit a0f5246
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 14 deletions.
diff --git a/src/Cubes/Cubes.jl b/src/Cubes/Cubes.jl
@@ -5,10 +5,10 @@ Data types that
 module Cubes
 using DiskArrays: DiskArrays, eachchunk, approx_chunksize, max_chunksize, grid_offset, GridChunks
 using Distributed: myid
-using Dates: TimeType
+using Dates: TimeType, Date
 using IntervalSets: Interval, (..)
 using Base.Iterators: take, drop
-using ..YAXArrays: workdir, YAXDefaults, findAxis
+using ..YAXArrays: workdir, YAXDefaults, findAxis, getAxis
 using YAXArrayBase: YAXArrayBase, iscompressed, dimnames, iscontdimval
 import YAXArrayBase: getattributes, iscontdim, dimnames, dimvals, getdata
 using DiskArrayTools: CFDiskArray
@@ -254,7 +254,7 @@ of this chunking, use `savecube` on the resulting array. The `chunks` argument c
 setchunks(c::YAXArray,chunks) = YAXArray(c.axes,c.data,c.properties,interpret_cubechunks(chunks,c),c.cleaner)
 cubechunks(c) = approx_chunksize(eachchunk(c))
 DiskArrays.eachchunk(c::YAXArray) = c.chunks
-getindex_all(a) = getindex(a, ntuple(_ -> Colon(), ndims(a))...)
+getindex_all(a) = getindex(a, ntuple(_ -> Colon(), ndims(a))...).data
 
 #=
 function Base.getindex(x::YAXArray, i...) 
@@ -455,7 +455,29 @@ function _subsetcube(z, subs; kwargs...)
 end
 
 
-Base.getindex(a::YAXArray, args::DD.Dimension...; kwargs...) = view(a, args...; kwargs...)
+function Base.getindex(a::YAXArray, args::DD.Dimension...; kwargs...) 
+    kwargsdict = Dict(kwargs...)
+    for ext in YAXDefaults.subsetextensions
+        ext(kwargsdict)
+    end
+    d2 = Dict()
+    for (k,v) in kwargsdict
+        d = getAxis(k,a)
+        if d !== nothing
+            if d isa DD.Ti
+                if v isa UnitRange{Int}
+                    v = Date(first(v))..Date(last(v),12,31)
+                end
+                d2[:Ti] = v
+            else
+                d2[DD.name(d)] = v
+            end
+        else
+            d2[k] = v
+        end
+    end
+    view(a, args...; d2...)
+end
 
 Base.read(d::YAXArray) = getindex_all(d)
 
@@ -474,6 +496,7 @@ cubesize(::YAXArray{T,0}) where {T} = sizeof(T)
 getCubeDes(::DD.Dimension) = "Cube axis"
 getCubeDes(::YAXArray) = "YAXArray"
 getCubeDes(::Type{T}) where {T} = string(T)
+
 function DD.show_after(io::IO,mime, c::YAXArray)
     foreach(getattributes(c)) do p
         if p[1] in ("labels", "name", "units")

diff --git a/src/DAT/dciterators.jl b/src/DAT/dciterators.jl
@@ -147,11 +147,11 @@ function CubeTable(; expandaxes = (), cubes...)
             axn = filter(collect(expandaxes)) do ax
                 findAxis(ax, i) !== nothing
             end
-            foreach(j -> push!(inaxnames, DD.name(getAxis(j, i))), axn)
+            foreach(j -> push!(inaxnames, string(DD.name(getAxis(j, i)))), axn)
             InDims(axn...)
         end
     end
-    axnames = map(i -> DD.name.(caxes(i)), c)
+    axnames = map(i -> string.(DD.name.(caxes(i))), c)
     foreach(1:length(axnames)) do i
         otheraxes = axnames[[1:i-1;i+1:length(axnames)]]
         if !isempty(otheraxes) && isempty(intersect(axnames[i], union(otheraxes...)))
@@ -160,7 +160,6 @@ function CubeTable(; expandaxes = (), cubes...)
     end
     allvars = union(axnames...)
     allnums = collect(1:length(allvars))
-
     configiter =
         mapCube(identity, c, debug = true, indims = indims, outdims = (), ispar = false)
     # if inax !== nothing

diff --git a/src/DAT/tablestats.jl b/src/DAT/tablestats.jl
@@ -180,12 +180,11 @@ function getStatOutAxes(tab, agg, ::Type{<:WeightedCovMatrix})
     s = varsym(agg)
     icube = findfirst(isequal(s), varn)
     ax = tab.dc.incubes[icube].axesSmall[1]
-    oldname = YAXArrays.Cubes.Axes.axname(ax)
+    oldname = DD.name(ax)
     coname = string("Co", oldname)
-    v = ax.values
-    axtype = axt(ax)
-    a1 = axtype(oldname, copy(v))
-    a2 = axtype(coname, copy(v))
+    v = ax.val
+    a1 = DD.Dim{oldname}(copy(v))
+    a2 = DD.Dim{Symbol(coname)}(copy(v))
     (a1, a2)
 end
 function getStatOutAxes(tab,agg,::Type{<:Union{Ash,HistogramStat, WeightedAdaptiveHist}})

diff --git a/src/DatasetAPI/Datasets.jl b/src/DatasetAPI/Datasets.jl
@@ -562,6 +562,7 @@ function savedataset(
     chunkoffset = [alloffsets[k] for k in keys(ds.axes)]
     axdata = arrayfromaxis.(axesall, chunkoffset)
 
+
 
     dshandle = if ispath(path)
         # We go into append mode
@@ -734,7 +735,7 @@ function createdataset(
             if hasmissings
                 v = CFDiskArray(v, attr)
             end
-            YAXArray(axlist, v, propfromattr(attr), cleaner = cleaner)
+            YAXArray((axlist...,), v, propfromattr(attr), cleaner = cleaner)
         end
         if groupaxis === nothing
             return allcubes[1], allcubes[1]
@@ -804,7 +805,7 @@ function createdataset(
     # function dataattfromaxis(ax::CubeAxis,n)
     #     prependrange(1:length(ax.values),n), Dict{String,Any}("_ARRAYVALUES"=>collect(ax.values))
     # end
-    function dataattfromaxis(ax::DD.Dimensions.Ti, n, T::Type{<:TimeType})
+    function dataattfromaxis(ax::DD.Dimension, n, T::Type{<:TimeType})
         data = timeencode(datetodatetime(DD.lookup(ax)), "days since 1980-01-01", defaultcal(T))
         prependrange(data, n),
         Dict{String,Any}("units" => "days since 1980-01-01", "calendar" => defaultcal(T))