diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2c370e3..1ae113b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +### 1.4.0 +- KdTree loading now parametrized and allows to create kdtrees + ### 1.3.0 - updated to Aardvark.Rendering 5.4 - [Rabbyte] simplified model and actions by using dropdownUnclearable diff --git a/src/2D3DLinking/App.fs b/src/2D3DLinking/App.fs index 3b33672..b751765 100644 --- a/src/2D3DLinking/App.fs +++ b/src/2D3DLinking/App.fs @@ -277,10 +277,11 @@ module App = for h in patchHierarchies do let rootTree = h.tree |> QTree.getRoot + let kd = (KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer false) false (fun _ -> failwith "no function for creating triangle sets") yield { patchHierarchy = h - kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath (KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer) + kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath kd localBB = rootTree.info.LocalBoundingBox globalBB = rootTree.info.GlobalBoundingBox neighborMap = HashMap.empty diff --git a/src/3DGis/App.fs b/src/3DGis/App.fs index 408b993..03f0de0 100644 --- a/src/3DGis/App.fs +++ b/src/3DGis/App.fs @@ -830,10 +830,11 @@ module App = for h in patchHierarchies do let rootTree = h.tree |> QTree.getRoot - + let kd = KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer false false (fun _ _ -> failwith "no triangleset function") + yield { patchHierarchy = h - kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath (KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer) + kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath kd localBB = rootTree.info.LocalBoundingBox globalBB = rootTree.info.GlobalBoundingBox neighborMap = HashMap.empty diff --git a/src/OPCViewer.Base/KdTrees.fs b/src/OPCViewer.Base/KdTrees.fs index c007601..e9a7869 100644 --- a/src/OPCViewer.Base/KdTrees.fs +++ b/src/OPCViewer.Base/KdTrees.fs @@ -1,210 +1,261 @@ namespace Aardvark.VRVis.Opc - +// copy of https://raw.githubusercontent.com/aardvark-platform/OpcViewer/master/src/OPCViewer.Base/KdTrees.fs +// should be consolidated open System.IO open Aardvark.Geometry open Aardvark.Base open Aardvark.Base.Coder open Aardvark.SceneGraph.Opc -open MBrace.FsPickler -open MBrace.FsPickler.Combinators +open MBrace.FsPickler +open MBrace.FsPickler.Combinators open FSharp.Data.Adaptive +open System.Collections.Generic + +module KdTrees = + + type LazyKdTree = + { kdTree: option + affine: Trafo3d + boundingBox: Box3d + kdtreePath: string + objectSetPath: string + coordinatesPath: string + texturePath: string } + + type InCoreKdTree = + { kdTree: ConcreteKdIntersectionTree + boundingBox: Box3d } + + type Level0KdTree = + | LazyKdTree of LazyKdTree + | InCoreKdTree of InCoreKdTree + + let relativePath (path: string) (remaining: int) = + path.Split(Path.DirectorySeparatorChar) + |> List.ofArray + |> List.rev + |> List.take remaining + |> List.rev + |> Path.combine + + let relativePath' (path: string) = relativePath path 3 + + let expandKdTreePaths basePath kd = + kd + |> HashMap.map (fun _ k -> + match k with + | Level0KdTree.LazyKdTree lkt -> + let kdTreeSub = lkt.kdtreePath |> relativePath' + let triangleSub = lkt.objectSetPath |> relativePath' + + LazyKdTree + { lkt with + kdtreePath = Path.Combine(basePath, kdTreeSub) + objectSetPath = Path.Combine(basePath, triangleSub) } + | Level0KdTree.InCoreKdTree ik -> InCoreKdTree ik) + + let makeInCoreKd a = + { kdTree = new ConcreteKdIntersectionTree() + boundingBox = a } + + let makeLazyTree a b c d e f = + { kdTree = None + affine = a + boundingBox = b + kdtreePath = c + objectSetPath = d + coordinatesPath = e + texturePath = f } + + // PICKLER + let incorePickler: Pickler = + Pickler.product makeInCoreKd + ^. Pickler.field (fun s -> s.boundingBox) Pickler.auto + + let lazyPickler: Pickler = + Pickler.product makeLazyTree + ^+ Pickler.field (fun (s: LazyKdTree) -> s.affine) Pickler.auto + ^+ Pickler.field (fun (s: LazyKdTree) -> s.boundingBox) Pickler.auto + ^+ Pickler.field (fun s -> s.kdtreePath) Pickler.string + ^+ Pickler.field (fun s -> s.objectSetPath) Pickler.string + ^+ Pickler.field (fun s -> s.coordinatesPath) Pickler.string + ^. Pickler.field (fun s -> s.texturePath) Pickler.string + + let level0KdTreePickler: Pickler = + Pickler.sum (fun x k1 k2 -> + match x with + | InCoreKdTree k -> k1 k + | LazyKdTree k -> k2 k) + ^+ Pickler.case InCoreKdTree incorePickler + ^. Pickler.case LazyKdTree lazyPickler + + // SAVE LOAD + let save path (b: BinarySerializer) (d: 'a) = + let arr = b.Pickle d + System.IO.File.WriteAllBytes(path, arr) + d + + let loadAs<'a> path (b: BinarySerializer) : 'a = + let arr = System.IO.File.ReadAllBytes(path) + b.UnPickle arr + + let loadKdtree path = + //Log.startTimed "loading tree" + use b = new BinaryReadingCoder(System.IO.File.OpenRead(path)) + let mutable kdTree = Unchecked.defaultof + b.CodeT(&kdTree) + //Log.stop() + ConcreteKdIntersectionTree(kdTree, Trafo3d.Identity) + + let saveKdTree (kdTree : KdIntersectionTree) path = + Log.startTimed "saving kd tree" + use b = new BinaryWritingCoder(System.IO.File.OpenWrite(path)) + b.CodeT(ref kdTree) + Log.stop() + + + let loadKdTrees' + (h: PatchHierarchy) + (trafo: Trafo3d) + (load: bool) + (mode: ViewerModality) + (b: BinarySerializer) + (forceRebuild : bool) + (ignoreMasterKdTree : bool) + (loadTriangles : Trafo3d -> string -> TriangleSet) + : HashMap = + + let masterKdPath = + mode + |> ViewerModality.matchy (h.kdTreeAggZero_FileAbsPath) (h.kdTreeAggZero2d_FileAbsPath) + + let cacheFile = System.IO.Path.ChangeExtension(masterKdPath, ".cache") + + let loadAndCreateCache () = + let patchInfos = + h.tree + |> QTree.getLeaves + |> Seq.toArray + |> Array.map (fun x -> x.info) + + let kd0Paths = + patchInfos + |> Array.map (fun x -> x, h.kdTree_FileAbsPath x.Name 0 mode) + + let missingKd0Paths = kd0Paths |> Array.filter (not << System.IO.File.Exists << snd) + Log.line "missing kd0 paths: %d/%d" missingKd0Paths.Length kd0Paths.Length -module KdTrees = - - type LazyKdTree = { - kdTree : option - affine : Trafo3d - boundingBox : Box3d - kdtreePath : string - objectSetPath : string - coordinatesPath : string - texturePath : string - } - - type InCoreKdTree = { - kdTree : ConcreteKdIntersectionTree - boundingBox : Box3d - } - - type Level0KdTree = - | LazyKdTree of LazyKdTree - | InCoreKdTree of InCoreKdTree - - let relativePath (path : string) (remaining : int) = - path.Split(Path.DirectorySeparatorChar) - |> List.ofArray - |> List.rev - |> List.take remaining - |> List.rev - |> Path.combine - - let relativePath' (path : string) = - relativePath path 3 - - let expandKdTreePaths basePath kd = - kd - |> HashMap.map(fun _ k -> - match k with - | Level0KdTree.LazyKdTree lkt -> - let kdTreeSub = lkt.kdtreePath |> relativePath' - let triangleSub = lkt.objectSetPath |> relativePath' - - LazyKdTree { - lkt with - kdtreePath = Path.Combine(basePath, kdTreeSub) - objectSetPath = Path.Combine(basePath, triangleSub) - } - | Level0KdTree.InCoreKdTree ik -> InCoreKdTree ik - ) - - let makeInCoreKd a = - { - kdTree = new ConcreteKdIntersectionTree() - boundingBox = a - } - - let makeLazyTree a b c d e f = - { - kdTree = None - affine = a - boundingBox = b - kdtreePath = c - objectSetPath = d - coordinatesPath = e - texturePath = f - } - - // PICKLER - let incorePickler : Pickler = - Pickler.product makeInCoreKd - ^. Pickler.field (fun s -> s.boundingBox) Pickler.auto - - let lazyPickler : Pickler = - Pickler.product makeLazyTree - ^+ Pickler.field (fun (s:LazyKdTree) -> s.affine) Pickler.auto - ^+ Pickler.field (fun (s:LazyKdTree) -> s.boundingBox) Pickler.auto - ^+ Pickler.field (fun s -> s.kdtreePath) Pickler.string - ^+ Pickler.field (fun s -> s.objectSetPath) Pickler.string - ^+ Pickler.field (fun s -> s.coordinatesPath) Pickler.string - ^. Pickler.field (fun s -> s.texturePath) Pickler.string - - let level0KdTreePickler : Pickler = - Pickler.sum (fun x k1 k2-> - match x with - | InCoreKdTree k -> k1 k - | LazyKdTree k -> k2 k) - ^+ Pickler.case InCoreKdTree incorePickler - ^. Pickler.case LazyKdTree lazyPickler - - // SAVE LOAD - let save path (b : BinarySerializer) (d : 'a) = - let arr = b.Pickle d - System.IO.File.WriteAllBytes(path, arr); - d - - let loadAs<'a> path (b : BinarySerializer) : 'a = - let arr = System.IO.File.ReadAllBytes(path) - b.UnPickle arr - - let loadKdtree path = - //Log.startTimed "loading tree" - use b = new BinaryReadingCoder(System.IO.File.OpenRead(path)) - let mutable kdTree = Unchecked.defaultof - b.CodeT(&kdTree) - //Log.stop() - ConcreteKdIntersectionTree(kdTree, Trafo3d.Identity) - - let loadKdTrees' (h : PatchHierarchy) (trafo:Trafo3d) (load : bool) (mode:ViewerModality) (b : BinarySerializer) : HashMap = - //ObjectBuilder - - let masterKdPath = - mode |> ViewerModality.matchy - (h.kdTreeAggZero_FileAbsPath) - (h.kdTreeAggZero2d_FileAbsPath) - - let cacheFile = System.IO.Path.ChangeExtension(masterKdPath, ".cache") - - let loadAndCreateCache() = - let patchInfos = - h.tree |> QTree.getLeaves |> Seq.toList |> List.map(fun x -> x.info) - - let kd0Paths = - patchInfos - |> List.map(fun x -> h.kdTree_FileAbsPath x.Name 0 mode) - - let kd0PathsExist = - kd0Paths |> List.forall(System.IO.File.Exists) - - match (System.IO.File.Exists(masterKdPath), kd0PathsExist) with - | (true, false) -> - Log.line "Found master kdtree - loading incore" - let tree = loadKdtree masterKdPath - let kd = { - kdTree = tree; - boundingBox = tree.KdIntersectionTree.BoundingBox3d.Transformed(trafo) - } - HashMap.add kd.boundingBox (InCoreKdTree kd) HashMap.empty - | (true, true) -> - Log.line "Found master kdtree and patch trees" - Log.startTimed "building lazy kdtree cache" - - let num = kd0Paths |> List.ofSeq |> List.length - - let bla = - kd0Paths - |> List.zip patchInfos - |> List.mapi( - fun i (info, kdPath) -> - let t = loadKdtree kdPath - let pos = - match mode with - | XYZ -> info.Positions - | SvBR -> info.Positions2d.Value - - let dir = h.opcPaths.Patches_DirAbsPath +/ info.Name + if not ignoreMasterKdTree && (File.Exists masterKdPath && not (Array.isEmpty missingKd0Paths)) && not forceRebuild then + + Log.line "Found master kdtree - loading incore" + let tree = loadKdtree masterKdPath + + let kd = + { kdTree = tree + boundingBox = tree.KdIntersectionTree.BoundingBox3d.Transformed(trafo) } + + HashMap.single kd.boundingBox (InCoreKdTree kd) + else + Log.line "Found master kdtree and patch trees" + Log.startTimed "building lazy kdtree cache" + + let num = kd0Paths |> List.ofSeq |> List.length + + + let kdTrees = + kd0Paths + |> Array.mapi (fun i (info,kdPath) -> - let lazyTree : LazyKdTree = { - kdTree = None - objectSetPath = dir +/ pos - coordinatesPath = dir +/ (List.head info.Coordinates) - texturePath = Patch.extractTexturePath (OpcPaths h.opcPaths.Opc_DirAbsPath) info 0 - kdtreePath = kdPath - affine = - mode - |> ViewerModality.matchy info.Local2Global info.Local2Global2d - boundingBox = t.KdIntersectionTree.BoundingBox3d.Transformed(trafo) - } - Report.Progress(float i / float num) - - (lazyTree.boundingBox, (LazyKdTree lazyTree)) - ) - - Log.stop() - - bla |> save cacheFile b |> ignore - - if load then - bla |> HashMap.ofList - else - HashMap.empty - | _ -> - Log.warn "Could not find level 0 kdtrees" - HashMap.empty - - if System.IO.File.Exists(cacheFile) then - Log.line "Found lazy kdtree cache" - if load then - try - let trees = loadAs> cacheFile b - // let trees = trees |> List.filter(fun (_,(LazyKdTree k)) -> k.kdtreePath = blar) - trees |> HashMap.ofList - with e -> - Log.warn "could not load lazy kdtree cache. (%A) rebuilding..." e - loadAndCreateCache() - else - HashMap.empty - else - loadAndCreateCache() - - let loadKdTrees (h : PatchHierarchy) (trafo:Trafo3d) (mode:ViewerModality) (b : BinarySerializer) : HashMap = - loadKdTrees' (h) (trafo) (true) mode b \ No newline at end of file + let dir = h.opcPaths.Patches_DirAbsPath +/ info.Name + let pos = + match mode with + | XYZ -> info.Positions + | SvBR -> info.Positions2d.Value + + let objectSetPath = dir +/ pos + let trafo = mode |> ViewerModality.matchy info.Local2Global info.Local2Global2d + + let createConcreteTree () : ConcreteKdIntersectionTree = + let triangleSet = loadTriangles trafo objectSetPath + + Log.startTimed $"Building KdTree for {info.Name}" + let flags = + KdIntersectionTree.BuildFlags.Picking + ||| KdIntersectionTree.BuildFlags.FastBuild + ||| KdIntersectionTree.BuildFlags.EmptySpaceOptimization + + let kdTree = KdIntersectionTree(triangleSet, flags ) + Log.stop() + Log.startTimed "saving KdTree to: %s" info.Name + saveKdTree kdTree kdPath + Log.stop() + let fi = FileInfo(kdPath) + Log.line $"{info.Name} has size: {Mem(fi.Length)}." + ConcreteKdIntersectionTree(kdTree, Trafo3d.Identity) + + let t = + if File.Exists kdPath && not forceRebuild then + try + loadKdtree kdPath + with e -> + Log.warn "[KdTrees] could not load kdtree: %A" e + createConcreteTree() + else + createConcreteTree() + + + + let lazyTree: LazyKdTree = + { kdTree = None + objectSetPath = objectSetPath + coordinatesPath = dir +/ (List.head info.Coordinates) + texturePath = Patch.extractTexturePath (OpcPaths h.opcPaths.Opc_DirAbsPath) info 0 + kdtreePath = kdPath + affine = + mode + |> ViewerModality.matchy info.Local2Global info.Local2Global2d + boundingBox = t.KdIntersectionTree.BoundingBox3d + } + + Report.Progress(float i / float num) + + (lazyTree.boundingBox, (LazyKdTree lazyTree))) + + + + Log.stop () + + kdTrees |> save cacheFile b |> ignore + + if load then + kdTrees |> HashMap.ofArray + else + HashMap.empty + + + if File.Exists cacheFile then + Log.line "Found lazy KdTree cache" + + if load then + + if forceRebuild then + loadAndCreateCache() + else + try + let trees = loadAs> cacheFile b + trees |> HashMap.ofList + with + | e -> + Log.warn "could not load lazy KdTree cache. (%A) rebuilding..." e + loadAndCreateCache () + else + HashMap.empty + else + loadAndCreateCache () + + let loadKdTrees + (h: PatchHierarchy) (trafo: Trafo3d) (mode: ViewerModality) + (b: BinarySerializer) (forceRebuild : bool) (ignoreMasterKdTree : bool) + (loadTriangles : Trafo3d -> string -> TriangleSet) : HashMap = + + loadKdTrees' h trafo true mode b forceRebuild ignoreMasterKdTree loadTriangles diff --git a/src/ViewPlannerDSS/App.fs b/src/ViewPlannerDSS/App.fs index cad8c90..42e3973 100644 --- a/src/ViewPlannerDSS/App.fs +++ b/src/ViewPlannerDSS/App.fs @@ -345,10 +345,12 @@ module App = for h in patchHierarchies do let rootTree = h.tree |> QTree.getRoot + let kd = KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer false false (fun _ _ -> failwith "no triangleset function") + yield { patchHierarchy = h - kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath (KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ OpcSelectionViewer.Serialization.binarySerializer) + kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath kd localBB = rootTree.info.LocalBoundingBox globalBB = rootTree.info.GlobalBoundingBox neighborMap = HashMap.empty diff --git a/src/ViewerMain/App.fs b/src/ViewerMain/App.fs index dfc7047..298901f 100644 --- a/src/ViewerMain/App.fs +++ b/src/ViewerMain/App.fs @@ -307,10 +307,11 @@ module App = for h in patchHierarchies do let rootTree = h.tree |> QTree.getRoot - + let kd = KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ Serialization.binarySerializer false false (fun _ _ -> failwith "no function for creating triangle sets") + yield { patchHierarchy = h - kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath (KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ Serialization.binarySerializer) + kdTree = Aardvark.VRVis.Opc.KdTrees.expandKdTreePaths h.opcPaths.Opc_DirAbsPath kd localBB = rootTree.info.LocalBoundingBox globalBB = rootTree.info.GlobalBoundingBox neighborMap = HashMap.empty diff --git a/src/ViewerMain/Outline/OutlineApp.fs b/src/ViewerMain/Outline/OutlineApp.fs index 74e7d6b..e14ded3 100644 --- a/src/ViewerMain/Outline/OutlineApp.fs +++ b/src/ViewerMain/Outline/OutlineApp.fs @@ -128,10 +128,11 @@ module OutlineApp = for h in patchHierarchies do let rootTree = h.tree |> QTree.getRoot + let kd = KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ Serialization.binarySerializer false false (fun _ -> failwith "no function for creating triangle sets") yield { patchHierarchy = h - kdTree = KdTrees.loadKdTrees' h Trafo3d.Identity true ViewerModality.XYZ Serialization.binarySerializer + kdTree = kd localBB = rootTree.info.LocalBoundingBox globalBB = rootTree.info.GlobalBoundingBox neighborMap = HashMap.empty