Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feat/batch processing refactor #16

Merged
merged 4 commits into from
Jul 11, 2024
Merged

Feat/batch processing refactor #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c0f0183
fix: store missing entities blocks in cache
etienne-85 Jul 10, 2024
6faa8a9
feat: batch processing refactor
etienne-85 Jul 11, 2024
ae88fe2
fix: additional fixes
etienne-85 Jul 11, 2024
48c62ae
style: lint
piellardj Jul 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
297 changes: 91 additions & 206 deletions src/procgen/PatchBaseCache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import { Heightmap, PatchCache } from '../index'
import { TreeType } from '../tools/TreeGenerator'

import { Biome, BiomeType, BlockType } from './Biome'
import { PatchBlocksCache } from './PatchBlocksCache'
import { EntityChunk, PatchBlocksCache } from './PatchBlocksCache'
import { PatchBatchProcessing } from './PatchBatchProcessing'
import { EntityData, Vegetation } from './Vegetation'

export type BlockData = {
Expand All @@ -18,19 +19,10 @@ export type BlockData = {

export type BlockIteratorRes = IteratorResult<BlockData, void>

enum PatchState {
export enum PatchState {
Empty,
Filled,
Done,
Final,
}

enum BatchProcessStep {
RegularGen,
PreTransitionGen,
TransitionGen,
PostProcessEntities,
Done,
Finalised,
}

const cacheSyncProvider = (batch: any) => {
Expand All @@ -45,28 +37,7 @@ export class PatchBaseCache extends PatchCache {
static instances: PatchBaseCache[] = []
static override bbox = new Box3()
static cacheRadius = 20
static batch: {
currentStep: BatchProcessStep
startTime: number
totalElapsedTime: number
count: number
totalCount: number
// eslint-disable-next-line no-use-before-define
regular: PatchBaseCache[]
// eslint-disable-next-line no-use-before-define
transition: PatchBaseCache[]
// eslint-disable-next-line no-use-before-define
skipped: PatchBaseCache[]
} = {
currentStep: BatchProcessStep.RegularGen,
startTime: 0,
totalElapsedTime: 0,
count: 0,
totalCount: 0,
regular: [],
transition: [],
skipped: [],
}
static pendingUpdate = false

spawnedEntities: EntityData[] = []
extEntities: EntityData[] = []
Expand Down Expand Up @@ -105,37 +76,13 @@ export class PatchBaseCache extends PatchCache {
* @returns
*/
getEntities() {
this.cacheExtEntities()
this.finalisePatches()
return [...this.spawnedEntities, ...this.extEntities]
}

cacheExtEntities() {
if (this.state < PatchState.Done) {
const nearPatches = this.getNearPatches()
// skip patches with incomplete edge patches count or already processed
if (nearPatches.length === 8) {
let isFinal = true
// extEntities
nearPatches.forEach(patch => {
isFinal = isFinal && patch.state >= PatchState.Filled
patch.spawnedEntities
// .filter(entity => entity.edgesOverlaps)
.filter(entity => entity.bbox.intersectsBox(this.bbox))
.forEach(entity => this.extEntities.push(entity))
})
this.state = PatchState.Final // isFinal ? PatchState.Final : PatchState.Done
return true
}
// else {
// console.log(`incomplete patch edges count: ${nearPatches.length}`)
// }
}
return false
}

static updateCache(center: Vector3, syncCache = cacheSyncProvider) {
static async updateCache(center: Vector3, cacheSync = cacheSyncProvider) {
const { patchSize } = PatchCache
const { batch, cacheRadius } = PatchBaseCache
const { cacheRadius } = PatchBaseCache
const cacheSize = patchSize * cacheRadius
const bbox = new Box3().setFromCenterAndSize(
center,
Expand All @@ -156,9 +103,9 @@ export class PatchBaseCache extends PatchCache {

if (
PatchBaseCache.instances.length === 0 ||
(batch.currentStep === BatchProcessStep.Done &&
nextCenter.distanceTo(prevCenter) > patchSize)
(!this.pendingUpdate && nextCenter.distanceTo(prevCenter) > patchSize)
) {
this.pendingUpdate = true
PatchBaseCache.bbox = bbox
const created: PatchBaseCache[] = []
const existing: PatchBaseCache[] = []
Expand All @@ -167,135 +114,46 @@ export class PatchBaseCache extends PatchCache {
const patchStart = new Vector2(xmin, zmin)
// look for existing patch in current cache
let patch = PatchBaseCache.getPatch(patchStart) // || new BlocksPatch(patchStart) //BlocksPatch.getPatch(patchBbox, true) as BlocksPatch
if (!patch || patch.state < PatchState.Final) {
if (!patch || patch.state < PatchState.Finalised) {
patch = new PatchBaseCache(patchStart)
created.push(patch)
} else {
existing.push(patch)
}
}
}
// const updated = existing.filter(patch => patch.state < PatchState.Finalised)
const removedCount = PatchBaseCache.instances.length - existing.length
PatchBaseCache.instances = [...existing, ...created]
const { batch } = PatchBaseCache

// sort created patches depending on their type
for (const patch of created) {
const nearPatches = patch.getNearPatches()
const isEdgePatch = nearPatches.length !== 8
if (!isEdgePatch) {
patch.isTransitionPatch =
patch.isBiomeTransition ||
!!nearPatches.find(edgePatch => edgePatch.isBiomeTransition)
patch.isTransitionPatch
? batch.transition.push(patch)
: batch.regular.push(patch)
} else {
batch.skipped.push(patch)
}
}

// batch.sort((p1, p2) => p1.bbox.getCenter(new Vector3()).distanceTo(center) - p2.bbox.getCenter(new Vector3()).distanceTo(center))
// PatchBaseCache.processBatch(batch)
// PatchBaseCache.cacheExtEntities()
const batchPatches: PatchBlocksCache[] = []
if (created.length > 0) {
console.log(
`[PatchBaseCache:update] START patch cache updating: enqueud ${created.length}, kept ${existing.length}, removed ${removedCount} )`,
`[PatchBaseCache:update] enqueud ${created.length} new patches, kept ${existing.length}, removed ${removedCount} )`,
)
syncCache({ kept: existing })
batch.count = 0
batch.totalCount = 0
batch.startTime = Date.now()
batch.totalElapsedTime = 0
batch.currentStep = BatchProcessStep.RegularGen
const promise = new Promise(resolve => {
const wrapper = (batch: any) =>
batch.created || batch.kept ? syncCache(batch) : resolve(true)
PatchBaseCache.buildNextPatch(wrapper)
})

return promise
}
}
return null
}

static buildNextPatch(syncCache: any) {
const { batch } = PatchBaseCache
switch (batch.currentStep) {
case BatchProcessStep.RegularGen: {
const nextPatch = batch.regular.shift()
if (nextPatch) {
const blocksPatch = nextPatch.genGroundBlocks()
syncCache({ created: [blocksPatch] })
batch.count++
} else {
const elapsedTime = Date.now() - batch.startTime
const avgTime = Math.round(elapsedTime / batch.count)
console.log(
`processed ${batch.count} regular patches in ${elapsedTime} ms (avg ${avgTime} ms per patch) `,
)
batch.totalElapsedTime += elapsedTime
batch.totalCount += batch.count
batch.count = 0
batch.startTime = Date.now()
batch.currentStep = BatchProcessStep.PreTransitionGen
cacheSync({ kept: existing })
const batchProcess = new PatchBatchProcessing(created)

// const batchIterator = patchBatch.getBatchIterator();
const regularPatchIter = batchProcess.iterRegularPatches()
for await (const batchRes of regularPatchIter) {
batchPatches.push(batchRes)
cacheSync({ created: [batchRes] })
}
break
}
case BatchProcessStep.PreTransitionGen: {
batch.transition.forEach(patch => {
patch.isCloseToRefPatch = !!patch
.getNearPatches()
.find(p => !p.isTransitionPatch && p.state >= PatchState.Filled)
})
// console.log(`switch state from PreTransitionGen to TransitionGen`)
batch.currentStep = BatchProcessStep.TransitionGen
break
}
case BatchProcessStep.TransitionGen: {
const nextPatch = batch.transition.shift()
if (nextPatch) {
const blocksPatch = nextPatch.genGroundBlocks()
syncCache({ created: [blocksPatch] })
batch.count++
} else {
const elapsedTime = Date.now() - batch.startTime
const avgTime = Math.round(elapsedTime / batch.count)
console.log(
`processed ${batch.count} transition patches in ${elapsedTime} ms (avg ${avgTime} ms per patch) `,
)
batch.totalElapsedTime += elapsedTime
batch.totalCount += batch.count
batch.count = 0
batch.startTime = Date.now()
batch.currentStep = BatchProcessStep.PostProcessEntities
const transitPatchIter = batchProcess.iterTransitionPatches()
for await (const batchRes of transitPatchIter) {
batchPatches.push(batchRes)
cacheSync({ created: [batchRes] })
}
break
}
case BatchProcessStep.PostProcessEntities: {
const count = PatchBaseCache.cacheExtEntities()
const elapsedTime = Date.now() - batch.startTime
console.log(`postprocessed ${count} patches in ${elapsedTime}ms`)
batch.totalElapsedTime += elapsedTime
const avgTime = Math.round(batch.totalElapsedTime / batch.totalCount)
console.log(
`[PatchBaseCache:buildNextPatch] DONE processed ${batch.totalCount} patches in ${batch.totalElapsedTime} ms (avg ${avgTime} ms per patch) `,
)
syncCache({})
batch.currentStep = BatchProcessStep.Done
break
PatchBaseCache.instances
.filter(patch => patch.state < PatchState.Finalised)
.forEach(patch => patch.finalisePatches())
batchProcess.finaliseBatch()
cacheSync({ kept: existing, created: batchPatches })
this.pendingUpdate = false
}
return { kept: existing, created: batchPatches }
}
if (batch.currentStep !== BatchProcessStep.Done)
setTimeout(() => PatchBaseCache.buildNextPatch(syncCache), 0)
}

static cacheExtEntities() {
const patchCount = PatchBaseCache.instances
.map(patch => patch.cacheExtEntities())
.filter(val => val).length
return patchCount
return null
}

buildRefPoints() {
Expand Down Expand Up @@ -381,29 +239,6 @@ export class PatchBaseCache extends PatchCache {
return Math.round(h)
}

*overBlocksIter() {
const entities = this.getEntities()
for (const entity of entities) {
const blocksIter = PatchBlocksCache.getPatch(
this.bbox.getCenter(new Vector3()),
).getBlocks(entity.bbox)
// let item: BlockIteratorRes = blocksIter.next()
for (const block of blocksIter) {
const overBlocksBuffer = Vegetation.singleton.fillBuffer(
block.pos,
entity,
[],
)
this.bbox.max.y = Math.max(
this.bbox.max.y,
block.pos.y + overBlocksBuffer.length,
)
block.buffer = overBlocksBuffer
yield block
}
}
}

static genOvergroundBlocks(baseBlock: BlockData) {
// find patch containing point in cache
const patch = this.getPatch(baseBlock.pos)
Expand All @@ -414,19 +249,69 @@ export class PatchBaseCache extends PatchCache {
patch
.getEntities()
.filter(entity => entity.bbox.containsPoint(pos))
.forEach(entity => Vegetation.singleton.fillBuffer(pos, entity, buffer))
.forEach(entity => Vegetation.instance.fillBuffer(pos, entity, buffer))
}
return buffer
}

// genOvergroundBlocks(){
// const { min } = this.bbox
// const blocksPatch = new PatchBlocksCache(new Vector2(min.x, min.z))
// const blocksPatchIter = blocksPatch.iterator()
// }
genEntitiesBlocks(
blocksPatch: PatchBlocksCache,
entities: EntityData[] = [...this.spawnedEntities, ...this.extEntities],
) {
blocksPatch.entitiesChunks = entities.map(entity => {
const blocksIter = blocksPatch.getBlocks(entity.bbox)
// let item: BlockIteratorRes = blocksIter.next()
const chunk: EntityChunk = {
bbox: new Box3(),
data: [],
}

for (const block of blocksIter) {
const blocksBuffer = Vegetation.instance.fillBuffer(
block.pos,
entity,
[],
)
this.bbox.max.y = Math.max(
this.bbox.max.y,
block.pos.y + blocksBuffer.length,
)
const serialized = blocksBuffer
.reduce((str, val) => str + ',' + val, '')
.slice(1)
chunk.data.push(serialized)
chunk.bbox.expandByPoint(block.pos)
}
blocksPatch.bbox.max.y = this.bbox.max.y
chunk.bbox = entity.bbox
return chunk
})
}

finalisePatches() {
const nearPatches = this.getNearPatches()
// skip patches with incomplete edge patches count
if (nearPatches.length === 8) {
let isFinal = true
// extEntities
nearPatches.forEach(patch => {
isFinal = isFinal && patch.state >= PatchState.Filled
patch.spawnedEntities
// .filter(entity => entity.edgesOverlaps)
.filter(entity => entity.bbox.intersectsBox(this.bbox))
.forEach(entity => this.extEntities.push(entity))
})
this.state = PatchState.Finalised // isFinal ? PatchState.Final : PatchState.Done
return true
}
// else {
// console.log(`incomplete patch edges count: ${nearPatches.length}`)
// }
return false
}

/**
* Gen blocks data that will be sent to external cache
* Gen blocks data that will be sent to blocks cache
*/
genGroundBlocks() {
const { min, max } = this.bbox
Expand Down
Loading