After calling ort.InferenceSession.create, strange workers are automatically created

[meded90]

After the WebAssembly files are downloaded, several workers are created for tasks and additional workers perform constant operations. The number of these workers depends on ort.env.wasm.numThreads.

They cannot process the task because I call the predict function once every second, expecting that the rest of the time they will be in standby mode.

Here is the initialization code:

import modelUrl from 'Src/modules/AI/best.onnx?url'
import nmsModelUrl from 'Src/modules/AI/nms-yolov8.onnx?url'
import { IPxNumber, IRatioNumber, IRealNumber } from 'Src/types/nominal'
import { scaleLinear } from 'd3-scale'
import { InferenceSession, Tensor } from 'onnxruntime-web'
import * as ort from 'onnxruntime-web/webgpu'
import dataYaml from './data.yaml'

// ort.env.logLevel = 'verbose'
// ort.env.trace = true

// ort.env.wasm.wasmPaths = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ort.env.versions.web}/dist/`
ort.env.wasm.wasmPaths = `https://cdnjs.cloudflare.com/ajax/libs/onnxruntime-web/${ort.env.versions.web}/`
// ort.env.wasm.wasmPaths = `https://unpkg.com/onnxruntime-web@dev/dist/`
ort.env.wasm.proxy = true
// ort.env.wasm.numThreads = 20

export interface Box {
  label: number
  class: string
  probability: IRatioNumber
  bounding: [IPxNumber, IPxNumber, IPxNumber, IPxNumber]
}
interface Selected {
  cpuData: Float32Array
  dataLocation: string
  dims: [number, number, number]
  size: number
  type: string
  data: Float32Array
}

export class OnnxRunTimeApi {
  session: InferenceSession | null = null
  nmsSession: InferenceSession | null = null

  targetSize = 320
  modelInputShape: [number, number, number, number] = [1, 3, this.targetSize, this.targetSize]
  topk: number = 100
  iouThreshold: number = 0.45
  scoreThreshold: number = 0.25
  private scale = scaleLinear<IRealNumber, IPxNumber>([0, 255], [0, 1]).clamp(true)
  resizedCanvas = new OffscreenCanvas(this.targetSize, this.targetSize)
  resizedCtx = this.resizedCanvas.getContext('2d', { willReadFrequently: true })!

  constructor() {
    makeObservable(this)
  }

  async loadModel(): Promise<void> {
    if (!this.session) {
      performance.mark('loadModel')

      this.session = await ort.InferenceSession.create(modelUrl, {
        executionProviders: [],
        enableProfiling: true
      })

      this.nmsSession = await ort.InferenceSession.create(nmsModelUrl, {
        executionProviders: []
      })

      console.log('::::  loadModel =', performance.measure('loadModel', 'loadModel').duration + 'ms')
    }
  }

  async predict(model: { canvas: OffscreenCanvas; ctx: OffscreenCanvasRenderingContext2D }): Promise<Box[]> {
    await this.loadModel()
    const { canvas, ctx } = model

    if (!ctx) {
      throw new Error('Unable to get 2d context')
    }

    performance.mark('Tensor')

    let minValue = 255
    let maxValue = 0

    performance.mark('max max')
    const originalData = ctx.getImageData(0, 0, canvas.width, canvas.height)

    // get max top line
    for (let i = 0; i < canvas.width; i++) {
      const v = originalData.data[i * 4]
      if (v) {
        minValue = Math.min(minValue, v)
        maxValue = Math.max(maxValue, v)
      }
    }
    // get max min middle line
    const half = Math.floor(canvas.width / 2)
    for (let i = 0; i < canvas.height; i++) {
      const v = originalData.data[i * canvas.width * 4 + half * 4]
      if (v) {
        minValue = Math.min(minValue, v)
        maxValue = Math.max(maxValue, v)
      }
    }
    this.scale.domain([minValue, maxValue])

    console.log(`::::  max max =`, performance.measure('max max', 'max max').duration + 'ms')

    this.resizedCtx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, this.targetSize, this.targetSize)
    if (canvas.height < this.targetSize) {
      this.resizedCtx.clearRect(0, canvas.height, this.targetSize, this.targetSize - canvas.height)
    }

    const imageData = this.resizedCtx.getImageData(0, 0, this.targetSize, this.targetSize)
    performance.mark('tensor')

    const floatData = new Float32Array(this.targetSize * this.targetSize * 3)

    for (let i = 0, len = this.targetSize * this.targetSize; i < len; i++) {
      const v = this.scale(imageData.data[i * 4] ?? 0)
      floatData[i] = v
      floatData[i + this.targetSize * this.targetSize] = v
      floatData[i + 2 * this.targetSize * this.targetSize] = v
    }

    const tensor = new Tensor('float32', floatData, this.modelInputShape)
    console.log(`::::  tensor =`, performance.measure('tensor', 'tensor').duration + 'ms')
    performance.mark('model.run')

    if (!this.session || !this.nmsSession) {
      throw new Error('Model not loaded')
    }

    performance.mark('detect')
    const inputName: string = this.session.inputNames[0]!
    const outputName: string = this.session.outputNames[0]!

    const feeds = { [inputName]: tensor }

    const output = await this.session.run(feeds)
    const output0 = output[outputName]! // Обработка одного выхода модели

    console.log(`::::  detect =`, performance.measure('detect', 'detect').duration + 'ms')
    performance.mark('selected')

    const config = new Tensor('float32', new Float32Array([this.topk, this.iouThreshold, this.scoreThreshold]))

    const selectedOutput = await this.nmsSession.run({ detection: output0, config: config })
    const outputNameSelected: string = this.nmsSession.outputNames[0]!

    const selected = selectedOutput[outputNameSelected] as unknown as Selected // Обработка одного выхода модели
    console.log(`::::  selected =`, performance.measure('selected', 'selected').duration + 'ms')

    const boxes: Box[] = []
    const xRatio = canvas.width / this.targetSize
    // const yRatio = canvas.height / this.targetSize

    for (let idx = 0; idx < selected.dims[1]; idx++) {
      const data = selected.data.slice(idx * selected.dims[2], (idx + 1) * selected.dims[2])
      const box = Array.from(data.slice(0, 4)) as [number, number, number, number]
      const scores = data.slice(4)
      const score = Math.max(...scores)
      const label = scores.indexOf(score)

      const [x, y, w, h] = [
        //
        (box[0] - 0.5 * box[2]) * xRatio,
        box[1] - 0.5 * box[3],
        box[2] * xRatio,
        box[3]
      ]

      boxes.push({
        label,
        class: dataYaml.names[label],
        probability: score,
        bounding: [x, y, w, h]
      })
    }

    return boxes
  }
}

chrome performance:
Trace-20240802T195049.json.zip

[meded90]

It turned out that the issue was with the parameter ort.env.wasm.numThreads https://onnxruntime.ai/docs/tutorials/web/env-flags-and-session-options.html#envwasmnumthreads

As I expected, this parameter indicates that the model will run in parallel across multiple threads, but in the end, threads are created that work 100% of the time and it's unclear what they are doing since execution speed does not increase.