Skip to content

Commit

Permalink
mlrt.avsi 1.1.0
Browse files Browse the repository at this point in the history 
mlrt_W2x: add swin_unet_art models.
Add backend `ov`.
Remove parameter backend_args.
  • Loading branch information
@Asd-g
Asd-g committed Mar 20, 2023
1 parent 8790ded commit 9b28194
Showing 1 changed file with 165 additions and 57 deletions.
222 changes: 165 additions & 57 deletions mlrt.avsi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@

### Usage ###
###
# mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", bool "preprocess", string "backend_args")
# mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", bool "preprocess")
#
# mlrt_DPIR(clip c, val "strength", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", string "backend_args")
# mlrt_DPIR(clip c, val "strength", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend")
#
# mlrt_RealESRGAN/mlrt_RealESRGANv2(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale", string "backend_args")
# mlrt_RealESRGAN/mlrt_RealESRGANv2(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale")
#
# mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", val "backend", int "version", bool "conformance", string "backend_args")
# mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", val "backend", int "version", bool "conformance")
###
## Parameters ##
#---------------
Expand All @@ -31,24 +31,22 @@
# mlrt_CUGAN overlap_w default: 4.
# overlap_h default: overlap_w.
#---------------
# backend (default "ncnn"): What backend to be used (currently only ncnn) and its tuning parameters.
# backend (default "ncnn"): What backend to be used (currently ncnn and ov) and its tuning parameters.
# It can be specified as single string.
# If specified as array of strings, the first element must be the backend type and the rest one are the tuning parameters.
# For example, backend=["ncnn", "fp16=true"] - backend type is "ncnn" and fp16 mode is True.
#---------------
# backend_args (default not specified): Additional backend arguments.
#---------------
# noise (mlrt_W2x, mlrt_CUGAN) (default -1): Denoise level.
# Large value means strong denoise effect, -1 - no effect.
# Must be between -1..3.
#---------------
# scale (mlrt_W2x, mlrt_RealESRGAN, mlrt_CUGAN): Upscale ratio.
# mlrt_W2x: default 2; must be either 1 or 2.
# mlrt_W2x: default 2; must be either 1, 2, or 4.
# mlrt_RealESRGAN default: not specified.
# mlrt_CUGAN: default 2; must be 2, 3, or 4.
#---------------
# model (mlrt_W2x, mlrt_DPIR, mlrt_RealESRGAN): What model to be used.
# Folder "models" must be in the same location as avs-mlrt.
# Folder "models" must be in the same location as mlrt_xxx.dll.
/*
mlrt_W2x:
0: "anime_style_art"
Expand All @@ -58,6 +56,7 @@ mlrt_W2x:
4: "upconv_7_photo"
5: "upresnet10"
6: "cunet"
7: "swin_unet_art"
Default 6.
*/
/*
Expand Down Expand Up @@ -88,36 +87,40 @@ Default 0.
# conformance (mlrt_CUGAN) (default True): Currently specifies dynamic range compression for cugan-pro.


### Version: 1.0.1
### Version: 1.1.0


### Changelog ###
#---------------
# mlrt_W2x: added swin_unet_art models.
# Added backend "ov".
# Removed parameter backend_args.
#---------------
# mlrt_W2x: fixed the model path when scale=1.
#---------------
# Initial version.


Function mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", bool "preprocess", string "backend_args")
Function mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", bool "preprocess")
{
noise = Default(noise, -1)
scale = Default(scale, 2)
model = Default(model, 6)
overlap_w = Default(overlap_w, Select(model, 8, 8, 8, 8, 8, 4, 4))
overlap_w = Default(overlap_w, Select(model, 8, 8, 8, 8, 8, 4, 4, 4))
overlap_h = Default(overlap_h, overlap_w)
preprocess = Default(preprocess, True)
backend_args = Defined(backend_args) ? (", " + backend_args) : ""
backend = backend_def(backend)

Assert(ComponentSize(c) == 4, "mlrt_W2x: clip must be in 32-bit planar format.")
Assert(noise >= -1 && noise <= 3, "mlrt_W2x: noise must be -1, 0, 1, 2, or 3.")
Assert(scale == 1 || scale == 2, "mlrt_W2x: scale must be 1 or 2.")
Assert(model >= 0 && model <= 6, "mlrt_W2x: model must be 0, 1, 2, 3, 4, 5, or 6.")
Assert(backend[0] == "ncnn", "mlrt_W2x: backend must be ncnn.")
Assert(scale == 1 || scale == 2 || scale == 4, "mlrt_W2x: scale must be 1, 2, or 4.")
Assert(model >= 0 && model <= 7, "mlrt_W2x: model must be 0, 1, 2, 3, 4, 5, 6, or 7.")
Assert(backend[0] == "ncnn" || backend[0] == "ov", "mlrt_W2x: backend must be either ncnn or ov.")

w2x_models = Select(model, "anime_style_art", "anime_style_art_rgb", "photo", "upconv_7_anime_style_art_rgb", "upconv_7_photo", "upresnet10", "cunet")
w2x_models = Select(model, "anime_style_art", "anime_style_art_rgb", "photo", "upconv_7_anime_style_art_rgb", "upconv_7_photo", "upresnet10", "cunet", "swin_unet_art")

Assert(!(model == 0 && noise == 0), "mlrt_W2x: anime_style_art model does not support noise reduction level 0.")
Assert((model < 7 && scale < 4) || model == 7, "mlrt_W2x: scale must be 1 or 2.")

if (model == 0)
{
Expand Down Expand Up @@ -152,16 +155,24 @@ Function mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize",
{
model_name = (noise == -1) ? "scale2.0x_model.onnx" : ("noise" + String(noise) + "_scale2.0x_model.onnx")
}
else
else if (model == 6)
{
scale_name = (scale == 1) ? "" : "scale2.0x_"

model_name = (noise == -1) ? "scale2.0x_model.onnx" : ("noise" + String(noise) + "_" + scale_name + "model.onnx")
}
else
{
Assert(noise != -1 && scale != 1, "mlrt_W2x: swin_unet model for noise=-1 and scale=1 does not exist.")

scale_name = (scale > 1) ? (scale > 2) ? "scale4x" : "scale2x" : ""

model_name = (noise > -1) ? (scale == 1) ? ("noise" + String(noise) + ".onnx") : ("noise" + String(noise) + "_" + scale_name + ".onnx") : (scale_name + ".onnx")
}

backend_defaults = backend_defaults(backend)

c = Eval("mlrt_" + backend[0] + "(c, network_path=folder_path+model_name, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true, " + backend_defaults + backend_args + ")")
c = Eval("mlrt_" + backend[0] + "(c, network_path=folder_path+model_name, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true" + backend_defaults + ")")

if (scale == 1 && (Width(c) / width == 2))
{
Expand All @@ -173,17 +184,16 @@ Function mlrt_W2x(clip c, int "noise", int "scale", val "tiles", val "tilesize",
return c
}

Function mlrt_DPIR(clip c, val "strength", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", string "backend_args")
Function mlrt_DPIR(clip c, val "strength", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend")
{
model = Default(model, 0)
overlap_w = Default(overlap_w, 0)
overlap_h = Default(overlap_h, overlap_w)
backend_args = Defined(backend_args) ? (", " + backend_args) : ""
backend = backend_def(backend)

Assert(ComponentSize(c) == 4, "mlrt_DPIR: clip must be in 32-bit planar format.")
Assert(model >= 0 && model <= 3, "mlrt_DPIR: model must be 0, 1, 2, or 3.")
Assert(backend[0] == "ncnn", "mlrt_DPIR: backend must be ncnn.")
Assert(backend[0] == "ncnn" || backend[0] == "ov", "mlrt_DPIR: backend must be either ncnn or ov.")

dpir_models = Select(model, "drunet_gray", "drunet_color", "drunet_deblocking_grayscale", "drunet_deblocking_color")

Expand Down Expand Up @@ -221,21 +231,20 @@ Function mlrt_DPIR(clip c, val "strength", val "tiles", val "tilesize", int "ove
network_path = "dpir/" + dpir_models + ".onnx"
backend_defaults = backend_defaults(backend)

return Eval("mlrt_" + backend[0] + "([c, strength], network_path=network_path, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true, " + backend_defaults + backend_args + ")")
return Eval("mlrt_" + backend[0] + "([c, strength], network_path=network_path, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true" + backend_defaults + ")")
}

Function mlrt_RealESRGAN(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale", string "backend_args")
Function mlrt_RealESRGAN(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale")
{
model = Default(model, 0)
overlap_w = Default(overlap_w, 8)
overlap_h = Default(overlap_h, overlap_w)
backend_args = Defined(backend_args) ? (", " + backend_args) : ""
backend = backend_def(backend)

Assert(ComponentSize(c) == 4, "mlrt_RealESRGAN: clip must be in 32-bit planar format.")
Assert(IsPlanarRGB(c), "mlrt_RealESRGAN: clip must be of planar RGB color family.")
Assert(model >= 0 && model <= 2, "mlrt_RealESRGAN: model must be 0, 1, or 2.")
Assert(backend[0] == "ncnn", "mlrt_RealESRGAN: backend must be ncnn.")
Assert(backend[0] == "ncnn" || backend[0] == "ov", "mlrt_RealESRGAN: backend must be either ncnn or ov.")

# v2, v2, v3 4x
realesrgan_models = Select(model, "animevideo-xsx2", "animevideo-xsx4", "animevideov3")
Expand All @@ -250,7 +259,7 @@ Function mlrt_RealESRGAN(clip c, val "tiles", val "tilesize", int "overlap_w", i
backend_defaults = backend_defaults(backend)

clip_org = c
c = Eval("mlrt_" + backend[0] + "(c, network_path=network_path, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true, " + backend_defaults + backend_args + ")")
c = Eval("mlrt_" + backend[0] + "(c, network_path=network_path, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true" + backend_defaults + ")")

if (Defined(scale))
{
Expand All @@ -271,12 +280,12 @@ Function mlrt_RealESRGAN(clip c, val "tiles", val "tilesize", int "overlap_w", i
return c
}

Function mlrt_RealESRGANv2(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale", string "backend_args")
Function mlrt_RealESRGANv2(clip c, val "tiles", val "tilesize", int "overlap_w", int "overlap_h", int "model", val "backend", float "scale")
{
return mlrt_RealESRGAN(c, tiles, tilesize, overlap_w, overlap_h, model, backend, scale, backend_args)
return mlrt_RealESRGAN(c, tiles, tilesize, overlap_w, overlap_h, model, backend, scale)
}

Function mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", val "backend", int "version", bool "conformance", string "backend_args")
Function mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize", int "overlap_w", int "overlap_h", val "backend", int "version", bool "conformance")
{
/*
denoising strength: 0 < -1 < 1 < 2 < 3
Expand All @@ -291,14 +300,13 @@ Function mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize
overlap_h = Default(overlap_h, overlap_w)
version = Default(version, 1)
conformance = Default(conformance, True) # currently specifies dynamic range compression for cugan-pro
backend_args = Defined(backend_args) ? (", " + backend_args) : ""
backend = backend_def(backend)

Assert(ComponentSize(c) == 4, "mlrt_CUGAN: clip must be in 32-bit planar format.")
Assert(IsPlanarRGB(c), "mlrt_CUGAN: clip must be of planar RGB color family.")
Assert(noise >= -1 && noise <= 3, "mlrt_CUGAN: noise must be -1, 0, 1, 2, or 3.")
Assert(scale == 2 || scale == 3 || scale == 4, "mlrt_CUGAN: scale must be 2, 3, or 4.")
Assert(backend[0] == "ncnn", "mlrt_CUGAN: backend must be ncnn.")
Assert(backend[0] == "ncnn" || backend[0] == "ov", "mlrt_CUGAN: backend must be either ncnn or ov.")
Assert(version == 1 || version == 2, "mlrt_CUGAN: version must be 1 (legacy) or 2 (pro)")
Assert(!(scale != 2 && (noise == 1 || noise == 2)), "mlrt_CUGAN: scale=" + String(scale) + " model.\nmlrt_CUGAN does not support noise reduction level " + String(noise) + ".")

Expand Down Expand Up @@ -328,7 +336,7 @@ Function mlrt_CUGAN(clip c, int "noise", int "scale", val "tiles", val "tilesize

backend_defaults = backend_defaults(backend)

c = Eval("mlrt_" + backend[0] + "(c, network_path=folder_path+model_name, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true, " + backend_defaults + backend_args + ")")
c = Eval("mlrt_" + backend[0] + "(c, network_path=folder_path+model_name, overlap_w=tile_overlap[2], overlap_h=tile_overlap[3], tilesize_w=tile_overlap[0], tilesize_h=tile_overlap[1], builtin=true" + backend_defaults + ")")

# https://github.com/bilibili/ailab/blob/e102bef22384c629f82552dbec3d6b5bab125639/Real-CUGAN/upcunet_v3.py#L269
if (conformance && version == 2)
Expand Down Expand Up @@ -370,21 +378,6 @@ Function calc_size(int width, int tiles, int overlap, int multiple)
return Ceil((width + 2 * overlap * (tiles - 1)) / (tiles * multiple)) * multiple
}

Function ncnn_defaults(bool "fp16", int "device_id", int "num_streams")
{
/*
basic performance tuning:
set fp16 = True (on modern GPUs)
increase num_streams
*/

fp16 = Default(fp16, True)
device_id = Default(device_id, 0)
num_streams = Default(num_streams, 1)

return "fp16=" + String(fp16) + ", " + "device_id=" + String(device_id) + ", " + "num_streams=" + String(num_streams)
}

Function backend_def(val "backend")
{
return Defined(backend) ? IsArray(backend) ? backend : \
Expand All @@ -393,21 +386,136 @@ Function backend_def(val "backend")

Function backend_defaults(string_array backend)
{
backend_string = backend_format(backend)
backend_string = (ArraySize(backend) == 1) ? "" : (", " + backend[1])

if (backend[0] == "ncnn")
for (i = 2, ArraySize(backend) - 1)
{
return eval("ncnn_defaults(" + backend_string + ")")
backend_string = backend_string + ", " + backend[i]
}
}

Function backend_format(string_array backend)
{
backend_string = ""
if (backend[0] == "ncnn")
{
/*
basic performance tuning:
set fp16 = True (on modern GPUs)
increase num_streams
*/

for (i = 1, ArraySize(backend) - 1)
if (FindStr(backend_string, "fp16") == 0)
{
backend_string = backend_string + ", fp16=true"
}
if (FindStr(backend_string, "device_id") == 0)
{
backend_string = backend_string + ", device_id=0"
}
if (FindStr(backend_string, "num_streams") == 0)
{
backend_string = backend_string + ", num_streams=1"
}
}
else if (backend[0] == "ov")
{
backend_string = (i == 1) ? backend[1] : (backend_string + ", " + backend[i])
if (FindStr(backend_string, "device") == 0 || FindStr(backend_string, """device="CPU"""") > 0)
{
/*
CPU basic performance tuning:
ENFORCE_BF16=YES (on Zen4)
increase num_streams
*/

if (FindStr(backend_string, "config") == 0)
{
backend_string = backend_string + """, config="CPU_THROUGHPUT_STREAMS=1 CPU_BIND_THREAD=YES CPU_THREADS_NUM=0""""
}
else
{
if (FindStr(backend_string, "CPU_THROUGHPUT_STREAMS") == 0)
{
beginning = MidStr(backend_string, FindStr(backend_string, "config"))

i = 9
while(MidStr(beginning, i, 1) != """"""")
{
i = i + 1
}

result = MidStr(beginning, 1, i-1) + " CPU_THROUGHPUT_STREAMS=1" + """""""
backend_string = ReplaceStr(backend_string, MidStr(beginning, 1, i), result)
}
if (FindStr(backend_string, "CPU_BIND_THREAD") == 0)
{
beginning = MidStr(backend_string, FindStr(backend_string, "config"))

i = 9
while(MidStr(beginning, i, 1) != """"""")
{
i = i + 1
}

result = MidStr(beginning, 1, i-1) + " CPU_BIND_THREAD=YES" + """""""
backend_string = ReplaceStr(backend_string, MidStr(beginning, 1, i), result)
}
if (FindStr(backend_string, "CPU_THREADS_NUM") == 0)
{
beginning = MidStr(backend_string, FindStr(backend_string, "config"))

i = 9
while(MidStr(beginning, i, 1) != """"""")
{
i = i + 1
}

result = MidStr(beginning, 1, i-1) + " CPU_THREADS_NUM=0" + """""""
backend_string = ReplaceStr(backend_string, MidStr(beginning, 1, i), result)
}
}
if (FindStr(backend_string, "fp16") == 0)
{
backend_string = backend_string + ", fp16=false"
}
if (FindStr(backend_string, "fp16_blacklist_ops") == 0)
{
backend_string = backend_string + """, fp16_blacklist_ops="""""
}
}
else
{
/*
GPU basic performance tuning:
set fp16 = True
increase num_streams
*/

if (FindStr(backend_string, "config") == 0)
{
backend_string = backend_string + """, config="GPU_THROUGHPUT_STREAMS=1""""
}
else
{
if (FindStr(backend_string, "GPU_THROUGHPUT_STREAMS") == 0)
{
beginning = MidStr(backend_string, FindStr(backend_string, "config"))

i = 9
while(MidStr(beginning, i, 1) != """"""")
{
i = i + 1
}

result = MidStr(beginning, 1, i-1) + " GPU_THROUGHPUT_STREAMS=1" + """""""
backend_string = ReplaceStr(backend_string, MidStr(beginning, 1, i), result)
}
}
if (FindStr(backend_string, "fp16") == 0)
{
backend_string = backend_string + ", fp16=false"
}
if (FindStr(backend_string, "fp16_blacklist_ops") == 0)
{
backend_string = backend_string + """, fp16_blacklist_ops="""""
}
}
}

return backend_string
Expand Down

0 comments on commit 9b28194

Please sign in to comment.