Source code for torch_tensorrt.dynamo._exporter
-import copy
+import base64
+import copy
import operator
-from typing import Any, Dict, Sequence, Tuple, cast
+from typing import Any, Dict, Optional, Sequence, Tuple, cast
import torch
from torch._guards import detect_fake_mode
@@ -481,24 +482,28 @@ Source code for torch_tensorrt.dynamo._exporter
<
OutputSpec,
TensorArgument,
)
+from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ENGINE_IDX, NAME_IDX
[docs]def export(
gm: torch.fx.GraphModule,
+ cross_compile_flag: Optional[bool] = False,
) -> ExportedProgram:
"""Export the result of TensorRT compilation into the desired output format.
Arguments:
gm (torch.fx.GraphModule): Compiled Torch-TensorRT module, generated by ``torch_tensorrt.dynamo.compile``
inputs (torch.Tensor): Torch input tensors
+ cross_compile_flag (bool): Flag to indicated whether it is cross_compilation enabled or not
"""
- patched_module = transform(gm)
+ patched_module = transform(gm, cross_compile_flag)
exp_program = create_trt_exp_program(patched_module)
return exp_program
def transform(
gm: torch.fx.GraphModule,
+ cross_compile_flag: Optional[bool] = False,
) -> torch.fx.GraphModule:
"""
Transforms the graphmodule by inlining Pytorch and TensorRT submodules.
@@ -508,6 +513,7 @@ Source code for torch_tensorrt.dynamo._exporter
<
Arguments:
gm (torch.fx.GraphModule): Compiled Torch-TensorRT module, generated by ``torch_tensorrt.dynamo.compile``
inputs (torch.Tensor): Torch input tensors
+ cross_compile_flag (bool): Flag to indicated whether it is cross_compilation enabled or not
Returns an inlined torch.fx.GraphModule
"""
@@ -516,7 +522,7 @@ Source code for torch_tensorrt.dynamo._exporter
<
gm = copy.deepcopy(gm)
# Inline TensorRT submodules
- inline_trt_modules(gm)
+ inline_trt_modules(gm, cross_compile_flag)
# Inline pytorch submodules
inline_torch_modules(gm)
@@ -815,7 +821,9 @@ Source code for torch_tensorrt.dynamo._exporter
<
return trt_exp_program
-def inline_trt_modules(gm: torch.fx.GraphModule) -> torch.fx.GraphModule:
+def inline_trt_modules(
+ gm: torch.fx.GraphModule, cross_compile_flag: Optional[bool] = False
+) -> torch.fx.GraphModule:
"""
Replace TRT submodules with trt engine nodes.
"""
@@ -838,25 +846,36 @@ Source code for torch_tensorrt.dynamo._exporter
<
num_outputs = len(trt_module_node.meta["val"])
# Insert a call_function node to perform inference on TRT engine
with gm.graph.inserting_before(trt_module_node):
- engine_name = f"{name}_engine"
- setattr(gm, engine_name, trt_module.engine)
- engine_node = gm.graph.get_attr(engine_name)
-
- trt_node = gm.graph.call_function(
- torch.ops.tensorrt.execute_engine.default,
- (trt_module_node.args, engine_node),
- )
+ if not cross_compile_flag:
+ # for the normal workflow: use the execute_engine node
+ engine_name = f"{name}_engine"
+ setattr(gm, engine_name, trt_module.engine)
+ engine_node = gm.graph.get_attr(engine_name)
+
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.execute_engine.default,
+ (trt_module_node.args, engine_node),
+ )
+ # meta["val"] should be a lighter version of a tensor. For eg: it should be a FakeTensor (with output shape and dtype properties)
+ # Lighter version of a custom_obj is not defined clearly. meta["val"] does not have any type expectations but
+ # for custom object nodes, it should be CustomObjArgument
+ engine_node.meta["val"] = CustomObjArgument(
+ name=engine_node.name, class_fqn=""
+ )
+ else:
+ # for the cross compile for windows workflow: use the no_op_placeholder node
+ engine_info = trt_module._pack_engine_info()
+ engine_bytes = engine_info[ENGINE_IDX]
+ engine_info[ENGINE_IDX] = base64.b64encode(engine_bytes).decode("utf-8")
+ # insert the no_placeholder node in the graph which should be replaced to the actual execute_engine node while load in the windows
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.no_op_placeholder_for_execute_engine.default,
+ (trt_module_node.args, *engine_info),
+ )
# set trt_node.meta with trt_module_node.meta
assert num_outputs > 0
trt_node.meta["val"] = trt_module_node.meta["val"]
- # meta["val"] should be a lighter version of a tensor. For eg: it should be a FakeTensor (with output shape and dtype properties)
- # Lighter version of a custom_obj is not defined clearly. meta["val"] does not have any type expectations but
- # for custom object nodes, it should be CustomObjArgument
- engine_node.meta["val"] = CustomObjArgument(
- name=engine_node.name, class_fqn=""
- )
-
if num_outputs == 1:
# Insert getitem nodes as outputs (for export serialization to work)
with gm.graph.inserting_after(trt_node):
@@ -876,6 +895,60 @@ Source code for torch_tensorrt.dynamo._exporter
<
gm.graph.erase_node(trt_module_node)
return gm
+
+
+def replace_execute_engine_no_op_node(
+ exp_program: ExportedProgram,
+) -> ExportedProgram:
+ gm = exp_program.graph_module
+ no_op_placeholder_nodes = []
+ for node in gm.graph.nodes:
+ if "no_op_placeholder_for_execute_engine" in node.name:
+ no_op_placeholder_nodes.append(node)
+ assert len(no_op_placeholder_nodes) > 0
+ for no_op_placeholder_node in no_op_placeholder_nodes:
+ if "val" not in no_op_placeholder_node.meta:
+ raise ValueError(f"metadata info is missing for the node: {node.name}")
+ with gm.graph.inserting_before(no_op_placeholder_node):
+ packed_engine_info = list(no_op_placeholder_node.args[1:])
+ engine_bytes = packed_engine_info[ENGINE_IDX]
+ engine_name = packed_engine_info[NAME_IDX]
+
+ packed_engine_info[ENGINE_IDX] = base64.b64decode(
+ engine_bytes.encode("utf-8")
+ )
+ trt_engine = torch.classes.tensorrt.Engine(tuple(packed_engine_info))
+ setattr(gm, engine_name, trt_engine)
+ engine_node = gm.graph.get_attr(engine_name)
+
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.execute_engine.default,
+ (no_op_placeholder_node.args[0], engine_node),
+ )
+ trt_node.meta["val"] = no_op_placeholder_node.meta["val"]
+ engine_node.meta["val"] = CustomObjArgument(
+ name=engine_node.name, class_fqn=""
+ )
+
+ if len(no_op_placeholder_node.meta["val"]) == 1:
+ with gm.graph.inserting_after(trt_node):
+ getitem_output = gm.graph.call_function(operator.getitem, (trt_node, 0))
+ getitem_output.meta["val"] = trt_node.meta["val"]
+ no_op_placeholder_node.replace_all_uses_with(getitem_output)
+ else:
+ no_op_placeholder_node.replace_all_uses_with(trt_node)
+ getitem_nodes = trt_node.users
+ for idx, getitem_node in enumerate(getitem_nodes):
+ getitem_node.meta["val"] = trt_node.meta["val"][idx]
+
+ gm.graph.erase_node(no_op_placeholder_node)
+
+ gm.delete_all_unused_submodules()
+ gm.graph.eliminate_dead_code()
+ gm.graph.lint()
+ gm.recompile()
+
+ return exp_program
diff --git a/docs/_modules/torch_tensorrt/dynamo/_refit.html b/docs/_modules/torch_tensorrt/dynamo/_refit.html
index 3cbebaf5f6..ec930e6f69 100644
--- a/docs/_modules/torch_tensorrt/dynamo/_refit.html
+++ b/docs/_modules/torch_tensorrt/dynamo/_refit.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo._refit — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo._refit — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/dynamo/_settings.html b/docs/_modules/torch_tensorrt/dynamo/_settings.html
index 259e97766e..41bd62ee7e 100644
--- a/docs/_modules/torch_tensorrt/dynamo/_settings.html
+++ b/docs/_modules/torch_tensorrt/dynamo/_settings.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo._settings — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo._settings — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -478,6 +478,7 @@ Source code for torch_tensorrt.dynamo._settings
<
DLA_LOCAL_DRAM_SIZE,
DLA_SRAM_SIZE,
DRYRUN,
+ ENABLE_CROSS_COMPILE_FOR_WINDOWS,
ENABLE_EXPERIMENTAL_DECOMPOSITIONS,
ENABLE_WEIGHT_STREAMING,
ENABLED_PRECISIONS,
@@ -549,6 +550,8 @@ Source code for torch_tensorrt.dynamo._settings
<
use_strong_typing (bool): This flag enables strong typing in TensorRT compilation which respects the precisions set in the Pytorch model. This is useful when users have mixed precision graphs.
use_fp32_acc (bool): This option inserts cast to FP32 nodes around matmul layers and TensorRT ensures the accumulation of matmul happens in FP32. Use this only when FP16 precision is configured in enabled_precisions.
enable_weight_streaming (bool): Enable weight streaming.
+ enable_cross_compile_for_windows (bool): By default this is False means TensorRT engines can only be executed on the same platform where they were built.
+ True will enable cross-platform compatibility which allows the engine to be built on Linux and run on Windows
"""
enabled_precisions: Set[dtype] = field(default_factory=lambda: ENABLED_PRECISIONS)
@@ -585,7 +588,8 @@ Source code for torch_tensorrt.dynamo._settings
<
reuse_cached_engines: bool = REUSE_CACHED_ENGINES
use_explicit_typing: bool = USE_EXPLICIT_TYPING
use_fp32_acc: bool = USE_FP32_ACC
- enable_weight_streaming: bool = ENABLE_WEIGHT_STREAMING
+ enable_weight_streaming: bool = ENABLE_WEIGHT_STREAMING
+ enable_cross_compile_for_windows: bool = ENABLE_CROSS_COMPILE_FOR_WINDOWS
_SETTINGS_TO_BE_ENGINE_INVARIANT = (
diff --git a/docs/_modules/torch_tensorrt/dynamo/_tracer.html b/docs/_modules/torch_tensorrt/dynamo/_tracer.html
index ba9e41c6b8..4b2f618d2e 100644
--- a/docs/_modules/torch_tensorrt/dynamo/_tracer.html
+++ b/docs/_modules/torch_tensorrt/dynamo/_tracer.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo._tracer — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo._tracer — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.html b/docs/_modules/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.html
index 28c3a44bd4..6c29913343 100644
--- a/docs/_modules/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.html
+++ b/docs/_modules/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo.runtime._MutableTorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo.runtime._MutableTorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/dynamo/runtime/_PythonTorchTensorRTModule.html b/docs/_modules/torch_tensorrt/dynamo/runtime/_PythonTorchTensorRTModule.html
index e421038739..a9f03017b1 100644
--- a/docs/_modules/torch_tensorrt/dynamo/runtime/_PythonTorchTensorRTModule.html
+++ b/docs/_modules/torch_tensorrt/dynamo/runtime/_PythonTorchTensorRTModule.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo.runtime._PythonTorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo.runtime._PythonTorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.html b/docs/_modules/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.html
index fe880a7d17..6519d9a754 100644
--- a/docs/_modules/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.html
+++ b/docs/_modules/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.html
@@ -9,7 +9,7 @@
- torch_tensorrt.dynamo.runtime._TorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo.runtime._TorchTensorRTModule — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -597,7 +597,11 @@ Source code for torch_tensorrt.dynamo.runtime._TorchTensorRTModule
self.serialized_engine = serialized_engine
self.engine = None
- if serialized_engine and not self.settings.lazy_engine_init:
+ if (
+ serialized_engine
+ and not self.settings.lazy_engine_init
+ and not self.settings.enable_cross_compile_for_windows
+ ):
self.setup_engine()
def _pack_engine_info(self) -> List[str | bytes]:
@@ -609,16 +613,16 @@ Source code for torch_tensorrt.dynamo.runtime._TorchTensorRTModule
metadata = {"settings": self.settings, "weight_name_map": self.weight_name_map}
target_platform = (
Platform.current_platform()
+ if not self.settings.enable_cross_compile_for_windows
+ else Platform.WIN_X86_64
) # Change to match target for engine
engine_info: List[str | bytes] = [""] * SERIALIZATION_LEN
-
engine_info[ABI_TARGET_IDX] = torch.ops.tensorrt.ABI_VERSION()
engine_info[NAME_IDX] = (
self.name + "_engine" if self.name != "" else "tensorrt_engine"
)
engine_info[DEVICE_IDX] = target_device._to_serialized_rt_device()
-
assert self.serialized_engine
engine_info[ENGINE_IDX] = self.serialized_engine
diff --git a/docs/_modules/torch_tensorrt/fx/fx2trt.html b/docs/_modules/torch_tensorrt/fx/fx2trt.html
index aa21dce754..b6de4cde0b 100644
--- a/docs/_modules/torch_tensorrt/fx/fx2trt.html
+++ b/docs/_modules/torch_tensorrt/fx/fx2trt.html
@@ -9,7 +9,7 @@
- torch_tensorrt.fx.fx2trt — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.fx.fx2trt — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html b/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html
index 720eadd8b2..30dd39a936 100644
--- a/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html
+++ b/docs/_modules/torch_tensorrt/fx/input_tensor_spec.html
@@ -9,7 +9,7 @@
- torch_tensorrt.fx.input_tensor_spec — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.fx.input_tensor_spec — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/fx/lower.html b/docs/_modules/torch_tensorrt/fx/lower.html
index 9b97a502c8..03d4d0ae65 100644
--- a/docs/_modules/torch_tensorrt/fx/lower.html
+++ b/docs/_modules/torch_tensorrt/fx/lower.html
@@ -9,7 +9,7 @@
- torch_tensorrt.fx.lower — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.fx.lower — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/fx/trt_module.html b/docs/_modules/torch_tensorrt/fx/trt_module.html
index b0ee9a0e1f..f7a57b18b9 100644
--- a/docs/_modules/torch_tensorrt/fx/trt_module.html
+++ b/docs/_modules/torch_tensorrt/fx/trt_module.html
@@ -9,7 +9,7 @@
- torch_tensorrt.fx.trt_module — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.fx.trt_module — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/logging.html b/docs/_modules/torch_tensorrt/logging.html
index aa798e0766..977301159b 100644
--- a/docs/_modules/torch_tensorrt/logging.html
+++ b/docs/_modules/torch_tensorrt/logging.html
@@ -9,7 +9,7 @@
- torch_tensorrt.logging — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.logging — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/runtime/_multi_device_safe_mode.html b/docs/_modules/torch_tensorrt/runtime/_multi_device_safe_mode.html
index 33183066b9..c9ff854d62 100644
--- a/docs/_modules/torch_tensorrt/runtime/_multi_device_safe_mode.html
+++ b/docs/_modules/torch_tensorrt/runtime/_multi_device_safe_mode.html
@@ -9,7 +9,7 @@
- torch_tensorrt.runtime._multi_device_safe_mode — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.runtime._multi_device_safe_mode — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/ts/_compile_spec.html b/docs/_modules/torch_tensorrt/ts/_compile_spec.html
index a613defb91..0ca5ae7d64 100644
--- a/docs/_modules/torch_tensorrt/ts/_compile_spec.html
+++ b/docs/_modules/torch_tensorrt/ts/_compile_spec.html
@@ -9,7 +9,7 @@
- torch_tensorrt.ts._compile_spec — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.ts._compile_spec — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/ts/_compiler.html b/docs/_modules/torch_tensorrt/ts/_compiler.html
index 2a4988ad2e..8ec9e198d9 100644
--- a/docs/_modules/torch_tensorrt/ts/_compiler.html
+++ b/docs/_modules/torch_tensorrt/ts/_compiler.html
@@ -9,7 +9,7 @@
- torch_tensorrt.ts._compiler — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.ts._compiler — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_modules/torch_tensorrt/ts/ptq.html b/docs/_modules/torch_tensorrt/ts/ptq.html
index df13b1409d..7f3e59fbfc 100644
--- a/docs/_modules/torch_tensorrt/ts/ptq.html
+++ b/docs/_modules/torch_tensorrt/ts/ptq.html
@@ -9,7 +9,7 @@
- torch_tensorrt.ts.ptq — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.ts.ptq — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/_sources/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.rst.txt b/docs/_sources/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.rst.txt
new file mode 100644
index 0000000000..dfc8544c0c
--- /dev/null
+++ b/docs/_sources/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.rst.txt
@@ -0,0 +1,142 @@
+
+.. DO NOT EDIT.
+.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
+.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
+.. "tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.py"
+.. LINE NUMBERS ARE GIVEN BELOW.
+
+.. only:: html
+
+ .. note::
+ :class: sphx-glr-download-link-note
+
+ :ref:`Go to the end `
+ to download the full example code
+
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py:
+
+
+.. _resnet_cross_runtime_compilation_for_windows_example:
+
+cross runtime compilation limitations:
+The cross compile and saved model can only be loaded in Windows, it can no longer be loaded in Linux
+The cross compile and saved model can only be loaded in the same Compute Capability as the Linux which it was cross compiled
+(for example, if the model was cross compiled in Linux with GeForceRTX 4080 which has Compute Capability of 8.9,
+It cannot be loaded in Windows with GeForceRTX 3080 which has Compute Capability of 8.6)
+
+Cross runtime compilation for windows example
+======================================================
+
+Compile and save the Resnet Model using Torch-TensorRT in Linux:
+
+python examples/dynamo/cross_runtime_compilation_for_windows.py --path trt_resnet.ep
+
+Load the Resnet Model saved in Windows:
+
+python examples/dynamo/cross_runtime_compilation_for_windows.py --path trt_resnet.ep --load True
+
+.. GENERATED FROM PYTHON SOURCE LINES 24-26
+
+Imports and Model Definition
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. GENERATED FROM PYTHON SOURCE LINES 26-53
+
+.. code-block:: python
+
+
+ import argparse
+ import platform
+
+ import torch
+ import torch_tensorrt as torchtrt
+ import torchvision.models as models
+
+ PARSER = argparse.ArgumentParser(
+ description="Cross runtime comilation for windows example: Resnet Model"
+ )
+ PARSER.add_argument(
+ "--load", default=False, type=bool, required=False, help="Load the model in Windows"
+ )
+ PARSER.add_argument(
+ "--path",
+ type=str,
+ required=True,
+ help="Path to the saved model file",
+ )
+
+ args = PARSER.parse_args()
+ torch.manual_seed(0)
+ model = models.resnet18().eval().cuda()
+ input = torch.rand((1, 3, 224, 224)).to("cuda")
+ inputs = [input]
+
+
+.. GENERATED FROM PYTHON SOURCE LINES 54-57
+
+According to the argument, it is either cross compile and save resnet model for windows in Linux
+or load the saved resnet model in Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. GENERATED FROM PYTHON SOURCE LINES 57-83
+
+.. code-block:: python
+
+ if args.load:
+ # load the saved model in Windows
+ if platform.system() != "Windows" or platform.machine() != "AMD64":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be loaded in Windows system"
+ )
+ loaded_model = torchtrt.load_cross_compiled_exported_program(args.path).module()
+ print(f"model has been successfully loaded from ${args.path}")
+ # inference
+ trt_output = loaded_model(input)
+ print(f"inference result: {trt_output}")
+ else:
+ if platform.system() != "Linux" or platform.architecture()[0] != "64bit":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be compiled in Linux system"
+ )
+ compile_spec = {
+ "debug": True,
+ "min_block_size": 1,
+ }
+ torchtrt.cross_compile_for_windows(
+ model, file_path=args.path, inputs=inputs, **compile_spec
+ )
+ print(
+ f"model has been successfully cross compiled and saved in Linux to {args.path}"
+ )
+
+
+.. rst-class:: sphx-glr-timing
+
+ **Total running time of the script:** ( 0 minutes 0.000 seconds)
+
+
+.. _sphx_glr_download_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py:
+
+.. only:: html
+
+ .. container:: sphx-glr-footer sphx-glr-footer-example
+
+
+
+
+ .. container:: sphx-glr-download sphx-glr-download-python
+
+ :download:`Download Python source code: cross_runtime_compilation_for_windows.py `
+
+ .. container:: sphx-glr-download sphx-glr-download-jupyter
+
+ :download:`Download Jupyter notebook: cross_runtime_compilation_for_windows.ipynb `
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+ `Gallery generated by Sphinx-Gallery `_
diff --git a/docs/_sources/tutorials/_rendered_examples/dynamo/index.rst.txt b/docs/_sources/tutorials/_rendered_examples/dynamo/index.rst.txt
index d3f55ac92c..da129a36fc 100644
--- a/docs/_sources/tutorials/_rendered_examples/dynamo/index.rst.txt
+++ b/docs/_sources/tutorials/_rendered_examples/dynamo/index.rst.txt
@@ -64,6 +64,23 @@ Model Zoo
+.. raw:: html
+
+
+
+.. only:: html
+
+ .. image:: /tutorials/_rendered_examples/dynamo/images/thumb/sphx_glr_cross_runtime_compilation_for_windows_thumb.png
+ :alt:
+
+ :ref:`sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py`
+
+.. raw:: html
+
+ cross runtime compilation limitations:
+
+
+
.. raw:: html
@@ -295,6 +312,7 @@ Model Zoo
/tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion
/tutorials/_rendered_examples/dynamo/torch_export_cudagraphs
+ /tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows
/tutorials/_rendered_examples/dynamo/refit_engine_example
/tutorials/_rendered_examples/dynamo/torch_compile_transformers_example
/tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage
diff --git a/docs/_sources/tutorials/_rendered_examples/index.rst.txt b/docs/_sources/tutorials/_rendered_examples/index.rst.txt
index 6a994d6a40..c84129b82d 100644
--- a/docs/_sources/tutorials/_rendered_examples/index.rst.txt
+++ b/docs/_sources/tutorials/_rendered_examples/index.rst.txt
@@ -76,6 +76,23 @@ Model Zoo
+.. raw:: html
+
+
+
+.. only:: html
+
+ .. image:: /tutorials/_rendered_examples/dynamo/images/thumb/sphx_glr_cross_runtime_compilation_for_windows_thumb.png
+ :alt:
+
+ :ref:`sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py`
+
+.. raw:: html
+
+ cross runtime compilation limitations:
+
+
+
.. raw:: html
diff --git a/docs/_static/documentation_options.js b/docs/_static/documentation_options.js
index fab9dcff18..aea1c97b9a 100644
--- a/docs/_static/documentation_options.js
+++ b/docs/_static/documentation_options.js
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
- VERSION: 'v2.6.0.dev0+e43833d',
+ VERSION: 'v2.6.0.dev0+bc95015',
LANGUAGE: 'en',
COLLAPSE_INDEX: false,
BUILDER: 'html',
diff --git a/docs/cli/torchtrtc.html b/docs/cli/torchtrtc.html
index 823baebf5c..dee9d2ac08 100644
--- a/docs/cli/torchtrtc.html
+++ b/docs/cli/torchtrtc.html
@@ -10,7 +10,7 @@
- torchtrtc — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torchtrtc — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/conversion.html b/docs/contributors/conversion.html
index ffedb16e73..7b2a125b58 100644
--- a/docs/contributors/conversion.html
+++ b/docs/contributors/conversion.html
@@ -10,7 +10,7 @@
- Conversion Phase — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Conversion Phase — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/dynamo_converters.html b/docs/contributors/dynamo_converters.html
index fa5359a3da..2f8cf9cf48 100644
--- a/docs/contributors/dynamo_converters.html
+++ b/docs/contributors/dynamo_converters.html
@@ -10,7 +10,7 @@
- Writing Dynamo Converters — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Writing Dynamo Converters — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/lowering.html b/docs/contributors/lowering.html
index ce709a213c..b25bbd1410 100644
--- a/docs/contributors/lowering.html
+++ b/docs/contributors/lowering.html
@@ -10,7 +10,7 @@
- Lowering Phase — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Lowering Phase — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/partitioning.html b/docs/contributors/partitioning.html
index 962b4e70b3..ace43e223b 100644
--- a/docs/contributors/partitioning.html
+++ b/docs/contributors/partitioning.html
@@ -10,7 +10,7 @@
- Partitioning Phase — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Partitioning Phase — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/phases.html b/docs/contributors/phases.html
index cb31b0631f..7fce911f26 100644
--- a/docs/contributors/phases.html
+++ b/docs/contributors/phases.html
@@ -10,7 +10,7 @@
- Compiler Phases — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiler Phases — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/runtime.html b/docs/contributors/runtime.html
index 5c496df381..30a71c3a91 100644
--- a/docs/contributors/runtime.html
+++ b/docs/contributors/runtime.html
@@ -10,7 +10,7 @@
- Runtime Phase — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Runtime Phase — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/system_overview.html b/docs/contributors/system_overview.html
index ca4b7ec1e1..9eefbca1f3 100644
--- a/docs/contributors/system_overview.html
+++ b/docs/contributors/system_overview.html
@@ -10,7 +10,7 @@
- System Overview — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ System Overview — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/ts_converters.html b/docs/contributors/ts_converters.html
index d7c28adbe9..7d66f772d9 100644
--- a/docs/contributors/ts_converters.html
+++ b/docs/contributors/ts_converters.html
@@ -10,7 +10,7 @@
- Writing TorchScript Converters — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Writing TorchScript Converters — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/useful_links.html b/docs/contributors/useful_links.html
index 0b9cd6a1dd..ffe89b0070 100644
--- a/docs/contributors/useful_links.html
+++ b/docs/contributors/useful_links.html
@@ -10,7 +10,7 @@
- Useful Links for Torch-TensorRT Development — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Useful Links for Torch-TensorRT Development — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/contributors/writing_dynamo_aten_lowering_passes.html b/docs/contributors/writing_dynamo_aten_lowering_passes.html
index 134cec11c0..41fdacdc86 100644
--- a/docs/contributors/writing_dynamo_aten_lowering_passes.html
+++ b/docs/contributors/writing_dynamo_aten_lowering_passes.html
@@ -10,7 +10,7 @@
- Writing Dynamo ATen Lowering Passes — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Writing Dynamo ATen Lowering Passes — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/dynamo/dynamo_export.html b/docs/dynamo/dynamo_export.html
index d6a79b8697..22bc36f2f4 100644
--- a/docs/dynamo/dynamo_export.html
+++ b/docs/dynamo/dynamo_export.html
@@ -10,7 +10,7 @@
- Compiling Exported Programs with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling Exported Programs with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/dynamo/torch_compile.html b/docs/dynamo/torch_compile.html
index 63703b6cf3..dc076f8a48 100644
--- a/docs/dynamo/torch_compile.html
+++ b/docs/dynamo/torch_compile.html
@@ -10,7 +10,7 @@
- TensorRT Backend for torch.compile — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ TensorRT Backend for torch.compile — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -488,7 +488,7 @@ Key Features¶
-
-class torch_tensorrt.dynamo.CompilationSettings(enabled_precisions: ~typing.Set[~torch_tensorrt._enums.dtype] = <factory>, debug: bool = False, workspace_size: int = 0, min_block_size: int = 5, torch_executed_ops: ~typing.Collection[~typing.Union[~typing.Callable[[...], ~typing.Any], str]] = <factory>, pass_through_build_failures: bool = False, max_aux_streams: ~typing.Optional[int] = None, version_compatible: bool = False, optimization_level: ~typing.Optional[int] = None, use_python_runtime: ~typing.Optional[bool] = False, truncate_double: bool = False, use_fast_partitioner: bool = True, enable_experimental_decompositions: bool = False, device: ~torch_tensorrt._Device.Device = <factory>, require_full_compilation: bool = False, disable_tf32: bool = False, assume_dynamic_shape_support: bool = False, sparse_weights: bool = False, make_refittable: bool = False, engine_capability: ~torch_tensorrt._enums.EngineCapability = <factory>, num_avg_timing_iters: int = 1, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, dryrun: ~typing.Union[bool, str] = False, hardware_compatible: bool = False, timing_cache_path: str = '/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init: bool = False, cache_built_engines: bool = False, reuse_cached_engines: bool = False, use_explicit_typing: bool = False, use_fp32_acc: bool = False, enable_weight_streaming: bool = False)[source]¶
+class torch_tensorrt.dynamo.CompilationSettings(enabled_precisions: ~typing.Set[~torch_tensorrt._enums.dtype] = <factory>, debug: bool = False, workspace_size: int = 0, min_block_size: int = 5, torch_executed_ops: ~typing.Collection[~typing.Union[~typing.Callable[[...], ~typing.Any], str]] = <factory>, pass_through_build_failures: bool = False, max_aux_streams: ~typing.Optional[int] = None, version_compatible: bool = False, optimization_level: ~typing.Optional[int] = None, use_python_runtime: ~typing.Optional[bool] = False, truncate_double: bool = False, use_fast_partitioner: bool = True, enable_experimental_decompositions: bool = False, device: ~torch_tensorrt._Device.Device = <factory>, require_full_compilation: bool = False, disable_tf32: bool = False, assume_dynamic_shape_support: bool = False, sparse_weights: bool = False, make_refittable: bool = False, engine_capability: ~torch_tensorrt._enums.EngineCapability = <factory>, num_avg_timing_iters: int = 1, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, dryrun: ~typing.Union[bool, str] = False, hardware_compatible: bool = False, timing_cache_path: str = '/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init: bool = False, cache_built_engines: bool = False, reuse_cached_engines: bool = False, use_explicit_typing: bool = False, use_fp32_acc: bool = False, enable_weight_streaming: bool = False, enable_cross_compile_for_windows: bool = False)[source]¶
Compilation settings for Torch-TensorRT Dynamo Paths
diff --git a/docs/fx/getting_started_with_fx_path.html b/docs/fx/getting_started_with_fx_path.html
index 3c66c40926..3c7c15cace 100644
--- a/docs/fx/getting_started_with_fx_path.html
+++ b/docs/fx/getting_started_with_fx_path.html
@@ -10,7 +10,7 @@
- Torch-TensorRT (FX Frontend) User Guide — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch-TensorRT (FX Frontend) User Guide — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/genindex.html b/docs/genindex.html
index aa28b75f87..8b3bf23186 100644
--- a/docs/genindex.html
+++ b/docs/genindex.html
@@ -9,7 +9,7 @@
- Index — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Index — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/getting_started/installation.html b/docs/getting_started/installation.html
index 632895bed3..9350c63fcc 100644
--- a/docs/getting_started/installation.html
+++ b/docs/getting_started/installation.html
@@ -10,7 +10,7 @@
- Installation — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Installation — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/getting_started/jetpack.html b/docs/getting_started/jetpack.html
index b035f01dfc..83c31f2a35 100644
--- a/docs/getting_started/jetpack.html
+++ b/docs/getting_started/jetpack.html
@@ -10,7 +10,7 @@
- Overview — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Overview — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/getting_started/quick_start.html b/docs/getting_started/quick_start.html
index fcdb0d4660..b44b907fee 100644
--- a/docs/getting_started/quick_start.html
+++ b/docs/getting_started/quick_start.html
@@ -10,7 +10,7 @@
- Quick Start — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Quick Start — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/index.html b/docs/index.html
index fbec4afbb8..b70f5cb06b 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -10,7 +10,7 @@
- Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -274,7 +274,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/indices/supported_ops.html b/docs/indices/supported_ops.html
index ec5a424566..63db9cdba4 100644
--- a/docs/indices/supported_ops.html
+++ b/docs/indices/supported_ops.html
@@ -10,7 +10,7 @@
- Operators Supported — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Operators Supported — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -274,7 +274,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/objects.inv b/docs/objects.inv
index 17390e6a73..22a52045c1 100644
Binary files a/docs/objects.inv and b/docs/objects.inv differ
diff --git a/docs/py-modindex.html b/docs/py-modindex.html
index 1510b93d70..336d62b086 100644
--- a/docs/py-modindex.html
+++ b/docs/py-modindex.html
@@ -9,7 +9,7 @@
- Python Module Index — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Python Module Index — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/py_api/dynamo.html b/docs/py_api/dynamo.html
index 19d038712f..462e46af1e 100644
--- a/docs/py_api/dynamo.html
+++ b/docs/py_api/dynamo.html
@@ -10,7 +10,7 @@
- torch_tensorrt.dynamo — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.dynamo — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -617,13 +617,14 @@ Functions
-torch_tensorrt.dynamo.export(gm: GraphModule) ExportedProgram[source]¶
+torch_tensorrt.dynamo.export(gm: GraphModule, cross_compile_flag: Optional[bool] = False) ExportedProgram[source]¶
Export the result of TensorRT compilation into the desired output format.
- Parameters
gm (torch.fx.GraphModule) – Compiled Torch-TensorRT module, generated by torch_tensorrt.dynamo.compile
inputs (torch.Tensor) – Torch input tensors
cross_compile_flag (bool) – Flag to indicated whether it is cross_compilation enabled or not
@@ -656,7 +657,7 @@ Functions¶
-
-class torch_tensorrt.dynamo.CompilationSettings(enabled_precisions: ~typing.Set[~torch_tensorrt._enums.dtype] = <factory>, debug: bool = False, workspace_size: int = 0, min_block_size: int = 5, torch_executed_ops: ~typing.Collection[~typing.Union[~typing.Callable[[...], ~typing.Any], str]] = <factory>, pass_through_build_failures: bool = False, max_aux_streams: ~typing.Optional[int] = None, version_compatible: bool = False, optimization_level: ~typing.Optional[int] = None, use_python_runtime: ~typing.Optional[bool] = False, truncate_double: bool = False, use_fast_partitioner: bool = True, enable_experimental_decompositions: bool = False, device: ~torch_tensorrt._Device.Device = <factory>, require_full_compilation: bool = False, disable_tf32: bool = False, assume_dynamic_shape_support: bool = False, sparse_weights: bool = False, make_refittable: bool = False, engine_capability: ~torch_tensorrt._enums.EngineCapability = <factory>, num_avg_timing_iters: int = 1, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, dryrun: ~typing.Union[bool, str] = False, hardware_compatible: bool = False, timing_cache_path: str = '/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init: bool = False, cache_built_engines: bool = False, reuse_cached_engines: bool = False, use_explicit_typing: bool = False, use_fp32_acc: bool = False, enable_weight_streaming: bool = False)[source]¶
+class torch_tensorrt.dynamo.CompilationSettings(enabled_precisions: ~typing.Set[~torch_tensorrt._enums.dtype] = <factory>, debug: bool = False, workspace_size: int = 0, min_block_size: int = 5, torch_executed_ops: ~typing.Collection[~typing.Union[~typing.Callable[[...], ~typing.Any], str]] = <factory>, pass_through_build_failures: bool = False, max_aux_streams: ~typing.Optional[int] = None, version_compatible: bool = False, optimization_level: ~typing.Optional[int] = None, use_python_runtime: ~typing.Optional[bool] = False, truncate_double: bool = False, use_fast_partitioner: bool = True, enable_experimental_decompositions: bool = False, device: ~torch_tensorrt._Device.Device = <factory>, require_full_compilation: bool = False, disable_tf32: bool = False, assume_dynamic_shape_support: bool = False, sparse_weights: bool = False, make_refittable: bool = False, engine_capability: ~torch_tensorrt._enums.EngineCapability = <factory>, num_avg_timing_iters: int = 1, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, dryrun: ~typing.Union[bool, str] = False, hardware_compatible: bool = False, timing_cache_path: str = '/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init: bool = False, cache_built_engines: bool = False, reuse_cached_engines: bool = False, use_explicit_typing: bool = False, use_fp32_acc: bool = False, enable_weight_streaming: bool = False, enable_cross_compile_for_windows: bool = False)[source]¶
Compilation settings for Torch-TensorRT Dynamo Paths
- Parameters
@@ -700,6 +701,8 @@ Classes
- torch_tensorrt.fx — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.fx — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/py_api/logging.html b/docs/py_api/logging.html
index a40cca11ef..668ac2ff8c 100644
--- a/docs/py_api/logging.html
+++ b/docs/py_api/logging.html
@@ -10,7 +10,7 @@
- torch_tensorrt.logging — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.logging — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/py_api/ptq.html b/docs/py_api/ptq.html
index 4493c11807..573eb71f00 100644
--- a/docs/py_api/ptq.html
+++ b/docs/py_api/ptq.html
@@ -10,7 +10,7 @@
- torch_tensorrt.ts.ptq — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.ts.ptq — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/py_api/runtime.html b/docs/py_api/runtime.html
index 03a6ac5b2d..ad57bfcc37 100644
--- a/docs/py_api/runtime.html
+++ b/docs/py_api/runtime.html
@@ -10,7 +10,7 @@
- torch_tensorrt.runtime — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.runtime — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -577,13 +577,13 @@ Classes
-
-class torch_tensorrt.runtime.PythonTorchTensorRTModule(serialized_engine: ~typing.Optional[bytes] = None, input_binding_names: ~typing.Optional[~typing.List[str]] = None, output_binding_names: ~typing.Optional[~typing.List[str]] = None, *, name: str = '', settings: ~torch_tensorrt.dynamo._settings.CompilationSettings = CompilationSettings(enabled_precisions={<dtype.f32: 7>}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, make_refittable=False, engine_capability=<EngineCapability.STANDARD: 1>, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init=False, cache_built_engines=False, reuse_cached_engines=False, use_explicit_typing=False, use_fp32_acc=False, enable_weight_streaming=False), weight_name_map: ~typing.Any = None)[source]¶
+class torch_tensorrt.runtime.PythonTorchTensorRTModule(serialized_engine: ~typing.Optional[bytes] = None, input_binding_names: ~typing.Optional[~typing.List[str]] = None, output_binding_names: ~typing.Optional[~typing.List[str]] = None, *, name: str = '', settings: ~torch_tensorrt.dynamo._settings.CompilationSettings = CompilationSettings(enabled_precisions={<dtype.f32: 7>}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, make_refittable=False, engine_capability=<EngineCapability.STANDARD: 1>, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init=False, cache_built_engines=False, reuse_cached_engines=False, use_explicit_typing=False, use_fp32_acc=False, enable_weight_streaming=False, enable_cross_compile_for_windows=False), weight_name_map: ~typing.Any = None)[source]¶
PythonTorchTensorRTModule is a PyTorch module which encompasses an arbitrary TensorRT Engine.
This module is backed by the Torch-TensorRT runtime and is only compatible with
FX / Dynamo / Python deployments. This module cannot be serialized to torchscript via torch.jit.trace for C++ deployment.
-
-__init__(serialized_engine: ~typing.Optional[bytes] = None, input_binding_names: ~typing.Optional[~typing.List[str]] = None, output_binding_names: ~typing.Optional[~typing.List[str]] = None, *, name: str = '', settings: ~torch_tensorrt.dynamo._settings.CompilationSettings = CompilationSettings(enabled_precisions={<dtype.f32: 7>}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, make_refittable=False, engine_capability=<EngineCapability.STANDARD: 1>, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init=False, cache_built_engines=False, reuse_cached_engines=False, use_explicit_typing=False, use_fp32_acc=False, enable_weight_streaming=False), weight_name_map: ~typing.Any = None)[source]¶
+__init__(serialized_engine: ~typing.Optional[bytes] = None, input_binding_names: ~typing.Optional[~typing.List[str]] = None, output_binding_names: ~typing.Optional[~typing.List[str]] = None, *, name: str = '', settings: ~torch_tensorrt.dynamo._settings.CompilationSettings = CompilationSettings(enabled_precisions={<dtype.f32: 7>}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, make_refittable=False, engine_capability=<EngineCapability.STANDARD: 1>, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/torch_tensorrt_engine_cache/timing_cache.bin', lazy_engine_init=False, cache_built_engines=False, reuse_cached_engines=False, use_explicit_typing=False, use_fp32_acc=False, enable_weight_streaming=False, enable_cross_compile_for_windows=False), weight_name_map: ~typing.Any = None)[source]¶
Takes a name, target device, serialized TensorRT engine, and binding names / order and constructs
a PyTorch torch.nn.Module around it. Uses TensorRT Python APIs to run the engine
diff --git a/docs/py_api/torch_tensorrt.html b/docs/py_api/torch_tensorrt.html
index 3e426876f4..338d46c2a4 100644
--- a/docs/py_api/torch_tensorrt.html
+++ b/docs/py_api/torch_tensorrt.html
@@ -10,7 +10,7 @@
- torch_tensorrt — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/py_api/ts.html b/docs/py_api/ts.html
index 79ddefc111..1c1a98b163 100644
--- a/docs/py_api/ts.html
+++ b/docs/py_api/ts.html
@@ -10,7 +10,7 @@
- torch_tensorrt.ts — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ torch_tensorrt.ts — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -691,7 +691,7 @@ Functions
-
-torch_tensorrt.ts.TensorRTCompileSpec(inputs: Optional[List[torch.Tensor | Input]] = None, input_signature: Optional[Any] = None, device: torch.device | Device = Device(type=DeviceType.GPU, gpu_id=0), disable_tf32: bool = False, sparse_weights: bool = False, enabled_precisions: Optional[Set[Union[dtype, dtype]]] = None, refit: bool = False, debug: bool = False, capability: EngineCapability = EngineCapability.STANDARD, num_avg_timing_iters: int = 1, workspace_size: int = 0, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, truncate_long_and_double: bool = False, calibrator: object = None, allow_shape_tensors: bool = False) <torch.ScriptClass object at 0x7efd148567b0>[source]¶
+torch_tensorrt.ts.TensorRTCompileSpec(inputs: Optional[List[torch.Tensor | Input]] = None, input_signature: Optional[Any] = None, device: torch.device | Device = Device(type=DeviceType.GPU, gpu_id=0), disable_tf32: bool = False, sparse_weights: bool = False, enabled_precisions: Optional[Set[Union[dtype, dtype]]] = None, refit: bool = False, debug: bool = False, capability: EngineCapability = EngineCapability.STANDARD, num_avg_timing_iters: int = 1, workspace_size: int = 0, dla_sram_size: int = 1048576, dla_local_dram_size: int = 1073741824, dla_global_dram_size: int = 536870912, truncate_long_and_double: bool = False, calibrator: object = None, allow_shape_tensors: bool = False) <torch.ScriptClass object at 0x7fdbd3710cf0>[source]¶
Utility to create a formatted spec dictionary for using the PyTorch TensorRT backend
- Keyword Arguments
diff --git a/docs/search.html b/docs/search.html
index b64d9bc7d9..62bc88d19f 100644
--- a/docs/search.html
+++ b/docs/search.html
@@ -9,7 +9,7 @@
- Search — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Search — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -272,7 +272,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/searchindex.js b/docs/searchindex.js
index fb41a175b4..afbb8d4006 100644
--- a/docs/searchindex.js
+++ b/docs/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["_cpp_api/classtorch__tensorrt_1_1DataType", "_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType", "_cpp_api/classtorch__tensorrt_1_1TensorFormat", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator", "_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502", "_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268", "_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e", "_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827", "_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b", "_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da", "_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59", "_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883", "_cpp_api/dir_cpp", "_cpp_api/dir_cpp_include", "_cpp_api/dir_cpp_include_torch_tensorrt", "_cpp_api/enum_namespacetorch__tensorrt_1_1logging_1a130f65408ad8cbaee060f05e8db69558", "_cpp_api/enum_namespacetorch__tensorrt_1a3fbe5d72e4fc624dbd038853079620eb", "_cpp_api/file_cpp_include_torch_tensorrt_logging.h", "_cpp_api/file_cpp_include_torch_tensorrt_macros.h", "_cpp_api/file_cpp_include_torch_tensorrt_ptq.h", "_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0593f776f469c20469e2f729fc7861a3", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0c012cb374addd90eb1f42eaec570650", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a56e110feaaba2c3fd44bd201fd21a76a", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a7cb50492421ea9de4e3db895819df6f2", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ac46ac0901cb97e3ae6e93b45f24e90b8", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ad2efd47b6c3689e58ccc595680579ae5", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1af8f3443813315af7901903d25dd495cc", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a226e3c83379d1012cde8578c1c86b16c", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a6186e305f47c1d94b6130ef6c7f7e178", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a5b405fd3bf3c8fc2e2a54cbbab979797", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a6e19490a08fb1553c9dd347a5ae79db9", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a81f9783517335dda877d8cfcf38987c9", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1ae8d56472106eeef37fbe51ff7f40c9b2", "_cpp_api/function_namespacetorch__tensorrt_1ac4ab8313ae72c2c899ea31548b528528", "_cpp_api/function_namespacetorch__tensorrt_1ad1acd06eaeaffbbcf6e7ebf426891384", "_cpp_api/function_namespacetorch__tensorrt_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1", "_cpp_api/namespace_torch_tensorrt", "_cpp_api/namespace_torch_tensorrt__logging", "_cpp_api/namespace_torch_tensorrt__ptq", "_cpp_api/namespace_torch_tensorrt__torchscript", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h", "_cpp_api/structtorch__tensorrt_1_1Device", "_cpp_api/structtorch__tensorrt_1_1GraphInputs", "_cpp_api/structtorch__tensorrt_1_1Input", "_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec", "_cpp_api/torch_tensort_cpp", "_cpp_api/unabridged_orphan", "cli/torchtrtc", "contributors/conversion", "contributors/dynamo_converters", "contributors/lowering", "contributors/partitioning", "contributors/phases", "contributors/runtime", "contributors/system_overview", "contributors/ts_converters", "contributors/useful_links", "contributors/writing_dynamo_aten_lowering_passes", "dynamo/dynamo_export", "dynamo/torch_compile", "fx/getting_started_with_fx_path", "getting_started/installation", "getting_started/jetpack", "getting_started/quick_start", "index", "indices/supported_ops", "py_api/dynamo", "py_api/fx", "py_api/logging", "py_api/ptq", "py_api/runtime", "py_api/torch_tensorrt", "py_api/ts", "sg_execution_times", "src/pytorch-sphinx-theme/docs/changelog", "src/pytorch-sphinx-theme/docs/configuring", "src/pytorch-sphinx-theme/docs/demo/api", "src/pytorch-sphinx-theme/docs/demo/demo", "src/pytorch-sphinx-theme/docs/demo/lists_tables", "src/pytorch-sphinx-theme/docs/demo/long", "src/pytorch-sphinx-theme/docs/demo/structure", "src/pytorch-sphinx-theme/docs/index", "src/pytorch-sphinx-theme/docs/installing", "ts/creating_torchscript_module_in_python", "ts/getting_started_with_cpp_api", "ts/getting_started_with_python_api", "ts/ptq", "ts/torchscript_frontend_from_pytorch", "tutorials/_rendered_examples/dynamo/converter_overloading", "tutorials/_rendered_examples/dynamo/custom_kernel_plugins", "tutorials/_rendered_examples/dynamo/engine_caching_bert_example", "tutorials/_rendered_examples/dynamo/engine_caching_example", "tutorials/_rendered_examples/dynamo/index", "tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example", "tutorials/_rendered_examples/dynamo/refit_engine_example", "tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage", "tutorials/_rendered_examples/dynamo/torch_compile_resnet_example", "tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion", "tutorials/_rendered_examples/dynamo/torch_compile_transformers_example", "tutorials/_rendered_examples/dynamo/torch_export_cudagraphs", "tutorials/_rendered_examples/dynamo/torch_export_gpt2", "tutorials/_rendered_examples/dynamo/torch_export_llama2", "tutorials/_rendered_examples/dynamo/vgg16_ptq", "tutorials/_rendered_examples/dynamo/weight_streaming_example", "tutorials/_rendered_examples/index", "tutorials/notebooks", "tutorials/serving_torch_tensorrt_with_triton", "user_guide/dynamic_shapes", "user_guide/mixed_precision", "user_guide/runtime", "user_guide/saving_models", "user_guide/torch_tensorrt_explained", "user_guide/using_dla"], "filenames": ["_cpp_api/classtorch__tensorrt_1_1DataType.rst", "_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.rst", "_cpp_api/classtorch__tensorrt_1_1TensorFormat.rst", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.rst", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.rst", "_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.rst", "_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.rst", "_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.rst", "_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.rst", "_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.rst", "_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.rst", "_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.rst", "_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.rst", "_cpp_api/dir_cpp.rst", "_cpp_api/dir_cpp_include.rst", "_cpp_api/dir_cpp_include_torch_tensorrt.rst", "_cpp_api/enum_namespacetorch__tensorrt_1_1logging_1a130f65408ad8cbaee060f05e8db69558.rst", "_cpp_api/enum_namespacetorch__tensorrt_1a3fbe5d72e4fc624dbd038853079620eb.rst", "_cpp_api/file_cpp_include_torch_tensorrt_logging.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0593f776f469c20469e2f729fc7861a3.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0c012cb374addd90eb1f42eaec570650.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a56e110feaaba2c3fd44bd201fd21a76a.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a7cb50492421ea9de4e3db895819df6f2.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ac46ac0901cb97e3ae6e93b45f24e90b8.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ad2efd47b6c3689e58ccc595680579ae5.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1af8f3443813315af7901903d25dd495cc.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a226e3c83379d1012cde8578c1c86b16c.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a6186e305f47c1d94b6130ef6c7f7e178.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a5b405fd3bf3c8fc2e2a54cbbab979797.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a6e19490a08fb1553c9dd347a5ae79db9.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a81f9783517335dda877d8cfcf38987c9.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1ae8d56472106eeef37fbe51ff7f40c9b2.rst", "_cpp_api/function_namespacetorch__tensorrt_1ac4ab8313ae72c2c899ea31548b528528.rst", "_cpp_api/function_namespacetorch__tensorrt_1ad1acd06eaeaffbbcf6e7ebf426891384.rst", "_cpp_api/function_namespacetorch__tensorrt_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.rst", "_cpp_api/namespace_torch_tensorrt.rst", "_cpp_api/namespace_torch_tensorrt__logging.rst", "_cpp_api/namespace_torch_tensorrt__ptq.rst", "_cpp_api/namespace_torch_tensorrt__torchscript.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst", "_cpp_api/structtorch__tensorrt_1_1Device.rst", "_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst", "_cpp_api/structtorch__tensorrt_1_1Input.rst", "_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.rst", "_cpp_api/torch_tensort_cpp.rst", "_cpp_api/unabridged_orphan.rst", "cli/torchtrtc.rst", "contributors/conversion.rst", "contributors/dynamo_converters.rst", "contributors/lowering.rst", "contributors/partitioning.rst", "contributors/phases.rst", "contributors/runtime.rst", "contributors/system_overview.rst", "contributors/ts_converters.rst", "contributors/useful_links.rst", "contributors/writing_dynamo_aten_lowering_passes.rst", "dynamo/dynamo_export.rst", "dynamo/torch_compile.rst", "fx/getting_started_with_fx_path.rst", "getting_started/installation.rst", "getting_started/jetpack.rst", "getting_started/quick_start.rst", "index.rst", "indices/supported_ops.rst", "py_api/dynamo.rst", "py_api/fx.rst", "py_api/logging.rst", "py_api/ptq.rst", "py_api/runtime.rst", "py_api/torch_tensorrt.rst", "py_api/ts.rst", "sg_execution_times.rst", "src/pytorch-sphinx-theme/docs/changelog.rst", "src/pytorch-sphinx-theme/docs/configuring.rst", "src/pytorch-sphinx-theme/docs/demo/api.rst", "src/pytorch-sphinx-theme/docs/demo/demo.rst", "src/pytorch-sphinx-theme/docs/demo/lists_tables.rst", "src/pytorch-sphinx-theme/docs/demo/long.rst", "src/pytorch-sphinx-theme/docs/demo/structure.rst", "src/pytorch-sphinx-theme/docs/index.rst", "src/pytorch-sphinx-theme/docs/installing.rst", "ts/creating_torchscript_module_in_python.rst", "ts/getting_started_with_cpp_api.rst", "ts/getting_started_with_python_api.rst", "ts/ptq.rst", "ts/torchscript_frontend_from_pytorch.rst", "tutorials/_rendered_examples/dynamo/converter_overloading.rst", "tutorials/_rendered_examples/dynamo/custom_kernel_plugins.rst", "tutorials/_rendered_examples/dynamo/engine_caching_bert_example.rst", "tutorials/_rendered_examples/dynamo/engine_caching_example.rst", "tutorials/_rendered_examples/dynamo/index.rst", "tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.rst", "tutorials/_rendered_examples/dynamo/refit_engine_example.rst", "tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.rst", "tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.rst", "tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.rst", "tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.rst", "tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.rst", "tutorials/_rendered_examples/dynamo/torch_export_gpt2.rst", "tutorials/_rendered_examples/dynamo/torch_export_llama2.rst", "tutorials/_rendered_examples/dynamo/vgg16_ptq.rst", "tutorials/_rendered_examples/dynamo/weight_streaming_example.rst", "tutorials/_rendered_examples/index.rst", "tutorials/notebooks.rst", "tutorials/serving_torch_tensorrt_with_triton.rst", "user_guide/dynamic_shapes.rst", "user_guide/mixed_precision.rst", "user_guide/runtime.rst", "user_guide/saving_models.rst", "user_guide/torch_tensorrt_explained.rst", "user_guide/using_dla.rst"], "titles": ["Class DataType", "Class Device::DeviceType", "Class TensorFormat", "Template Class Int8CacheCalibrator", "Template Class Int8Calibrator", "Define STR", "Define TORCH_TENSORRT_PATCH_VERSION", "Define TORCH_TENSORRT_MAJOR_VERSION", "Define TORCH_TENSORRT_MINOR_VERSION", "Define TORCHTRT_API", "Define XSTR", "Define TORCHTRT_HIDDEN", "Define TORCH_TENSORRT_VERSION", "Directory cpp", "Directory include", "Directory torch_tensorrt", "Enum Level", "Enum EngineCapability", "File logging.h", "File macros.h", "File ptq.h", "File torch_tensorrt.h", "Function torch_tensorrt::logging::get_logging_prefix", "Function torch_tensorrt::logging::get_reportable_log_level", "Function torch_tensorrt::logging::get_is_colored_output_on", "Function torch_tensorrt::logging::set_reportable_log_level", "Function torch_tensorrt::logging::log", "Function torch_tensorrt::logging::set_is_colored_output_on", "Function torch_tensorrt::logging::set_logging_prefix", "Template Function torch_tensorrt::ptq::make_int8_cache_calibrator", "Template Function torch_tensorrt::ptq::make_int8_calibrator", "Function torch_tensorrt::torchscript::check_method_operator_support", "Function torch_tensorrt::torchscript::compile", "Function torch_tensorrt::torchscript::embed_engine_in_new_module", "Function torch_tensorrt::torchscript::convert_method_to_trt_engine", "Function torch_tensorrt::get_build_info", "Function torch_tensorrt::set_device", "Function torch_tensorrt::dump_build_info", "Namespace torch_tensorrt", "Namespace torch_tensorrt::logging", "Namespace torch_tensorrt::ptq", "Namespace torch_tensorrt::torchscript", "Program Listing for File logging.h", "Program Listing for File macros.h", "Program Listing for File ptq.h", "Program Listing for File torch_tensorrt.h", "Struct Device", "Struct GraphInputs", "Struct Input", "Struct CompileSpec", "Torch-TensorRT C++ API", "Full API", "torchtrtc", "Conversion Phase", "Writing Dynamo Converters", "Lowering Phase", "Partitioning Phase", "Compiler Phases", "Runtime Phase", "System Overview", "Writing TorchScript Converters", "Useful Links for Torch-TensorRT Development", "Writing Dynamo ATen Lowering Passes", "Compiling Exported Programs with Torch-TensorRT", "TensorRT Backend for torch.compile", "Torch-TensorRT (FX Frontend) User Guide", "Installation", "Overview", "Quick Start", "Torch-TensorRT", "Operators Supported", "torch_tensorrt.dynamo", "torch_tensorrt.fx", "torch_tensorrt.logging", "torch_tensorrt.ts.ptq", "torch_tensorrt.runtime", "torch_tensorrt", "torch_tensorrt.ts", "Computation times", "Changelog", "Configuration", "5. :mod:`test_py_module`", "3. Paragraph Level Markup", "4. Lists & Tables", "1. Long Sticky Nav", "1. Structural Elements", "<no title>", "Installation", "Creating a TorchScript Module", "Using Torch-TensorRT in C++", "Using Torch-TensorRT in Python", "Post Training Quantization (PTQ)", "Using Torch-TensorRT TorchScript Frontend Directly From PyTorch", "Overloading Torch-TensorRT Converters with Custom Converters", "Using Custom Kernels within TensorRT Engines with Torch-TensorRT", "Engine Caching (BERT)", "Engine Caching", "Dependencies", "Mutable Torch TensorRT Module", "Refitting Torch-TensorRT Programs with New Weights", "Torch Compile Advanced Usage", "Compiling ResNet with dynamic shapes using the torch.compile backend", "Compiling Stable Diffusion model using the torch.compile backend", "Compiling BERT using the torch.compile backend", "Torch Export with Cudagraphs", "Compiling GPT2 using the dynamo backend", "Compiling Llama2 using the dynamo backend", "Deploy Quantized Models using Torch-TensorRT", "Weight Streaming", "Torch-TensorRT Tutorials", "Legacy notebooks", "Serving a Torch-TensorRT model with Triton", "Dynamic shapes with Torch-TensorRT", "Compile Mixed Precision models with Torch-TensorRT", "Deploying Torch-TensorRT Programs", "Saving models compiled with Torch-TensorRT", "Torch-TensorRT Explained", "DLA"], "terms": {"defin": [0, 1, 2, 3, 4, 33, 43, 46, 47, 48, 49, 51, 52, 54, 65, 68, 75, 76, 80, 88, 89, 90, 91, 93, 94, 96, 100, 103, 104, 105, 106, 110], "file": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 46, 47, 48, 49, 52, 54, 56, 58, 59, 64, 65, 66, 67, 68, 71, 72, 74, 76, 77, 78, 80, 81, 83, 87, 89, 91, 111, 112, 115], "torch_tensorrt": [0, 1, 2, 14, 16, 17, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 54, 56, 62, 63, 64, 65, 68, 69, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 117], "h": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 46, 47, 48, 49, 50, 51, 52, 55, 68, 76, 89, 91], "support": [0, 1, 2, 27, 31, 46, 48, 49, 52, 54, 56, 61, 63, 65, 67, 68, 69, 72, 75, 76, 77, 80, 81, 88, 89, 90, 93, 94, 99, 101, 103, 105, 106, 107, 108, 111, 113, 116, 117], "data": [0, 2, 3, 4, 29, 30, 44, 46, 48, 49, 52, 53, 56, 57, 59, 60, 64, 65, 70, 71, 72, 74, 76, 77, 82, 86, 90, 91, 94, 96, 107, 108, 110], "type": [0, 1, 2, 30, 49, 50, 52, 53, 56, 58, 60, 62, 63, 64, 65, 71, 72, 74, 75, 76, 77, 82, 89, 90, 91, 93, 94, 96, 107, 108, 110, 113, 115], "can": [0, 1, 4, 29, 30, 34, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 71, 74, 75, 76, 77, 80, 82, 88, 89, 90, 91, 92, 93, 94, 96, 98, 99, 100, 103, 104, 107, 108, 110, 111, 112, 113, 114, 115, 116], "us": [0, 1, 2, 3, 4, 29, 30, 32, 34, 36, 43, 44, 45, 46, 48, 49, 52, 53, 54, 56, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 76, 77, 78, 80, 81, 82, 83, 88, 91, 96, 97, 98, 99, 108, 109, 111, 113, 114, 115, 116, 117], "tensorrt": [0, 1, 3, 4, 29, 30, 31, 32, 33, 34, 37, 44, 45, 46, 48, 49, 52, 53, 54, 55, 56, 57, 59, 60, 62, 67, 68, 71, 72, 74, 75, 76, 77, 88, 91, 96, 97, 100, 101, 102, 103, 104, 108], "engin": [0, 1, 17, 32, 33, 34, 45, 46, 48, 49, 52, 53, 56, 57, 59, 62, 63, 64, 69, 71, 72, 75, 76, 77, 80, 89, 90, 91, 92, 93, 97, 99, 101, 103, 108, 109, 112, 114, 116, 117], "thi": [0, 1, 2, 29, 30, 42, 43, 44, 45, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 71, 72, 75, 76, 77, 80, 81, 82, 84, 85, 88, 89, 91, 92, 93, 94, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116], "compat": [0, 1, 46, 55, 58, 64, 65, 71, 75, 76, 77, 116], "c10": [0, 1, 45, 46, 48, 49, 89, 91], "check": [0, 1, 31, 46, 52, 55, 60, 65, 67, 71, 75, 77, 89, 94, 98, 99, 111, 114], "trt": [0, 1, 3, 4, 46, 48, 53, 55, 58, 60, 62, 64, 65, 67, 68, 70, 71, 75, 76, 89, 93, 94, 103, 105, 106, 108, 112, 114, 115], "so": [0, 44, 52, 53, 54, 55, 58, 59, 60, 62, 64, 65, 66, 67, 72, 75, 76, 81, 82, 83, 89, 91, 93, 94, 96, 100, 101, 103, 105, 106, 112], "should": [0, 3, 4, 29, 45, 49, 52, 53, 54, 55, 56, 57, 59, 60, 63, 64, 65, 67, 71, 75, 76, 77, 80, 82, 85, 91, 93, 94, 95, 96, 99, 104, 111], "reason": [0, 65, 88, 93, 94, 96, 116], "you": [0, 1, 2, 29, 30, 46, 48, 49, 52, 53, 54, 55, 56, 58, 59, 60, 63, 65, 66, 67, 68, 71, 75, 76, 77, 80, 82, 83, 84, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116], "need": [0, 1, 2, 25, 29, 43, 46, 53, 54, 55, 60, 65, 66, 67, 71, 72, 75, 76, 82, 89, 90, 91, 93, 94, 95, 96, 98, 99, 110, 111, 112, 114], "explictli": 0, "public": [0, 1, 2, 3, 4, 44, 45, 46, 47, 48, 49, 83, 91], "enum": [0, 1, 2, 42, 45, 46, 71, 77, 91, 93], "valu": [0, 1, 2, 16, 17, 45, 46, 48, 53, 56, 58, 60, 63, 70, 71, 74, 76, 80, 89, 98, 100, 101, 103, 108, 110], "underli": [0, 1, 2, 46, 60], "In": [0, 1, 2, 46, 53, 54, 56, 57, 58, 59, 60, 64, 65, 66, 75, 76, 82, 83, 85, 90, 91, 93, 94, 98, 110, 111, 112, 113, 114, 115], "case": [0, 1, 2, 46, 49, 53, 54, 56, 58, 60, 62, 64, 65, 66, 67, 75, 76, 91, 93, 94, 98, 99, 112, 113, 114], "itself": [0, 1, 2, 46, 52, 55, 92, 93, 111], "interfac": [0, 1, 2, 46, 58, 59, 60, 64, 69, 91], "vs": [0, 1, 2, 46, 55, 66, 71, 76, 77, 92], "normal": [0, 1, 2, 46, 65, 82, 88, 89, 91, 93, 98, 99, 104, 107, 111, 117], "instatin": [0, 1, 2, 46], "ex": [0, 1, 2, 33, 46, 67, 77, 83, 85], "kfloat": [0, 45, 49], "enumer": [0, 1, 2, 16, 17, 46], "klong": [0, 45], "int64": [0, 76, 77, 108], "kdoubl": [0, 45], "fp64": [0, 76], "fp32": [0, 48, 49, 52, 64, 65, 71, 76, 77, 91, 105, 106, 110, 111, 113], "khalf": [0, 45, 89], "fp16": [0, 48, 49, 52, 64, 65, 71, 72, 76, 89, 90, 98, 102, 105, 106, 108, 113, 117], "kchar": [0, 45], "int8": [0, 44, 48, 49, 52, 64, 71, 76, 77, 91, 107, 117], "kint": [0, 45], "int": [0, 3, 4, 36, 44, 45, 49, 52, 54, 56, 63, 64, 70, 71, 72, 76, 77, 80, 89, 94, 107, 108], "kbool": [0, 45], "bool": [0, 1, 2, 3, 4, 24, 27, 30, 31, 42, 44, 45, 46, 49, 55, 60, 64, 70, 71, 72, 74, 75, 76, 77, 80, 89, 91, 94], "kunknown": [0, 2, 45], "sentinel": [0, 2, 76], "function": [0, 1, 2, 3, 4, 46, 48, 49, 54, 55, 56, 58, 60, 62, 64, 65, 66, 88, 89, 91, 92, 93, 94, 99, 100, 103, 104, 105, 106, 110, 111, 112, 114, 116, 117], "default": [0, 1, 2, 3, 4, 16, 29, 30, 33, 43, 45, 46, 48, 49, 52, 54, 56, 62, 64, 65, 66, 71, 72, 75, 76, 77, 80, 81, 82, 89, 90, 91, 92, 93, 94, 96, 107, 112, 114, 115, 116], "construct": [0, 1, 2, 3, 4, 46, 48, 49, 53, 54, 55, 57, 59, 60, 65, 74, 75, 76, 82, 83, 89, 91, 93, 94, 96, 112], "new": [0, 1, 2, 3, 4, 32, 33, 46, 48, 49, 56, 58, 59, 60, 62, 64, 65, 68, 69, 71, 77, 82, 89, 96, 97, 98, 101, 103, 104, 109, 111, 114], "object": [0, 1, 2, 3, 4, 46, 48, 49, 52, 58, 60, 62, 63, 64, 71, 75, 76, 77, 91, 92, 93, 112, 115], "inlin": [0, 1, 2, 3, 4, 29, 30, 44, 46, 48, 55, 83, 86, 89], "constexpr": [0, 1, 2, 45, 46, 94], "t": [0, 1, 2, 45, 46, 55, 60, 65, 66, 70, 76, 80, 82, 83, 88, 89, 91, 93, 94, 107, 111, 112], "constructor": [0, 2, 46, 48, 49, 58, 88], "from": [0, 1, 2, 3, 4, 29, 30, 44, 46, 48, 49, 52, 53, 55, 56, 57, 58, 59, 60, 63, 64, 65, 67, 69, 71, 72, 75, 76, 77, 78, 80, 81, 82, 83, 88, 89, 91, 93, 94, 95, 96, 98, 99, 102, 103, 105, 106, 107, 108, 110, 111, 114, 115, 116], "torchtrt_api": [0, 2, 19, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33, 34, 35, 36, 37, 42, 43, 44, 45, 48, 49, 50], "scalartyp": [0, 45, 70], "torch": [0, 1, 2, 4, 20, 21, 29, 30, 31, 32, 33, 34, 37, 44, 45, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 67, 71, 72, 74, 75, 76, 77, 78, 88, 91, 95, 96, 97, 108, 117], "paramet": [0, 1, 2, 3, 4, 25, 26, 27, 29, 30, 31, 32, 33, 34, 36, 46, 48, 49, 53, 54, 55, 60, 64, 65, 71, 72, 74, 75, 76, 77, 86, 88, 89, 99, 105, 106], "oper": [0, 1, 2, 3, 4, 31, 44, 45, 46, 49, 52, 53, 55, 56, 57, 58, 59, 60, 62, 63, 65, 69, 71, 76, 77, 90, 91, 93, 99, 101, 103, 116, 117], "const": [0, 1, 2, 3, 4, 29, 30, 31, 32, 33, 34, 36, 44, 45, 46, 55, 60, 70, 89, 91], "get": [0, 1, 2, 3, 4, 23, 35, 44, 46, 55, 56, 60, 62, 63, 65, 67, 75, 76, 89, 91, 93, 96, 105, 106, 108, 110, 111], "return": [0, 1, 2, 3, 4, 23, 24, 29, 30, 31, 32, 33, 34, 35, 42, 43, 44, 45, 46, 54, 55, 56, 57, 58, 59, 60, 62, 64, 65, 71, 72, 75, 76, 77, 88, 89, 90, 91, 93, 94, 96, 99, 100, 107, 108, 111, 112, 113], "explicit": [0, 1, 2, 3, 4, 45, 46, 55, 65, 72, 75, 82, 91, 116], "delet": [0, 1, 2, 45, 46, 55], "other": [0, 1, 2, 45, 46, 52, 53, 55, 58, 62, 64, 65, 66, 70, 71, 75, 76, 81, 82, 89, 90, 93, 114], "comparis": [0, 2], "true": [0, 1, 2, 4, 46, 49, 55, 56, 60, 62, 64, 65, 70, 71, 72, 75, 76, 77, 80, 83, 89, 91, 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 111, 113, 114, 117], "fals": [0, 1, 2, 3, 4, 44, 45, 46, 49, 54, 62, 64, 65, 70, 71, 72, 75, 76, 77, 80, 81, 82, 83, 89, 91, 92, 93, 94, 95, 96, 98, 99, 100, 102, 103, 104, 105, 106, 107, 108, 114], "struct": [1, 21, 38, 41, 45, 54, 91], "onli": [1, 3, 4, 16, 29, 44, 46, 48, 52, 54, 55, 56, 59, 60, 64, 65, 67, 68, 71, 72, 75, 76, 82, 91, 93, 94, 98, 99, 106, 108, 113, 114, 117], "applic": [1, 29, 46, 52, 55, 59, 64, 71, 75, 76, 89, 90, 92, 114, 117], "kcuda": [1, 46, 56, 89], "which": [1, 2, 29, 32, 34, 46, 49, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 71, 72, 74, 75, 76, 77, 80, 82, 83, 88, 89, 90, 91, 92, 93, 94, 96, 100, 101, 104, 105, 106, 110, 111, 112, 113, 114, 115, 116], "map": [1, 46, 53, 54, 55, 57, 59, 60, 65, 76, 89, 91, 92, 96, 100, 110, 111], "kgpu": [1, 45, 46], "To": [1, 46, 52, 54, 56, 64, 66, 71, 80, 88, 89, 90, 92, 94, 99, 105, 106, 111], "datatyp": [1, 21, 38, 45, 46, 48, 49, 50, 71, 76, 77, 90, 94, 111, 113], "target": [1, 33, 45, 46, 48, 49, 52, 54, 56, 58, 59, 64, 65, 66, 69, 71, 75, 76, 77, 90, 91, 92, 93, 94, 99, 116, 117], "gpu": [1, 32, 34, 36, 45, 46, 52, 64, 65, 71, 75, 76, 77, 89, 91, 92, 94, 105, 106, 108, 111, 114, 116, 117], "run": [1, 34, 46, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 64, 65, 66, 67, 68, 71, 72, 75, 76, 77, 82, 88, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 110, 111, 112, 113, 114, 115, 116, 117], "kdla": [1, 45, 46, 117], "dla": [1, 45, 46, 49, 52, 64, 69, 71, 76, 77], "intern": [1, 16, 46, 60, 63, 73, 75, 82, 89], "note": [1, 46, 48, 54, 60, 62, 65, 66, 67, 75, 76, 80, 82, 89, 94, 99, 112, 117], "The": [1, 46, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 71, 75, 76, 77, 80, 83, 88, 90, 91, 92, 93, 94, 95, 96, 98, 99, 101, 104, 105, 108, 110, 111, 112, 115, 116], "valid": [1, 46, 56, 60, 62, 71, 75, 76, 93], "kcpu": [1, 46], "comparison": [1, 46], "an": [2, 3, 4, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 64, 65, 66, 68, 71, 72, 74, 75, 76, 77, 80, 82, 83, 88, 89, 90, 91, 93, 94, 96, 99, 100, 104, 105, 106, 108, 110, 111, 112, 114, 115, 116], "memeori": 2, "layout": [2, 48, 70, 71, 76, 77], "store": [2, 4, 49, 52, 53, 58, 60, 64, 65, 71, 75, 76, 77, 88, 89, 94, 96, 99], "tensor": [2, 33, 44, 45, 48, 49, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 70, 71, 72, 75, 76, 77, 88, 89, 90, 91, 93, 94, 100, 108, 110], "kcontigu": [2, 45, 48], "contigu": [2, 48, 49, 52, 71, 76, 77], "nchw": [2, 71, 76, 77], "linear": [2, 56, 70, 76, 88, 94, 107, 113], "kchannelslast": [2, 45], "channel": [2, 76, 81], "last": [2, 55, 65, 76, 107], "nhwc": [2, 52], "memoryformat": [2, 45], "ptq": [3, 4, 15, 18, 19, 38, 50, 51, 52, 69, 71, 76, 77], "privat": [3, 4, 44, 45, 91], "algorithm": [3, 4, 29, 30, 44, 65, 74, 91, 106], "typenam": [3, 4, 29, 30, 44], "gener": [3, 4, 29, 52, 55, 58, 59, 60, 62, 64, 65, 66, 71, 72, 80, 82, 83, 86, 88, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 114], "int8calibr": [3, 20, 30, 40, 44, 50], "implement": [3, 4, 55, 56, 58, 63, 65, 75, 81, 89, 91, 94, 96, 114], "specifi": [3, 4, 33, 52, 54, 60, 64, 65, 66, 71, 76, 77, 80, 82, 90, 92, 108, 111, 112, 113, 115, 116], "calibr": [3, 4, 29, 30, 44, 49, 52, 71, 74, 76, 77, 89, 91], "read": [3, 4, 29, 30, 44, 80, 82, 91], "nvinfer1": [3, 4, 29, 30, 44, 45, 49, 60, 91], "iint8calibr": [3, 4, 29, 30, 44, 45, 49, 71, 76, 77, 91], "iint8entropycalibrator2": [3, 4, 29, 30, 44, 91], "std": [3, 4, 22, 26, 28, 29, 30, 31, 33, 34, 35, 42, 44, 45, 47, 48, 49, 56, 89, 91, 111, 117], "string": [3, 4, 18, 20, 21, 22, 26, 28, 29, 30, 31, 33, 34, 35, 42, 44, 45, 49, 54, 56, 58, 60, 64, 71, 76, 80, 89, 91], "cache_file_path": [3, 4, 29, 30, 44], "8": [3, 52, 55, 63, 64, 66, 75, 76, 82, 83, 86, 89, 94, 101, 104, 111, 112], "cach": [3, 4, 29, 30, 44, 52, 64, 65, 69, 71, 72, 74, 76, 89, 91, 97, 109, 114], "getbatchs": [3, 4, 44], "noexceptoverrid": [3, 4], "batch": [3, 4, 44, 64, 65, 72, 75, 91, 96, 101, 103, 107, 108, 111, 112, 117], "size": [3, 4, 44, 48, 49, 52, 55, 56, 64, 65, 70, 71, 72, 76, 77, 80, 89, 91, 94, 96, 101, 103, 107, 110, 112], "next": [3, 4, 53, 54, 58, 63, 72, 76, 80, 82, 83, 91, 93, 100, 104, 107, 111], "alwai": [3, 4, 27, 52, 76, 82, 99, 108], "1": [3, 4, 33, 44, 45, 48, 49, 52, 54, 55, 56, 58, 60, 62, 63, 64, 65, 66, 70, 71, 72, 74, 75, 76, 77, 79, 80, 82, 83, 86, 88, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, 101, 103, 104, 105, 106, 107, 108, 110, 112, 113, 115, 117], "due": [3, 4, 66, 81, 82, 107], "issu": [3, 4, 64, 71, 76, 89, 100, 103], "getbatch": [3, 4, 44], "void": [3, 4, 25, 26, 27, 28, 36, 37, 42, 44, 45], "bind": [3, 4, 33, 44, 75, 77, 82], "char": [3, 4, 44, 52, 89], "name": [3, 4, 31, 33, 34, 44, 54, 56, 58, 60, 65, 66, 67, 72, 74, 75, 76, 77, 82, 83, 88, 89, 92, 93, 94, 99, 104, 107, 111, 113], "nbbind": [3, 4, 44], "Not": 3, "arrai": [3, 4, 33, 53, 54, 76, 77, 93, 94, 108], "pointer": [3, 4, 91], "fed": [3, 4, 48], "buffer": [3, 4, 65, 94], "each": [3, 4, 49, 53, 55, 56, 58, 60, 64, 65, 66, 71, 72, 75, 80, 82, 89, 93, 99, 106, 114], "input": [3, 4, 21, 29, 33, 38, 44, 45, 47, 49, 50, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 68, 70, 71, 72, 73, 75, 76, 77, 83, 88, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, 100, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117], "number": [3, 4, 49, 52, 54, 55, 56, 60, 63, 64, 65, 71, 72, 76, 77, 80, 89, 90, 94, 99, 101, 103, 108, 110, 116], "readcalibrationcach": [3, 4, 44], "size_t": [3, 4, 44, 91], "length": [3, 4, 44, 65, 70, 83, 108], "how": [3, 4, 66, 67, 82, 84, 86, 88, 92, 93, 94, 96, 98, 100, 107, 108, 110, 111, 112, 114], "enabl": [3, 4, 24, 49, 52, 54, 56, 57, 59, 64, 65, 66, 71, 72, 74, 75, 76, 77, 80, 96, 98, 99, 101, 103, 104, 105, 106, 108, 113, 114], "use_cach": [3, 4, 30, 44, 74, 91, 105, 106, 108], "set": [3, 4, 16, 21, 25, 27, 29, 32, 34, 36, 45, 46, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 65, 66, 71, 72, 75, 76, 77, 80, 84, 87, 88, 89, 90, 91, 93, 94, 99, 105, 107, 108, 110, 112, 113, 114, 116, 117], "writecalibrationcach": [3, 4, 44], "write": [3, 4, 29, 30, 44, 65, 69, 82, 89, 91, 111], "provid": [3, 4, 49, 52, 54, 56, 58, 60, 62, 64, 65, 66, 68, 71, 72, 75, 76, 77, 82, 89, 90, 91, 92, 93, 96, 97, 99, 100, 104, 108, 109, 111, 112, 114, 115, 116], "cast": [3, 4, 55, 64, 71, 105, 106, 113], "convienc": [3, 4, 49], "convert": [3, 4, 31, 32, 34, 52, 55, 56, 57, 59, 63, 64, 69, 71, 76, 77, 90, 92, 94, 97, 101, 103, 108, 109, 110, 114], "easili": [3, 4, 98], "assign": [3, 4, 81], "ptq_calibr": [3, 4, 45, 49, 91], "field": [3, 4, 63, 72, 76, 91], "compilespec": [3, 4, 21, 32, 34, 41, 45, 50, 56, 77, 89, 91, 117], "dataloaderuniqueptr": [4, 44], "libtorch": [4, 37, 60, 66, 68, 89, 91, 116], "dataload": [4, 29, 30, 44, 49, 74, 91, 107], "unique_ptr": [4, 30], "unqiue_ptr": 4, "A": [4, 29, 30, 32, 33, 47, 48, 54, 55, 56, 60, 65, 66, 71, 72, 76, 77, 83, 91, 102, 111], "uniqu": [4, 90], "what": [4, 54, 55, 65, 68, 76, 82, 88, 89, 90, 105, 106, 116], "make_data_load": [4, 91], "factori": [4, 29, 30, 64, 71, 91], "path": [4, 13, 14, 15, 29, 30, 52, 64, 65, 66, 67, 71, 74, 76, 88, 89, 91, 96, 104, 107, 111, 116], "find": [4, 65, 66, 67, 89, 94, 108], "whether": [4, 52, 54, 64, 65, 71, 72, 76, 81, 91, 101, 103, 114], "exist": [4, 31, 32, 34, 54, 63, 64, 65, 67, 71, 74, 76, 77, 91, 96, 110], "There": [4, 53, 54, 59, 60, 62, 63, 65, 66, 83, 88, 91, 99, 110, 111, 112, 114], "consum": [4, 53, 88], "macro": [5, 6, 7, 8, 9, 10, 11, 12, 15, 18, 20, 21, 42, 44, 45, 50, 51], "x": [5, 10, 33, 43, 55, 56, 66, 67, 68, 75, 77, 83, 88, 89, 93, 94, 96, 100, 104, 107, 108, 112, 113, 115], "includ": [13, 15, 16, 35, 37, 42, 43, 44, 45, 51, 52, 54, 56, 57, 58, 59, 62, 64, 65, 66, 67, 68, 71, 72, 75, 76, 80, 82, 88, 89, 91, 94, 114], "parent": [14, 15, 18, 19, 20, 21], "cpp": [14, 15, 42, 43, 44, 45, 51, 55, 59, 66, 89, 91], "log": [15, 16, 19, 20, 38, 44, 50, 51, 55, 60, 64, 65, 69, 70, 71, 72, 76, 93, 101, 103, 113], "emum": [16, 17], "messag": [16, 25, 26, 52, 73], "sever": [16, 26, 73], "kinternal_error": [16, 42], "print": [16, 31, 44, 62, 64, 67, 71, 77, 82, 89, 92, 93, 94, 95, 96, 98, 99, 101, 103, 105, 106, 107, 108, 111], "error": [16, 49, 52, 53, 55, 59, 64, 65, 71, 73, 76, 77, 82, 89, 112], "kerror": [16, 42], "all": [16, 42, 43, 44, 45, 49, 52, 54, 55, 56, 58, 62, 64, 65, 66, 67, 71, 73, 75, 76, 78, 82, 83, 88, 89, 90, 91, 93, 94, 105, 106, 109, 110, 111, 113, 114, 116], "kwarn": [16, 42], "warn": [16, 44, 52, 60, 73, 75], "kinfo": [16, 42, 44], "info": [16, 32, 34, 45, 52, 60, 73, 75, 76, 113], "kdebug": [16, 42, 44], "debug": [16, 27, 45, 49, 52, 60, 62, 64, 71, 73, 75, 76, 77, 92, 94, 95, 96, 98, 99, 100, 101, 103, 107, 113], "kgraph": [16, 42, 55], "everyth": [16, 64, 71, 76], "intermedi": [16, 49, 52, 54, 64, 71, 73, 76, 77, 88, 113, 116], "graph": [16, 31, 32, 34, 45, 49, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 65, 71, 72, 73, 76, 77, 88, 89, 93, 94, 96, 98, 99, 101, 103, 104, 110, 112, 114], "lower": [16, 54, 63, 65, 69, 71, 72, 73, 76, 83, 94, 96, 101, 103, 108, 110, 116], "phase": [16, 60, 63, 89, 93, 99, 112, 116], "select": [17, 29, 30, 34, 49, 52, 58, 64, 65, 66, 70, 71, 76, 77, 81, 84, 90, 91, 94, 116], "capabl": [17, 45, 49, 52, 58, 71, 76, 77, 92, 93], "kstandard": [17, 45, 49], "ksafeti": [17, 45], "kdla_standalon": [17, 45], "directori": [18, 19, 20, 21, 42, 43, 44, 45, 50, 66, 67, 71, 91, 96], "program": [18, 19, 20, 21, 29, 51, 52, 57, 58, 59, 69, 71, 88, 96, 97, 105, 106, 109, 112], "list": [18, 19, 20, 21, 31, 49, 51, 53, 56, 58, 60, 62, 63, 65, 68, 70, 71, 72, 75, 76, 77, 86, 89, 90, 93, 94, 111], "torchscript": [19, 21, 38, 43, 45, 49, 50, 52, 56, 57, 58, 59, 63, 68, 71, 72, 74, 75, 76, 77, 90, 110, 112, 117], "str": [19, 43, 44, 50, 54, 64, 65, 70, 71, 74, 75, 76, 77, 93, 94, 96, 107], "torch_tensorrt_major_vers": [19, 43, 50], "torch_tensorrt_minor_vers": [19, 43, 50], "torch_tensorrt_patch_vers": [19, 43, 50], "torch_tensorrt_vers": [19, 43, 50], "torchtrt_hidden": [19, 43, 50], "xstr": [19, 43, 50], "nvinfer": [20, 44], "fstream": [20, 44], "iostream": [20, 21, 44, 45, 89], "iter": [20, 44, 49, 52, 53, 64, 71, 74, 76, 77, 95, 96, 107, 108], "memori": [20, 21, 44, 45, 55, 60, 71, 76, 77, 89, 90, 94, 96, 105, 106, 108], "sstream": [20, 44], "vector": [20, 21, 33, 44, 45, 47, 48, 49, 56, 58, 76, 89, 91, 117], "templat": [20, 40, 44, 45, 50, 80, 89], "int8cachecalibr": [20, 29, 40, 44, 50], "cuda_runtim": [21, 45], "custom_class": [21, 45], "devic": [21, 33, 36, 38, 45, 49, 50, 52, 58, 64, 70, 71, 72, 74, 75, 76, 77, 90, 91, 92, 94, 98, 102, 105, 106, 108, 110, 117], "graphinput": [21, 38, 45, 49, 50], "devicetyp": [21, 38, 45, 46, 50, 75, 76, 77, 91, 92, 94, 117], "tensorformat": [21, 38, 45, 48, 50, 76, 94], "level": [23, 25, 26, 39, 42, 44, 50, 54, 55, 56, 59, 64, 65, 71, 76, 77, 86, 88, 93, 94, 111, 116], "current": [23, 54, 56, 58, 60, 62, 63, 64, 65, 66, 67, 71, 72, 75, 76, 77, 80, 93, 94, 98, 105, 106, 107, 108, 114], "report": [23, 44, 75], "Is": [24, 76], "color": [24, 27, 82], "output": [24, 27, 33, 49, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 66, 71, 73, 75, 76, 77, 80, 82, 83, 89, 93, 94, 96, 98, 99, 102, 108, 110, 111, 112, 113, 115], "lvl": [25, 26, 42], "inform": [25, 33, 35, 37, 48, 52, 53, 56, 58, 62, 64, 65, 66, 71, 72, 73, 76, 82, 88, 89, 91, 92, 94, 96, 108, 112], "ad": [25, 52, 53, 54, 56, 62, 65, 66, 94, 98], "abov": [25, 54, 56, 62, 65, 66, 73, 81, 82, 89, 94, 101, 103, 113, 115], "msg": [26, 42], "add": [26, 53, 54, 55, 56, 60, 63, 66, 70, 80, 82, 87, 89, 90, 93, 94], "global": [26, 52, 64, 71, 76, 89], "colored_output_on": [27, 42], "prefix": [27, 28, 42, 82], "help": [27, 52, 53, 60, 64, 65, 89, 96, 107, 108, 110, 114], "when": [27, 44, 45, 46, 52, 53, 55, 56, 57, 58, 59, 60, 64, 65, 66, 71, 75, 76, 77, 80, 82, 84, 88, 89, 91, 93, 94, 96, 98, 99, 108, 110, 112, 114], "termin": [27, 52, 89], "If": [27, 33, 53, 54, 55, 56, 62, 63, 64, 65, 66, 68, 71, 72, 76, 80, 82, 89, 90, 91, 93, 94, 96, 99, 100, 104, 108, 111, 112, 113, 114, 116, 117], "build": [29, 30, 35, 49, 52, 53, 57, 59, 60, 63, 64, 65, 71, 75, 76, 81, 86, 89, 91, 93, 94, 101, 103, 108, 112], "post": [29, 30, 49, 52, 63, 69, 89, 96], "train": [29, 30, 49, 52, 69, 70, 89, 90, 96, 108], "quantiz": [29, 30, 52, 64, 69, 74, 76, 89, 97, 109], "creat": [29, 30, 33, 52, 53, 54, 56, 58, 60, 65, 69, 76, 77, 82, 89, 93, 94, 99, 108, 111], "previous": [29, 33, 89, 96, 99], "therefor": [29, 58, 65, 66, 75, 82, 89, 110, 114], "have": [29, 33, 44, 52, 53, 54, 55, 56, 60, 62, 63, 64, 65, 66, 67, 71, 72, 74, 75, 76, 77, 82, 88, 89, 90, 91, 94, 97, 101, 103, 107, 109, 110, 111, 112], "requir": [29, 49, 52, 53, 54, 55, 63, 64, 65, 66, 67, 71, 76, 77, 80, 89, 91, 93, 94, 97, 107, 108, 109, 111, 112, 114], "dataset": [29, 74, 91, 110], "save": [29, 44, 52, 58, 64, 65, 68, 69, 71, 75, 76, 77, 89, 90, 95, 96, 99, 102, 108, 110, 111, 114, 116], "later": [29, 71, 89, 99, 115, 116], "differ": [29, 55, 56, 59, 64, 65, 66, 76, 80, 88, 93, 94, 96, 98, 105, 108, 110, 114, 116], "scratch": [29, 96, 99], "depend": [29, 35, 53, 59, 64, 65, 67, 68, 71, 89, 90, 108, 111, 114], "howev": [29, 66, 80, 81, 89, 93, 94, 96, 111, 112, 116], "network": [29, 30, 54, 60, 65, 76, 89, 91, 93, 94, 108, 110, 111, 117], "also": [29, 53, 54, 60, 62, 64, 66, 68, 80, 82, 83, 89, 90, 91, 96, 104, 107, 110], "recalibr": 29, "its": [29, 53, 56, 58, 60, 66, 75, 76, 82, 94, 107, 111, 114, 116], "structur": [29, 46, 49, 56, 59, 60, 64, 71, 76, 80, 82, 86, 88, 94, 111], "chang": [29, 55, 56, 59, 62, 64, 65, 75, 76, 77, 80, 91, 93, 96, 98, 99, 111, 114, 116], "respons": [29, 54, 58, 82, 114], "ensur": [29, 54, 55, 56, 62, 64, 66, 67, 71, 75, 105, 106], "By": [29, 30, 51, 56, 66, 80, 88, 96, 112], "entropi": [29, 30, 91], "v2": [29, 30, 82], "perform": [29, 30, 54, 62, 63, 71, 75, 76, 91, 94, 104, 108, 110, 111, 113, 114, 115, 116], "recommend": [29, 30, 65, 66, 76, 82, 89, 94, 111, 112], "feed": [29, 30, 89], "forward": [29, 30, 32, 33, 56, 58, 60, 64, 68, 71, 75, 76, 77, 88, 89, 90, 91, 92, 93, 94, 100, 107, 112, 113], "overrid": [29, 30, 44, 54, 65, 91], "minmax": [29, 30, 91], "recomend": [29, 30], "nlp": [29, 30, 91], "task": [29, 30, 65, 91, 110], "call": [29, 30, 32, 49, 54, 55, 58, 60, 65, 71, 72, 75, 76, 77, 82, 88, 89, 92, 93, 94, 96, 98, 100, 103, 110, 112, 114, 116], "make_int8_calibr": [29, 40, 44, 50, 91], "class": [29, 30, 44, 45, 46, 51, 58, 60, 64, 65, 73, 77, 82, 83, 88, 89, 90, 91, 93, 94, 96, 100, 107, 110, 112, 113], "e": [29, 30, 52, 55, 60, 65, 66, 67, 68, 72, 76, 88, 89, 91, 94, 96, 99], "g": [29, 30, 52, 55, 65, 66, 67, 72, 76, 82, 91, 94, 99], "iint8minmaxcalibr": [29, 30, 91], "calibration_cache_fil": [29, 30, 91], "move": [30, 44, 55, 58, 77, 89, 91, 93, 105, 106], "calibration_dataload": [30, 91], "contain": [30, 31, 52, 53, 54, 55, 56, 60, 65, 66, 72, 75, 76, 82, 83, 88, 89, 91, 94, 96, 111, 114], "jit": [31, 32, 33, 34, 45, 47, 49, 52, 53, 55, 56, 57, 58, 59, 60, 61, 64, 68, 69, 71, 75, 76, 77, 88, 89, 90, 92, 94, 99, 111, 115, 116], "modul": [31, 32, 33, 34, 45, 49, 52, 56, 57, 58, 59, 60, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 76, 77, 81, 82, 83, 90, 91, 92, 93, 94, 97, 99, 100, 107, 109, 110, 112, 113, 115, 117], "method_nam": [31, 34, 45, 52, 76, 77, 89], "see": [31, 55, 56, 58, 62, 64, 65, 66, 76, 77, 82, 88, 89, 90, 93, 94, 96, 99, 100], "fulli": [31, 52, 55, 64, 71, 75, 76, 77, 89, 91, 94, 117], "compil": [31, 34, 41, 45, 49, 50, 52, 54, 55, 56, 58, 60, 62, 65, 71, 72, 73, 75, 76, 77, 78, 80, 88, 90, 91, 92, 93, 94, 95, 97, 98, 107, 109, 111, 114, 117], "take": [31, 32, 33, 34, 53, 54, 57, 58, 59, 60, 62, 65, 71, 72, 75, 76, 77, 80, 82, 89, 91, 92, 93, 94, 100, 110, 112], "method": [31, 32, 33, 34, 48, 52, 55, 60, 66, 71, 76, 77, 82, 88, 89, 92, 96, 110], "pure": [31, 71, 76], "Will": 31, "out": [31, 44, 53, 55, 56, 57, 59, 60, 64, 66, 71, 76, 77, 82, 89, 94, 98, 107, 108, 111, 112], "unsupport": [31, 49, 54, 64, 76, 94, 116], "script": [31, 55, 56, 68, 76, 77, 88, 89, 90, 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 114, 116], "nvidia": [32, 34, 42, 43, 44, 45, 52, 61, 64, 65, 66, 67, 71, 76, 77, 89, 100, 103, 111, 116, 117], "configur": [32, 34, 48, 62, 64, 66, 71, 75, 76, 77, 86, 89, 91, 94, 108, 111, 112], "equival": [32, 57, 59, 60, 71, 76, 77, 88, 89, 91, 93, 94, 101, 103], "specif": [32, 49, 54, 55, 57, 59, 62, 64, 71, 76, 77, 82, 93, 108, 116], "traget": 32, "input_binding_nam": [33, 45, 75, 77], "output_binding_nam": [33, 45, 75, 77], "emb": [33, 52, 63, 77, 83], "pre": [33, 55, 74, 77, 91, 96, 108, 114], "built": [33, 52, 58, 59, 64, 66, 71, 75, 76, 77, 96, 99], "serial": [33, 34, 52, 57, 59, 66, 71, 75, 76, 77, 89, 94, 96, 116], "regist": [33, 54, 58, 60, 65, 75, 77, 93, 94], "execut": [33, 49, 52, 55, 57, 58, 59, 63, 64, 65, 66, 69, 71, 72, 75, 76, 77, 78, 88, 89, 91, 93, 94, 111], "must": [33, 48, 49, 52, 54, 55, 56, 60, 62, 65, 66, 71, 72, 76, 77, 82, 83, 89, 96, 112, 114, 116], "follow": [33, 52, 54, 56, 58, 62, 63, 64, 65, 66, 77, 80, 82, 83, 87, 88, 89, 91, 93, 94, 96, 97, 101, 105, 106, 109, 110, 111, 112, 113, 114], "format": [33, 45, 48, 49, 52, 70, 71, 76, 77, 82, 83, 90, 94, 96, 107, 110, 111, 113, 115], "symbol": [33, 65, 66, 77, 82, 114], "index": [33, 61, 62, 66, 67, 69, 70, 77, 80, 86, 91, 94], "0": [33, 43, 44, 45, 49, 52, 54, 56, 59, 60, 62, 64, 65, 66, 67, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 82, 89, 91, 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 111, 112, 113, 117], "2": [33, 43, 54, 56, 60, 63, 64, 65, 66, 67, 69, 70, 71, 74, 75, 76, 77, 80, 82, 83, 86, 88, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 103, 105, 106, 107, 108, 112, 115], "y": [33, 56, 77, 83, 93, 94, 100], "compilesepc": 33, "order": [33, 49, 54, 56, 60, 62, 65, 66, 71, 72, 75, 76, 77, 89, 90, 93, 96, 113], "pass": [33, 53, 54, 56, 57, 58, 59, 60, 63, 64, 65, 66, 69, 73, 74, 75, 76, 77, 88, 89, 91, 93, 94, 96, 99], "origin": [33, 65, 72, 76, 94, 96, 98, 116], "pytorch": [33, 48, 49, 52, 54, 55, 56, 57, 58, 59, 60, 63, 64, 66, 67, 68, 71, 74, 75, 76, 77, 88, 89, 90, 91, 93, 96, 98, 99, 107, 111, 112, 113, 114, 115, 116], "assum": [33, 75, 92, 94, 97, 109], "convent": 33, "below": [33, 56, 60, 62, 63, 64, 65, 66, 67, 82, 89, 90, 96, 102, 111], "equivil": 34, "librari": [35, 42, 43, 44, 45, 52, 54, 57, 58, 59, 60, 76, 89, 94, 97, 109], "version": [35, 37, 59, 62, 64, 65, 67, 71, 75, 76, 80, 83, 94, 110, 111, 115], "gpu_id": [36, 45, 46, 52, 75, 76, 77, 91, 92, 94, 117], "id": [36, 45, 52, 76, 80, 81, 85, 117], "cudasetdevic": 36, "dump": [37, 52, 94], "base": [37, 50, 58, 63, 64, 66, 71, 72, 76, 82, 88, 90, 91, 95, 99, 103, 110, 116], "stdout": [37, 75], "enginecap": [38, 45, 49, 50, 64, 71, 75, 76, 77, 92, 94], "dump_build_info": [38, 45, 50], "get_build_info": [38, 45, 50], "set_devic": [38, 45, 50, 114], "get_is_colored_output_on": [39, 42, 50], "get_logging_prefix": [39, 42, 50], "get_reportable_log_level": [39, 42, 50], "set_is_colored_output_on": [39, 42, 50], "set_logging_prefix": [39, 42, 50], "set_reportable_log_level": [39, 42, 50], "make_int8_cache_calibr": [40, 44, 50, 91], "check_method_operator_support": [41, 45, 50], "convert_method_to_trt_engin": [41, 45, 50, 76, 77, 89, 92], "embed_engine_in_new_modul": [41, 45, 50, 77], "document": [42, 43, 44, 45, 50, 59, 80, 82, 83, 87, 88, 89, 91, 92, 111, 112, 114], "copyright": [42, 43, 44, 45, 83, 89], "c": [42, 43, 44, 45, 52, 59, 64, 67, 70, 71, 72, 75, 76, 83, 90, 94, 98, 111, 114, 117], "corpor": [42, 43, 44, 45], "right": [42, 43, 44, 45, 55, 59, 60, 82], "reserv": [42, 43, 44, 45, 105, 106], "licens": [42, 43, 44, 45, 89], "under": [42, 43, 44, 45, 59, 65, 82, 93, 101, 116], "bsd": [42, 43, 44, 45], "style": [42, 43, 44, 45, 64, 68, 80, 82, 83], "found": [42, 43, 44, 45, 63, 66, 75, 82, 89, 91, 93, 94, 96, 114], "root": [42, 43, 44, 45, 66, 80, 91, 107], "sourc": [42, 43, 44, 45, 54, 59, 64, 65, 67, 71, 72, 73, 74, 75, 76, 77, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "tree": [42, 43, 44, 45, 80, 91, 107, 114], "pragma": [42, 43, 44, 45, 91], "onc": [42, 43, 44, 45, 53, 55, 56, 58, 64, 65, 66, 67, 76, 91, 94, 106, 108, 111, 114], "namespac": [42, 43, 44, 45, 51, 55, 69, 76, 91, 94], "ar": [42, 46, 49, 52, 53, 54, 55, 56, 58, 59, 60, 62, 63, 64, 65, 66, 71, 74, 75, 76, 77, 80, 82, 83, 84, 88, 89, 91, 92, 93, 94, 95, 96, 98, 99, 101, 105, 106, 108, 110, 111, 112, 113, 114, 115, 116], "ones": [42, 56, 57, 59, 66, 82, 89, 93, 94, 116], "necessari": [42, 62, 64, 66, 75, 93, 99, 114], "user": [42, 48, 54, 56, 57, 58, 59, 62, 63, 64, 66, 67, 71, 82, 83, 89, 90, 91, 93, 96, 99, 108, 111, 112, 113, 114, 116], "dont": 42, "know": [42, 60, 80, 82, 93, 94], "we": [42, 44, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 72, 75, 80, 82, 88, 89, 91, 93, 94, 96, 97, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116], "want": [42, 56, 65, 66, 67, 68, 72, 88, 89, 91, 92, 93, 94, 99, 100, 111], "use_cmake_generated_export_head": 43, "torch_tensorrt_export": 43, "els": [43, 44, 48, 77, 82, 83, 95, 96, 107], "__gnuc__": 43, "__attribute__": 43, "__visibility__": 43, "hidden": [43, 80], "endif": [43, 44, 45], "doe": [43, 44, 55, 56, 60, 62, 65, 66, 76, 82, 91, 94, 101, 103], "gaurd": 43, "someth": [43, 55, 82, 111], "6": [43, 55, 56, 58, 66, 70, 82, 86, 88, 89, 94], "setup": [43, 67, 91, 111], "alias": 43, "eas": 43, "ts": [43, 52, 56, 68, 69, 76, 88, 89, 90, 92, 112, 115], "torchtrt": [43, 56, 94, 107], "ifndef": [44, 45], "doxygen_should_skip_thi": [44, 45], "get_batch_impl": 44, "element_typ": 44, "super": [44, 88, 93, 94, 100, 107, 112, 113], "batchtyp": 44, "dataloader_": 44, "cache_file_path_": 44, "use_cache_": 44, "auto": [44, 56, 60, 64, 68, 71, 82, 83, 89, 91, 105, 106, 108, 117], "batched_data_": 44, "push_back": [44, 56], "it_": 44, "begin": [44, 65, 66, 82, 100, 104], "noexcept": [44, 91], "hack": 44, "explict": 44, "work": [44, 55, 59, 60, 64, 65, 68, 71, 74, 75, 76, 82, 83, 91, 94, 99, 100, 104, 108, 112], "here": [44, 53, 54, 56, 58, 63, 64, 65, 66, 68, 80, 82, 83, 88, 89, 91, 93, 94, 97, 104, 105, 106, 107, 109, 111, 112, 114, 115], "explic": 44, "just": [44, 45, 55, 56, 64, 65, 69, 73, 75, 82, 84, 88, 89, 90, 92, 94, 96, 98, 110, 114], "still": [44, 56, 65, 66, 91, 93, 100, 116], "static_cast": 44, "option": [44, 48, 52, 56, 57, 59, 62, 63, 64, 65, 71, 75, 76, 77, 82, 86, 91, 93, 94, 95, 96, 100, 102, 113, 114, 115, 117], "batch_siz": [44, 91, 107], "end": [44, 52, 60, 62, 70, 71, 76, 77, 82, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "statu": [44, 83], "reset": [44, 95, 96, 100, 103, 114], "incas": 44, "go": [44, 55, 56, 65, 68, 88, 89, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 116], "again": [44, 58, 60, 82, 94, 98], "stringstream": 44, "ss": 44, "cache_": 44, "clear": 44, "ifstream": 44, "io": [44, 67, 111], "binari": [44, 91], "noskipw": 44, "good": [44, 60, 65, 82, 96], "copi": [44, 60, 65, 67, 70, 74, 83, 108, 111], "istream_iter": 44, "back_insert": 44, "nullptr": [44, 45, 49], "ofstream": [44, 89], "cache_fil": [44, 74, 91], "reinterpret_cast": 44, "cache_size_": 44, "int8_t": 45, "arrayref": [45, 48, 49], "friend": 45, "ostream": 45, "os": [45, 67, 96], "dtype": [45, 48, 49, 52, 63, 64, 65, 70, 71, 72, 75, 76, 77, 90, 94, 95, 101, 103, 104, 108, 110, 112, 113], "device_typ": [45, 46, 76, 91, 92, 117], "int64_t": [45, 46, 48, 49, 91, 117], "core": [45, 52, 55, 56, 59, 64, 71, 76, 89, 93, 116, 117], "agx": 45, "platform": [45, 52, 59, 66, 67, 111, 117], "xavier": [45, 117], "dla_cor": [45, 46, 52, 76, 91, 92, 117], "allow_gpu_fallback": [45, 46, 71, 76, 77, 91, 92, 117], "customclasshold": [45, 48], "min_shap": [45, 48, 63, 65, 71, 76, 77, 90, 101, 104, 110, 112], "opt_shap": [45, 48, 63, 71, 76, 77, 90, 101, 104, 110, 112], "max_shap": [45, 48, 63, 65, 71, 76, 77, 90, 101, 104, 110, 112], "shape": [45, 47, 48, 49, 52, 56, 60, 63, 65, 69, 70, 71, 72, 75, 76, 77, 78, 90, 93, 94, 97, 104, 107, 108, 109, 111, 114, 117], "doubl": [45, 48, 49, 52, 63, 71, 76, 77, 82, 114], "tensor_domain": [45, 48, 76], "input_is_dynam": 45, "ivalu": [45, 47, 49, 53, 58, 60, 89], "input_signatur": [45, 47, 49, 77, 90], "nest": [45, 49, 50, 82, 83], "full": [45, 49, 52, 60, 64, 71, 73, 76, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 111, 114, 117], "spec": [45, 48, 49, 52, 73, 76, 77, 92, 96], "flatten": [45, 47, 70, 88, 89, 107], "fixed_s": [45, 49], "reflect": [45, 76], "builderconfig": 45, "graph_input": [45, 49], "enabled_precis": [45, 49, 63, 64, 71, 75, 76, 77, 89, 90, 91, 92, 94, 95, 96, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 111, 113, 117], "disable_tf32": [45, 49, 64, 71, 75, 76, 77, 91, 94, 105, 106], "sparse_weight": [45, 49, 64, 65, 71, 75, 76, 77, 94], "refit": [45, 49, 64, 69, 71, 76, 77, 92, 94, 96, 97, 98, 109], "truncate_long_and_doubl": [45, 49, 63, 64, 77, 102], "allow_shape_tensor": [45, 49, 77], "uint64_t": [45, 49], "num_avg_timing_it": [45, 49, 64, 71, 75, 76, 77, 92, 94], "workspace_s": [45, 49, 52, 64, 71, 75, 76, 77, 94, 99, 101, 103], "dla_sram_s": [45, 49, 52, 64, 71, 75, 76, 77, 94], "1048576": [45, 49, 64, 71, 75, 76, 77, 94], "dla_local_dram_s": [45, 49, 52, 64, 71, 75, 76, 77, 94], "1073741824": [45, 49, 64, 71, 75, 76, 77, 94], "dla_global_dram_s": [45, 49, 52, 64, 71, 75, 76, 77, 94], "536870912": [45, 49, 64, 71, 75, 76, 77, 94], "require_full_compil": [45, 49, 64, 71, 75, 76, 77, 94], "min_block_s": [45, 49, 56, 63, 64, 71, 75, 76, 77, 93, 94, 95, 96, 99, 100, 101, 103, 107], "3": [45, 49, 52, 55, 56, 58, 63, 64, 65, 67, 68, 70, 71, 74, 76, 77, 82, 83, 86, 88, 89, 91, 92, 94, 95, 96, 98, 99, 101, 104, 105, 106, 107, 108, 110, 112, 115, 117], "torch_executed_op": [45, 49, 56, 63, 64, 71, 75, 76, 77, 94, 99, 100, 101, 103], "torch_executed_modul": [45, 49, 56, 71, 76, 77], "member": [46, 47, 48, 49], "hold": [46, 47, 48, 53, 60, 76, 91], "relat": [46, 82, 100, 103], "let": [46, 52, 55, 60, 65, 71, 76, 77, 80, 82, 110, 111, 116], "layer": [46, 49, 52, 53, 55, 60, 62, 64, 65, 71, 75, 76, 77, 89, 91, 93, 94, 105, 106, 107, 110, 111, 112, 113, 116, 117], "thei": [46, 52, 53, 54, 55, 58, 60, 65, 74, 75, 76, 80, 82, 90, 93, 96], "complex": [47, 49, 64, 66, 88, 90, 98, 106], "either": [47, 48, 52, 60, 62, 71, 76, 77, 80, 82, 88, 89, 90, 93, 94, 96, 115], "one": [47, 54, 55, 60, 64, 65, 67, 71, 75, 76, 82, 88, 89, 90, 93, 94, 100, 103, 105, 106, 111], "rang": [48, 49, 52, 65, 76, 94, 95, 96, 101, 108, 110, 112], "optim": [48, 52, 63, 64, 65, 69, 71, 72, 74, 76, 88, 89, 90, 99, 101, 102, 103, 108, 110, 112, 116], "profil": [48, 72, 75, 113], "singl": [48, 52, 55, 56, 65, 76, 82, 88, 89, 91, 108, 114], "repres": [48, 49, 54, 60, 65, 68, 82], "signifi": [48, 55], "static": [48, 49, 53, 60, 63, 64, 71, 76, 77, 80, 89, 107, 112], "three": [48, 57, 59, 65, 72, 76, 82, 83, 110, 111], "min": [48, 52, 60, 70, 76, 96, 101, 112], "optimin": 48, "max": [48, 52, 60, 70, 76, 80, 96, 101, 107, 112], "allow": [48, 49, 52, 53, 54, 55, 56, 62, 64, 65, 66, 71, 76, 77, 80, 93, 94, 96, 99, 101, 103, 108, 114], "argument": [48, 52, 54, 55, 58, 60, 62, 64, 65, 71, 75, 76, 77, 82, 83, 89, 90, 93, 94, 112], "expect": [48, 54, 55, 60, 76, 89, 90, 110], "tradit": [48, 71, 76, 77, 91], "convect": 48, "produc": [48, 53, 54, 58, 60, 63, 76, 82, 89, 110], "low": [48, 65, 93, 98], "high": [48, 55, 56, 80, 93, 94, 116], "weight": [48, 49, 52, 53, 64, 65, 69, 70, 71, 76, 77, 82, 89, 96, 97, 98, 102, 109, 110], "first": [48, 53, 54, 55, 65, 68, 82, 83, 89, 90, 91, 93, 94, 96, 98, 100, 111, 112, 115, 116], "calcul": [48, 53, 56, 89, 94, 108], "detect": [48, 58, 76], "float32": [48, 49, 52, 63, 64, 65, 71, 76, 77, 94, 98, 102, 105, 106, 108, 112, 113], "dynam": [48, 49, 63, 65, 69, 71, 72, 76, 77, 78, 93, 96, 97, 100, 102, 103, 106, 108, 109, 114], "opt": [48, 66, 75, 76, 104], "minimum": [48, 49, 52, 56, 63, 64, 71, 76, 77, 94, 108], "maximum": [48, 49, 52, 64, 65, 71, 72, 76, 77, 101, 103, 108, 111], "accept": [48, 52, 54, 58, 60, 66, 76, 89, 90, 100, 115], "exampl": [48, 56, 58, 59, 60, 65, 66, 71, 73, 75, 76, 77, 78, 80, 81, 83, 86, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 114, 115], "s": [48, 49, 53, 56, 58, 60, 63, 65, 66, 67, 69, 71, 72, 75, 76, 80, 82, 83, 88, 89, 91, 93, 94, 96, 108, 110, 111, 112, 114, 115], "cannot": [48, 55, 56, 65, 66, 71, 75, 76, 77, 81, 88, 94], "through": [48, 53, 54, 55, 56, 58, 64, 65, 71, 73, 74, 82, 89, 90, 94, 98, 99, 110, 116], "altern": [48, 56, 62, 63, 76, 90, 93, 104, 110, 115], "refer": [48, 54, 57, 59, 65, 81, 86, 89, 91, 94, 107, 111, 112, 115], "given": [48, 49, 52, 54, 55, 65, 71, 72, 74, 76, 77, 88, 89, 90, 92, 93, 112], "kernel": [48, 49, 52, 60, 64, 65, 69, 71, 76, 77, 93, 97, 109, 113, 114], "ani": [48, 52, 53, 54, 60, 62, 64, 65, 70, 71, 74, 75, 76, 77, 80, 82, 89, 90, 91, 93, 94, 101, 112], "event": [48, 64, 95, 96], "place": [48, 55, 62, 65, 82, 83, 84, 91, 94, 107], "variabl": [48, 65, 75, 76], "dimens": [48, 55, 65, 72, 76, 101, 110, 112, 113], "domain": [48, 76, 83, 91], "convien": 49, "fix": [49, 65, 82, 94, 114, 117], "describ": [49, 56, 60, 76, 88, 92, 111], "entri": [49, 60, 96], "okai": 49, "ha": [49, 53, 54, 55, 56, 57, 59, 60, 62, 64, 65, 66, 67, 71, 72, 76, 82, 83, 88, 89, 91, 93, 96, 99, 107, 110, 112, 116], "flaten": 49, "precis": [49, 52, 63, 64, 65, 69, 71, 76, 89, 90, 91, 101, 103, 105, 106, 108, 117], "dure": [49, 52, 54, 56, 60, 63, 64, 71, 74, 76, 91, 93, 105, 106, 108, 110, 112, 114], "prevent": [49, 52, 54, 56], "tf32": [49, 52, 64, 71], "comput": [49, 64, 65, 66, 67, 71, 75, 82, 91, 97, 109, 110], "inner": [49, 83, 110], "product": [49, 67, 76], "round": [49, 71, 76, 77, 94], "10": [49, 66, 67, 71, 72, 76, 77, 86, 88, 89, 91, 107, 108, 110, 111, 112, 113], "bit": [49, 60, 65, 66, 71, 76, 77, 89], "mantissa": [49, 71, 76, 77], "befor": [49, 54, 55, 56, 59, 60, 65, 71, 76, 77, 89, 111, 112], "multipli": [49, 71, 76, 77], "accumul": [49, 64, 71, 76, 77, 105, 106], "sum": [49, 65, 70, 71, 76, 77, 94, 107], "23": [49, 55, 71, 76, 77, 83], "behavior": [49, 56, 65, 71, 76, 77, 93, 105, 106, 112, 114, 115], "sparsiti": [49, 52, 65, 71, 76, 77], "conv": [49, 52, 89, 94], "fc": [49, 52, 55], "truncat": [49, 52, 63, 64, 71, 76, 77], "long": [49, 52, 53, 63, 76, 82, 83], "float": [49, 52, 63, 64, 70, 76, 88, 89, 90, 91, 92, 94, 95, 96, 99, 100, 103, 104, 113], "ishap": 49, "restrict": [49, 64, 71, 76, 77, 112], "cuda": [49, 58, 63, 65, 67, 68, 71, 72, 75, 76, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, 101, 102, 104, 105, 106, 107, 108, 111, 112, 113, 114, 115], "safeti": [49, 52, 76], "averag": [49, 52, 64, 71, 76, 77, 94], "time": [49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 64, 65, 66, 68, 69, 71, 72, 75, 76, 77, 80, 82, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "workspac": [49, 52, 64, 65, 66, 71, 72, 76, 77, 94, 100, 101, 103], "fast": [49, 52, 64, 68, 71, 76, 77], "softwar": [49, 52, 64, 71, 76, 77, 82], "manag": [49, 52, 53, 55, 57, 59, 60, 64, 66, 67, 71, 73, 75, 76, 77, 89, 104, 114], "ram": [49, 52, 64, 71, 76, 77], "commun": [49, 52, 64, 71, 76, 77, 89], "within": [49, 52, 57, 59, 64, 69, 71, 75, 76, 77, 80, 82, 97, 105, 106, 109, 110], "host": [49, 52, 64, 66, 71, 76, 77, 94, 108, 111], "share": [49, 52, 64, 66, 71, 75, 76, 77, 96], "across": [49, 52, 55, 56, 64, 71, 76, 77, 80], "metadata": [49, 52, 54, 58, 60, 64, 71, 76, 77, 80, 99, 112, 113], "quantizatiom": 49, "instead": [49, 52, 53, 54, 55, 66, 71, 75, 76, 89, 93, 99, 107, 114], "potenti": [49, 71, 76, 85], "subgraph": [49, 52, 53, 54, 55, 60, 62, 89, 94, 96, 116], "aten": [49, 54, 55, 56, 60, 61, 64, 69, 70, 71, 76, 77, 89, 93, 100, 116], "thrown": [49, 71, 76, 77], "empti": [49, 71, 72, 76, 77, 83, 88, 94], "torch_tensorrtnamespac": 50, "loggingenum": 50, "levelnamespac": 50, "ptqtemplat": 50, "int8cachecalibratortempl": 50, "int8calibratornamespac": 50, "torchscriptstruct": 50, "compilespecenum": 50, "enginecapabilitystruct": 50, "deviceclass": 50, "devicetypestruct": 50, "graphinputsstruct": 50, "inputclass": 50, "datatypeclass": 50, "cppdirectori": 50, "includedirectori": 50, "torch_tensorrtfil": 50, "hfile": 50, "relationship": 50, "inherit": [50, 65, 71, 91], "subdirectori": 51, "definit": [51, 54, 60, 82], "cli": [52, 90], "It": [52, 54, 55, 56, 57, 59, 60, 65, 66, 69, 76, 80, 82, 94, 108, 110, 114, 116], "serv": [52, 58, 65, 69, 71, 76], "easi": [52, 53, 55, 89, 91], "wai": [52, 64, 65, 66, 88, 89, 91, 93, 94, 96, 99, 110, 114, 115], "command": [52, 64, 66, 82, 83, 88, 89, 111], "line": [52, 66, 83, 89, 98], "quickli": [52, 89, 91], "part": [52, 56, 59, 65, 75, 80, 81, 82, 94, 96], "deploy": [52, 75, 89, 90, 91, 110, 111, 114, 117], "pipelin": [52, 89, 98, 102, 117], "basic": [52, 56, 65, 83, 111], "featur": [52, 56, 65, 66, 89, 91, 92, 102, 107, 108, 110, 116], "though": [52, 59, 60, 88, 89, 116], "alreadi": [52, 53, 54, 55, 89, 91, 93, 94, 97, 109, 112], "two": [52, 55, 60, 62, 64, 65, 66, 76, 82, 83, 87, 88, 90, 91, 93, 96, 111, 112], "embed": [52, 54, 58, 70, 77, 82, 117], "plan": [52, 59, 63, 64, 71], "after": [52, 53, 55, 56, 62, 65, 71, 75, 76, 88, 89, 90, 100, 103, 111, 114], "link": [52, 53, 62, 69, 80, 81, 86, 89, 94, 114], "against": [52, 89, 93], "libtorchtrt": [52, 66, 89], "python": [52, 56, 59, 62, 64, 65, 67, 71, 72, 75, 76, 77, 82, 83, 89, 92, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 114, 117], "import": [52, 55, 56, 63, 64, 65, 66, 67, 68, 75, 80, 82, 88, 89, 90, 92, 93, 94, 95, 96, 98, 111, 112, 114, 115], "packag": [52, 55, 64, 67, 89], "aspect": 52, "ident": [52, 62, 71, 76, 99], "standard": [52, 58, 66, 69, 71, 75, 76, 77, 82, 92, 93, 94, 98, 110, 114], "load": [52, 56, 58, 64, 65, 68, 71, 74, 75, 76, 77, 89, 90, 91, 92, 94, 95, 96, 98, 99, 108, 110, 111, 114, 116], "like": [52, 53, 55, 58, 60, 65, 66, 68, 76, 81, 82, 88, 89, 90, 91, 93, 94, 96, 98, 99, 108, 111, 114], "would": [52, 54, 60, 64, 65, 66, 67, 75, 89, 90, 92, 93, 94, 111, 114], "input_file_path": [52, 117], "output_file_path": [52, 117], "input_spec": [52, 65, 72], "displai": [52, 62, 64, 73, 80, 114], "menu": [52, 80, 82], "verbios": 52, "v": [52, 67, 83, 107, 111], "verbos": [52, 64, 65, 71, 72, 83, 101, 103], "about": [52, 53, 58, 60, 66, 75, 80, 89, 111, 112], "process": [52, 56, 64, 76, 81, 82, 88, 91, 92, 99, 100, 104, 110, 111, 114], "onto": [52, 58], "consol": 52, "w": [52, 66, 76], "disabl": [52, 64, 66, 71, 75, 80, 81, 93, 96, 108, 114], "i": [52, 55, 60, 66, 68, 70, 82, 83, 88, 89, 91, 94, 95, 96, 105, 107], "debugg": [52, 71, 76, 77], "fallback": [52, 57, 59, 60, 99, 117], "model": [52, 56, 58, 63, 68, 71, 72, 73, 74, 76, 78, 88, 89, 90, 91, 92, 95, 96, 98, 112, 114, 116], "throw": [52, 55, 76, 89], "spars": [52, 54, 64, 70, 71], "p": [52, 70, 89, 111, 117], "repeat": [52, 70], "f32": [52, 71, 75, 76, 94], "half": [52, 64, 76, 82, 89, 90, 91, 92, 94, 100, 101, 105, 106, 108, 111, 113, 117], "float16": [52, 76, 94, 98, 102, 113], "f16": [52, 76, 89, 117], "i8": [52, 76], "d": [52, 67, 76, 82, 83, 89, 117], "multi": [52, 75], "dlacor": 52, "avail": [52, 54, 60, 62, 64, 65, 66, 67, 71, 75, 76, 80, 94, 108, 110, 116, 117], "dla_standalon": [52, 76], "file_path": [52, 76, 115], "teo": 52, "op_nam": 52, "op": [52, 53, 54, 55, 56, 57, 59, 60, 62, 63, 64, 75, 76, 89, 93, 100, 114, 116], "partial": [52, 82], "tem": 52, "module_nam": 52, "mod": [52, 56, 65, 71, 86, 89, 91, 113], "mb": [52, 78], "num_op": 52, "block": [52, 53, 55, 56, 64, 71, 86, 116], "treat": 52, "num": 52, "avg": 52, "num_it": 52, "sram": 52, "local": [52, 55, 66, 67, 80, 89], "dram": 52, "atol": 52, "absolut": [52, 66], "toler": 52, "threshold": 52, "numer": [52, 65, 83], "deviat": 52, "1e": [52, 98, 99], "rtol": 52, "rel": [52, 56], "5": [52, 56, 58, 59, 64, 65, 66, 67, 71, 75, 76, 82, 83, 86, 88, 89, 93, 94, 98, 100, 108, 111], "skip": 52, "complianc": 52, "64bit": 52, "32bit": 52, "custom": [52, 62, 63, 65, 66, 69, 97, 105, 106, 109], "dll": 52, "n": [52, 60, 62, 76, 89, 91, 93, 94, 95], "min_n": 52, "min_c": 52, "min_h": 52, "min_w": 52, "opt_n": 52, "opt_c": 52, "opt_h": 52, "opt_w": 52, "max_n": 52, "max_c": 52, "max_h": 52, "max_w": 52, "32": [52, 76, 88, 89, 90, 91, 105, 106, 107, 117], "flag": [52, 56, 57, 59, 64, 66, 71, 74, 76, 90, 104, 105, 106, 114, 115], "forc": [52, 63, 65, 71, 76, 77, 80], "posit": [52, 54, 65, 76, 80], "test": [52, 56, 59, 65, 66, 67, 71, 76, 82, 83, 91, 107, 110, 111], "ssd_trace": 52, "pt": [52, 65, 89, 105, 106, 111], "ssd_trt": 52, "300": [52, 92], "512": [52, 71, 76, 77, 107, 110], "1024": [52, 71, 76, 77, 105, 110], "simplifi": [53, 94], "form": [53, 75, 76, 82, 90, 111], "up": [53, 55, 56, 57, 58, 59, 62, 65, 66, 71, 76, 82, 88, 93, 94, 96, 99, 100, 103, 108, 110], "context": [53, 57, 58, 59, 64, 73, 75, 93, 104, 114], "inetworkdefinit": [53, 54], "record": [53, 88, 95, 96, 104, 114], "togeth": [53, 60, 89], "start": [53, 56, 65, 70, 74, 76, 83, 89, 92, 94, 95, 96, 110], "look": [53, 54, 55, 68, 71, 76, 88, 91, 92, 93, 96, 111, 112], "assembl": [53, 62, 89], "resourc": [53, 91, 94], "coupl": [53, 59, 65, 114], "state": [53, 54, 60, 62, 75, 89, 93, 98], "been": [53, 60, 64, 66, 67, 74, 83, 89, 96, 99, 116], "evaluated_value_map": [53, 60], "stage": [53, 65], "arg": [53, 54, 62, 65, 71, 74, 75, 76, 86, 89, 93, 94, 96, 107, 110], "itensor": [53, 54, 60, 65, 89, 93, 94], "value_tensor_map": [53, 60], "typic": [53, 60, 76, 111], "abl": [53, 55, 60, 62, 65, 91, 92, 94, 99], "system": [53, 60, 62, 64, 69, 71, 75, 76, 77, 93, 94, 96, 99, 116], "registri": [53, 54, 89, 94], "enter": [53, 76], "recurs": 53, "resolv": [53, 55, 57, 59, 100, 103], "until": [53, 56, 59, 60, 66, 71, 76, 116], "final": [53, 56, 57, 59, 66, 93, 94, 100, 103, 110], "some": [53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 65, 66, 76, 81, 82, 89, 91, 93, 94, 96, 112, 116], "These": [53, 54, 56, 58, 62, 64, 66, 71, 74, 75, 76, 80, 82, 91, 93, 111, 116], "those": [53, 54, 62, 64, 82], "do": [53, 54, 55, 56, 60, 63, 65, 81, 83, 88, 89, 90, 91, 93, 94, 105, 106, 117], "theori": [53, 82], "kind": [53, 65], "common": [53, 55, 65, 72, 82, 93, 96], "prim": [53, 55, 56, 58, 70, 88, 89], "constant": [53, 54, 55, 56, 89, 94], "emit": 53, "listconstruct": [53, 56, 58, 89], "make": [53, 54, 65, 66, 67, 71, 76, 82, 84, 89, 90, 91, 94, 96, 110, 111, 117], "associ": [53, 60, 89, 96, 114], "where": [53, 54, 55, 60, 62, 64, 65, 71, 75, 76, 77, 83, 89, 91, 93, 99], "result": [53, 55, 56, 66, 68, 71, 73, 75, 76, 77, 80, 88, 90, 94, 98, 99, 108, 111, 113, 116], "done": [53, 56, 59, 94, 99, 111, 115], "mai": [53, 54, 56, 58, 59, 65, 66, 71, 75, 76, 77, 82, 83, 88, 89, 90, 91, 93, 94, 99, 100, 103, 108, 111, 114], "For": [53, 56, 62, 63, 64, 65, 66, 68, 72, 76, 80, 82, 83, 88, 89, 91, 92, 93, 94, 98, 100, 107, 110, 111, 114, 115], "more": [53, 64, 65, 66, 67, 69, 71, 76, 80, 83, 88, 89, 90, 91, 92, 94, 96, 98, 101, 103, 111, 114], "writing_convert": [53, 89], "locat": [54, 62, 66, 91, 93, 94], "py": [54, 55, 59, 62, 65, 66, 67, 78, 80, 82, 87, 88, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 112], "convers": [54, 55, 56, 58, 63, 64, 65, 71, 76, 77, 89, 93, 94, 110, 112], "decror": 54, "dynamo_tensorrt_convert": [54, 93, 94], "signatur": [54, 77], "leaky_relu": [54, 70], "def": [54, 62, 65, 82, 88, 90, 93, 94, 95, 96, 100, 107, 108, 111, 112, 113], "leaky_relu_convert": 54, "ctx": [54, 60, 89, 93, 94, 108], "conversionctx": [54, 60, 89, 93], "tupl": [54, 58, 63, 65, 71, 72, 75, 76, 77, 90, 93, 94, 96, 99, 112, 113], "kwarg": [54, 65, 71, 74, 75, 76, 93, 94, 110], "dict": [54, 71, 75, 76, 77, 93, 94, 96], "union": [54, 60, 64, 71, 75, 76, 77, 89, 93], "sequenc": [54, 62, 65, 71, 72, 75, 76, 77, 82, 93, 94, 108, 110], "decor": [54, 62, 65, 93], "kei": [54, 82, 88, 96, 111, 112], "node": [54, 55, 56, 57, 59, 60, 62, 64, 65, 71, 72, 89, 93, 94, 107, 110, 112], "capability_valid": [54, 93], "lambda": [54, 60, 82, 89, 93, 111], "fx": [54, 62, 63, 71, 75, 76, 89, 90, 93, 94, 99, 115], "determin": [54, 55, 64, 65, 76, 93, 108, 112, 114], "properli": [54, 66], "handl": [54, 55, 56, 58, 64, 65, 75, 76, 94], "partition": [54, 71, 76, 94], "sure": [54, 66, 67, 89, 90, 111, 117], "prioriti": [54, 93], "develop": [54, 65, 66, 67, 69, 82, 83, 89, 93, 94], "bodi": [54, 82, 83], "nativ": [54, 59, 61, 89, 93, 94, 99], "numpi": [54, 76, 94, 95, 96, 98, 99, 108, 111], "frozen": 54, "attribut": [54, 55, 56, 58, 65, 76, 82, 89], "previou": [54, 80, 100], "correspond": [54, 60, 65, 66, 75, 76, 93, 96, 98, 107, 114], "edg": [54, 82], "well": [54, 63, 66, 69, 73, 75, 82, 89, 91, 93, 96, 104, 115], "being": [54, 65, 66, 71, 89, 93, 94, 99], "truth": 54, "http": [54, 61, 64, 66, 67, 80, 82, 88, 89, 91, 93, 94, 98, 100, 103, 107, 110, 111, 112, 114], "github": [54, 61, 64, 66, 67, 80, 89, 91, 100, 103, 107, 111, 114], "com": [54, 61, 64, 66, 67, 89, 91, 98, 100, 103, 107, 111, 114], "blob": [54, 61, 66, 80, 91, 96], "main": [54, 55, 56, 57, 58, 59, 60, 63, 65, 66, 80, 82, 84, 89, 93, 94, 105, 107], "src": [54, 58, 61, 70], "native_funct": [54, 61], "yaml": [54, 61], "sinc": [54, 55, 64, 65, 67, 75, 82, 88, 89, 91, 93, 95, 96, 99], "mani": [54, 56, 64, 65, 80, 82, 83, 93, 96, 99, 116], "composit": [54, 89], "raw": [54, 80, 93], "impl": [54, 93], "subpackag": 54, "chain": [54, 60], "primarili": [54, 59, 66, 89, 93], "manipul": [54, 62, 76], "net": [54, 60, 82, 83, 89, 94], "addit": [54, 55, 64, 65, 75, 76, 89, 93, 94, 96, 99, 110, 112], "call_modul": 54, "call_funct": [54, 62, 65], "eg": [54, 111, 113], "aten_": 54, "_leaky_relu": 54, "opoverloadpacket": 54, "while": [54, 56, 66, 75, 91, 93, 98, 108, 110, 111, 114, 116], "opoverload": 54, "particular": [54, 64, 96], "collect": [54, 56, 64, 71, 76, 77, 89, 90, 107], "trtinterpret": [54, 65, 72], "along": [54, 76], "match": [54, 55, 93, 99], "special": [54, 56], "account": [54, 111], "illustr": [54, 65, 101, 105, 106, 110], "scale_grad_by_freq": [54, 70], "embedding_param_valid": 54, "establish": 54, "subset": [54, 64, 71, 76, 91, 110], "converter_util": [54, 94], "enforce_tensor_typ": 54, "dictionari": [54, 76, 77, 92, 100], "between": [54, 55, 56, 60, 66, 76, 82, 83, 91, 96, 98, 105, 108], "possibl": [54, 66, 82, 93, 94, 96, 110, 111], "prefer": [54, 64, 66, 89], "keyword": [54, 62, 71, 75, 76, 77, 93, 100, 103], "both": [54, 56, 64, 66, 69, 71, 72, 75, 76, 80, 82, 88, 91, 93, 94, 96], "enforc": [54, 89], "situat": 54, "partit": [54, 55, 63, 64, 71, 76, 93, 116], "greater": [54, 71, 73, 76], "than": [54, 55, 64, 66, 71, 76, 81, 82, 93, 95, 96, 98, 108, 110, 114], "3d": [54, 65], "autocast": 54, "therebi": [54, 58, 94, 110], "limit": [54, 55, 73, 81, 91, 96, 108, 116], "author": [54, 83], "conv_nod": 54, "7": [54, 56, 58, 59, 75, 76, 86, 89, 94, 100, 101, 103, 107, 112], "ignor": [54, 71, 75, 76, 94], "misc": [54, 94], "trttensor": 54, "np": [54, 93, 94, 95, 96, 98, 99, 108, 111], "ndarrai": [54, 94], "aten_ops_convolut": 54, "conversioncontext": [54, 93, 94], "side": [54, 55, 80, 89, 93], "effect": [54, 55, 64, 65, 71, 80, 89, 91, 93, 94, 110], "term": [54, 76, 82, 83, 91, 93, 94, 110], "getitem": 54, "categor": 54, "modif": [54, 62, 76], "op_evalu": 54, "capbility_valid": 54, "opcod": 54, "decompos": 54, "suboper": 54, "separ": [54, 56, 57, 59, 66], "Such": 54, "via": [54, 64, 65, 67, 69, 71, 75, 76, 77, 80, 86, 90, 91, 100, 101, 103, 105, 106, 110, 112, 114, 115, 116], "register_torch_trt_decomposit": 54, "addmm_replac": 54, "replac": [54, 56, 62, 66, 67, 74, 94, 107, 116], "input_": 54, "mat1": 54, "mat2": [54, 70], "beta": [54, 65, 70, 77], "alpha": [54, 65, 70, 83], "mul": [54, 56, 70, 93], "matmul": [54, 55, 64, 70, 71, 89, 105, 106, 112], "modifi": [54, 56, 62, 65, 83, 98, 112], "edit": [54, 66, 80], "torch_enabled_decomposit": 54, "torch_disabled_decomposit": 54, "disjoint": 54, "preced": [54, 82], "over": [54, 57, 59, 65, 82, 107, 108, 111, 116], "much": [54, 60, 80, 82, 91], "significantli": [54, 55, 80, 96], "easier": [54, 57, 59, 60, 65, 71, 75, 76, 89, 91, 94, 98], "tri": 54, "made": [55, 57, 59, 76, 82], "represent": [55, 60, 65, 88, 110, 116], "instanc": [55, 62, 64, 66, 71, 74, 75, 88, 89, 93, 110, 114], "idea": [55, 82, 93], "reduc": [55, 56, 57, 59, 65, 71, 76, 91, 94, 96, 110, 114], "actual": [55, 58, 60, 65, 88, 89, 94], "aim": [55, 116], "closer": 55, "scope": [55, 94, 100, 103], "csrc": [55, 61], "common_subexpression_elimin": 55, "subexpress": 55, "dead_code_elimin": 55, "exception_elimin": 55, "wa": [55, 58, 62, 64, 65, 71, 75, 76, 82, 89, 93, 116], "1013": 55, "ne": [55, 70], "1012": 55, "24": [55, 67], "lib": [55, 66, 67, 89], "python3": [55, 66, 89], "site": [55, 66, 82, 89], "nn": [55, 61, 65, 71, 72, 75, 76, 77, 88, 89, 90, 93, 94, 100, 107, 112, 113, 116], "batchnorm": 55, "248": 55, "11": [55, 66, 82, 86, 89, 111], "block0": 55, "raiseexcept": 55, "249": 55, "12": [55, 56, 67, 82, 86, 88, 89, 101, 111, 112], "block1": 55, "guard_elimin": 55, "whose": [55, 65, 101], "freeze_modul": 55, "propag": 55, "fuse_addmm_branch": 55, "variant": [55, 114], "caught": 55, "ret": 55, "622": 55, "self": [55, 58, 60, 70, 75, 76, 88, 89, 90, 93, 94, 96, 100, 107, 110, 112, 113, 117], "bia": [55, 70, 89, 107], "x9": 55, "3677": 55, "output0": [55, 113], "add_": [55, 70, 89, 93], "fuse_linear": 55, "back": [55, 56, 58, 59, 75, 76, 82, 88, 89, 94, 116], "fuse_flatten_linear": 55, "implicitli": [55, 76], "connect": [55, 71, 76, 77, 82, 98, 111, 117], "higher": [55, 64, 71, 76, 80, 82, 88, 108], "1d": 55, "lower_graph": 55, "access": [55, 60, 65, 80, 89, 92, 116], "rather": 55, "getattr": [55, 58, 88, 89], "trainabl": 55, "remain": [55, 76, 91, 116], "lower_tupl": 55, "lowersimpletupl": 55, "tupleconstruct": [55, 58], "tupleunpack": 55, "leav": [55, 62, 64, 71], "statement": [55, 82, 93], "loweralltupl": 55, "_all_": 55, "rais": [55, 65, 76], "onnx": 55, "module_fallback": 55, "consist": [55, 65, 82, 94, 114, 116], "pair": [55, 60, 66, 82, 91, 110], "delimit": 55, "around": [55, 58, 60, 64, 66, 71, 75, 82, 85, 88, 94], "second": [55, 65, 82, 90, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "mark": [55, 56, 80, 96], "notatemoduleforfallback": 55, "marknodesforfallback": 55, "tell": [55, 56, 57, 58, 59, 60, 82, 116], "them": [55, 56, 58, 63, 64, 65, 66, 71, 75, 80, 89, 94, 96, 106, 110, 112, 116], "peephole_optimz": 55, "intent": [55, 82], "catch": [55, 76, 89], "small": [55, 94, 95, 111], "might": [55, 66, 80, 99, 112], "interest": [55, 82], "now": [55, 56, 59, 60, 65, 66, 76, 82, 89, 92, 93, 94, 96, 99, 108, 113, 114], "expand": [55, 70], "simpli": [55, 100, 110], "remove_contigu": 55, "remove_dropout": 55, "infer": [55, 64, 65, 71, 76, 77, 89, 91, 99, 100, 108, 110, 112, 114, 115, 116], "remove_to": 55, "unpack_addmm": 55, "reus": [55, 65, 91, 96], "dedic": [55, 83], "unpack_log_softmax": 55, "softmax": [55, 65, 70, 107], "loop_unrol": 55, "suffici": [55, 66, 76], "short": [55, 64, 71, 82, 83, 99], "tile_to_repeat": 55, "instruct": [56, 57, 59, 65, 66, 89, 111], "criteria": [56, 57, 59, 64], "lack": [56, 57, 59, 65, 94, 108], "explicitli": [56, 57, 59, 66, 77, 90, 91, 92, 105, 106, 113], "On": 56, "segment": [56, 63, 94, 101, 103, 110], "verifi": [56, 71, 93, 94, 99], "Then": [56, 91, 92, 99], "roughli": 56, "analysi": 56, "everi": [56, 72, 75, 76, 89, 114], "complet": [56, 63, 71, 76, 88, 89], "mean": [56, 60, 65, 70, 72, 100, 108, 111, 116], "trace": [56, 65, 71, 75, 77, 88, 89, 112, 115, 116], "tensorlist": [56, 60], "figur": [56, 83, 85], "our": [56, 59, 63, 88, 89, 111], "stitch": [56, 89], "altogeth": [56, 80], "brief": 56, "descript": [56, 83, 107], "partitioninfo": 56, "api": [56, 59, 60, 62, 63, 64, 65, 75, 76, 77, 81, 89, 90, 91, 92, 94, 100, 101, 104, 108, 110, 111, 112, 114, 115], "maintain": [56, 58, 60, 76, 98, 116], "code": [56, 59, 62, 64, 65, 66, 81, 83, 88, 89, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 112], "mymodel": [56, 63, 68, 90, 94, 112, 115], "ts_model": [56, 89], "trt_model": [56, 92, 94, 101, 105, 106, 107, 108, 111, 115], "off": [56, 58, 104], "consecut": [56, 63], "satisfi": [56, 62, 65], "forced_fallback_op": 56, "randn": [56, 63, 68, 71, 76, 77, 89, 92, 93, 96, 101, 104, 112, 113, 115], "224": [56, 63, 68, 71, 72, 76, 77, 89, 96, 98, 99, 101, 104, 110, 111, 112, 115], "trt_ts_modul": [56, 90], "input_s": 56, "inputrang": 56, "cfg": [56, 89], "relu": [56, 70, 88, 89, 100, 107], "trt_mod": [56, 68, 89, 91, 117], "consid": [56, 77, 89, 94, 113], "segmentmodelwithdependencyawar": 56, "test_segment": 56, "20": [56, 67, 86, 99, 101, 103], "x_lgamma": 56, "lgamma": 56, "y_lgamma": 56, "div": [56, 70], "div_lgamma": 56, "27": [56, 89], "cat": [56, 66, 67, 70, 107, 108], "greedi": [56, 105, 106, 108], "strategi": [56, 76], "travers": [56, 57, 59, 64], "gather": 56, "same": [56, 58, 62, 64, 65, 66, 71, 76, 80, 82, 88, 89, 92, 94, 96, 99, 101, 103, 111, 112, 114, 115], "encount": [56, 64, 66, 93, 100, 103], "4": [56, 58, 63, 64, 65, 66, 70, 76, 78, 80, 82, 83, 86, 89, 94, 100, 102, 103, 104, 107, 112], "suboptim": 56, "arithmet": 56, "split": [56, 65, 70], "own": [56, 60, 64, 66, 71, 82, 89, 96, 107, 111], "could": [56, 64, 65, 94, 101, 103, 114], "rewrit": [56, 62], "portion": [56, 82, 94, 102], "without": [56, 60, 68, 71, 80, 82, 89, 91, 94, 95, 96, 99, 114], "reorder": 56, "seri": 56, "cleanli": 56, "approach": [56, 96], "achiev": [56, 110], "hit": 56, "larger": [56, 71, 76, 80, 108, 110], "boundari": [56, 74, 76], "guarante": [56, 75], "trigger": [56, 64, 65, 76, 89, 96, 98, 99, 116], "appear": [56, 82], "adjac": [56, 71, 76, 82], "As": [56, 65, 66, 76, 89, 93, 94, 96, 99, 116], "clean": [56, 62, 82, 100, 103], "step": [56, 65, 67, 70, 76, 91, 94, 99, 110], "consolid": [56, 88], "further": [56, 64, 65, 114, 116], "merg": 56, "identifi": 56, "do_not_merg": 56, "combin": [56, 64, 65], "condit": [56, 82, 116], "loop": [56, 64, 65, 105, 106], "ir": [57, 59, 60, 63, 64, 68, 71, 76, 88, 89, 90, 97, 100, 101, 103, 104, 109, 112], "larg": [57, 59, 80, 82, 89, 91, 99, 108, 110], "opset": [57, 59, 93], "compon": [57, 59, 66, 67, 74, 88, 114, 116], "evalu": [57, 58, 59, 107], "deploi": [57, 59, 69, 89, 91, 97, 109, 111], "instanti": [57, 58, 59, 60, 89, 102], "wrap": [57, 58, 59, 65, 82, 85, 89, 92, 100, 103], "extend": [57, 59, 60, 70, 89, 96, 110], "providi": [57, 59], "stand": [58, 82], "interpret": [58, 65, 82], "execute_engin": [58, 75, 89], "stack": [58, 70, 91, 107, 116], "machin": [58, 66, 91, 111], "pop": 58, "push": 58, "element": [58, 65, 82, 83, 86], "realiz": 58, "abstract": [58, 60, 83, 93], "__torch__": [58, 88, 89], "portabl": [58, 66, 77], "serializ": [58, 64, 88, 116], "instnanti": 58, "whatev": [58, 65, 94], "self_1": [58, 89], "torchvis": [58, 91, 92, 96, 98, 99, 101, 104, 107, 111], "resnet": [58, 69, 78, 97, 98, 109, 110, 111], "___torch_mangle_4847": 58, "resnet_trt": 58, "input_0": [58, 89], "__torch___torchvision_models_resnet____torch_mangle_4847_resnet_trt_engin": 58, "listunpack": [58, 89], "multipl": [58, 66, 71, 75, 76, 82, 83, 91, 108, 111, 114], "repack": 58, "ssd": 58, "ssd300_trt": 58, "__torch___pytorch_detection_ssd_src_model_ssd300_trt_engin": 58, "holder": [58, 84], "torchbind": 58, "pickler": 58, "seril": 58, "zip": [58, 66, 98, 99, 109], "depickl": 58, "encod": [58, 110], "sm": 58, "correct": [58, 66, 80, 97, 98, 99, 107, 109], "bazel": [59, 66, 67], "linux": [59, 67, 89], "x86_64": [59, 66], "aarch64": 59, "gcc": [59, 89], "untest": 59, "try": [59, 76, 82, 83, 89, 92, 94, 96, 116], "older": 59, "repositori": [59, 66, 80, 87, 111], "notebook": [59, 69, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "doc": [59, 61, 66, 67, 80, 81, 82, 87, 93, 94, 112], "docsrc": 59, "third_parti": [59, 66], "toolchain": [59, 66, 67], "unstabl": 59, "subject": [59, 62, 116], "matur": 59, "most": [59, 65, 66, 72, 94, 99, 111, 114, 116], "hood": [59, 101, 116], "major": [59, 65, 76], "top": [59, 80, 84], "coordin": [59, 76], "ingest": 59, "flow": [60, 65, 82, 88, 110], "ilay": 60, "analogu": 60, "goal": [60, 64, 96], "registernodeconversionpattern": [60, 89], "helper": [60, 93], "pattern": [60, 76, 89, 108], "schema": [60, 89, 93, 94], "caus": [60, 64, 80, 100, 101, 103, 108, 114], "acthardtanh": 60, "torchtrt_unus": 60, "hardtanh": [60, 70], "scalar": [60, 70], "min_val": [60, 70], "max_val": [60, 70], "unwraptodoubl": 60, "new_lay": 60, "addactiv": 60, "activationtyp": [60, 65], "kclip": 60, "torchtrt_check": 60, "unabl": [60, 89, 94], "setalpha": 60, "setbeta": 60, "setnam": [60, 89], "util": [60, 62, 74, 77, 89, 91, 100, 103, 105, 106, 107, 108, 110, 111, 116], "node_info": [60, 89], "c_str": [60, 89], "out_tensor": [60, 89], "associatevalueandtensor": [60, 89], "getoutput": [60, 89], "log_debug": 60, "getdimens": [60, 89], "accord": [60, 64, 77], "unwrap": 60, "tool": [60, 64, 65, 66, 89, 93, 96, 110], "don": [60, 65, 80, 82, 83, 91, 93, 107, 111, 112], "annot": [60, 89], "your": [60, 63, 64, 66, 67, 68, 75, 80, 82, 83, 87, 88, 89, 90, 92, 96, 112, 114], "Its": [60, 82], "track": [60, 91], "sort": [60, 70, 92], "live": [60, 82], "directli": [60, 62, 63, 66, 69, 74, 76, 91, 93, 94, 100, 115], "associatevalueandivalu": 60, "inspect": [60, 88, 89], "dataflow": [60, 89], "mechan": [60, 64, 65, 94, 99, 110], "safe": [60, 64, 71, 75, 76, 77], "unsur": 60, "deep": [60, 64, 69, 80, 91, 94, 117], "straight": 60, "chanc": 60, "none": [60, 64, 65, 70, 71, 72, 74, 75, 76, 77, 80, 82, 93, 94, 96, 100, 107, 108], "wrapper": [60, 65, 115], "similar": [60, 63, 64, 65, 66, 89, 92, 94, 105, 106], "tocustomclass": 60, "tensorcontain": 60, "istensor": 60, "iscustomclass": 60, "lot": [60, 63], "singular": 60, "becaus": [60, 65, 66, 72, 88, 89, 93, 94, 95, 96, 108, 113], "alloc": 60, "freed": 60, "destructor": 60, "destroi": [60, 83], "realli": 60, "think": [60, 82], "becom": [60, 66, 98], "benefit": [60, 89, 96, 108], "deal": [60, 96], "quit": [60, 66, 89, 110], "effici": 60, "batch_norm": [60, 70], "fusion": [60, 62, 65], "deeplearn": [61, 65, 67], "sdk": [61, 67, 116], "matrix": 61, "html": [61, 66, 67, 82, 88, 91, 93, 94, 112], "c_api": 61, "python_api": 61, "org": [61, 66, 80, 82, 88, 89, 91, 93, 94, 112, 114], "stabl": [61, 67, 69, 77, 78, 80, 97, 109, 112], "master": [61, 66, 91, 114], "overview": [61, 69, 100, 104], "md": 61, "appli": [62, 63, 91, 99], "desir": [62, 71, 83, 91, 96], "coalesc": 62, "insert": [62, 64, 71, 89, 91, 93, 96, 99], "graphmodul": [62, 63, 71, 72, 76, 89, 90, 94, 99, 115, 116], "caller": 62, "invok": [62, 64, 65, 88, 89, 114], "lint": 62, "recompil": [62, 71, 76, 93, 96, 99, 103, 112, 116], "repair": 62, "disallow": 62, "repair_input_as_output": 62, "gm": [62, 71], "sample_input": [62, 65, 100], "scenario": [62, 64, 98, 108], "clone": [62, 66, 70, 94], "modified_graph": 62, "extract": [62, 89, 110], "placehold": [62, 93], "isinst": [62, 65, 94, 107], "issubclass": 62, "direct": [62, 86, 99, 114], "len": [62, 70, 94], "direct_output": 62, "inserting_aft": 62, "cloned_placehold": 62, "replace_input_with": 62, "date": [62, 83, 116], "eliminate_dead_cod": 62, "logger": [62, 73], "f": [62, 64, 65, 67, 76, 82, 88, 93, 94, 107, 108], "__init__": [62, 75, 76, 82, 88, 93, 94, 96, 100, 107, 112, 113], "pass_manag": 62, "passmanag": 62, "backend": [62, 68, 69, 77, 78, 81, 92, 95, 96, 97, 100, 107, 109, 112], "offer": [62, 64], "registr": [62, 65], "conveni": [62, 91, 103, 110, 114, 116], "control": [62, 65, 88, 99, 108, 114], "_aten_lowering_pass": 62, "my_custom_pass": 62, "front": [62, 71], "passlist": 62, "arbitrari": [62, 75], "remov": [62, 63, 71, 80, 95, 96, 107], "dump_lowering_pass": 62, "apply_lowering_pass": 62, "graph_modul": [62, 71], "_remove_lowering_pass": 62, "evolv": 62, "introduc": [63, 65, 110], "exportedprogram": [63, 68, 71, 76, 99, 105, 106, 108, 112, 116], "dynamo": [63, 64, 66, 68, 74, 75, 76, 78, 89, 93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 107, 108, 109, 112, 113], "frontend": [63, 71, 74, 90, 94, 97, 101, 103, 107, 109, 112], "simpl": [63, 64, 65, 82, 83, 88, 110, 111, 112], "usag": [63, 65, 69, 74, 78, 82, 89, 97, 108, 109, 112, 115], "eval": [63, 68, 89, 90, 93, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 111, 112, 113, 115], "exp_program": [63, 96, 99, 107, 112], "trt_gm": [63, 68, 96, 99, 112, 113, 115], "interact": [63, 82, 98, 100, 101, 102, 103, 104], "ideal": 63, "discuss": [63, 64, 111], "section": [63, 65, 80, 82, 83, 84, 86, 89, 91, 111, 115], "frequent": 63, "builder": [63, 64, 65, 71], "respect": [63, 64, 66, 71, 76, 105, 106, 113], "releas": [63, 64, 67, 82], "insid": [63, 82, 94, 111], "decomposit": [63, 64, 71, 76, 94], "downstream": [63, 110], "constraint": [63, 108], "guid": [64, 81], "present": [64, 99], "learn": [64, 66, 69, 89, 91, 94, 111, 117], "acceler": [64, 72, 76, 114, 116, 117], "workflow": [64, 65, 68, 69, 71, 72, 76, 89, 92, 96, 97, 98, 101, 102, 103, 105, 106, 109, 110], "wide": [64, 76, 86], "varieti": [64, 111], "primari": [64, 93, 96, 115], "simplic": 64, "optimized_model": [64, 68, 95, 100, 101, 103], "depth": [64, 80, 110], "challeng": [64, 98, 111], "addition": [64, 94], "fit": [64, 82], "compilationset": [64, 71, 75, 93, 94, 100], "_enum": [64, 71], "callabl": [64, 71, 76], "pass_through_build_failur": [64, 71, 75, 76, 94], "max_aux_stream": [64, 71, 75, 76, 94], "version_compat": [64, 71, 75, 76, 94], "optimization_level": [64, 71, 75, 76, 94, 100], "use_python_runtim": [64, 71, 75, 76, 94, 95, 96, 98, 99, 100], "truncate_doubl": [64, 71, 75, 76, 94, 95, 105, 106, 108], "use_fast_partition": [64, 71, 75, 76, 94], "enable_experimental_decomposit": [64, 71, 75, 76, 94], "_devic": [64, 71], "assume_dynamic_shape_support": [64, 71, 75, 76], "make_refitt": [64, 71, 75, 76, 95, 96, 98, 99], "engine_cap": [64, 71, 75, 76, 94], "dryrun": [64, 71, 75, 76, 94], "hardware_compat": [64, 71, 75, 76, 94], "timing_cache_path": [64, 71, 75, 76, 96], "tmp": [64, 71, 75, 76, 89, 95], "torch_tensorrt_engine_cach": [64, 71, 75, 76], "timing_cach": [64, 65, 71, 75, 76], "bin": [64, 66, 67, 71, 75, 76], "lazy_engine_init": [64, 71, 75, 76], "cache_built_engin": [64, 71, 75, 95, 96], "reuse_cached_engin": [64, 71, 75, 95, 96, 99], "use_explicit_typ": [64, 71, 75, 105, 106, 108, 113], "use_fp32_acc": [64, 71, 75, 105, 106], "enable_weight_stream": [64, 71, 75, 108], "dpython": [64, 71, 76, 77], "per": [64, 71, 94, 114], "regardless": [64, 71, 83, 101, 103], "fail": [64, 71, 76, 89, 98, 99, 107, 117], "auxiliari": [64, 71], "stream": [64, 69, 71, 76, 94, 97, 109], "impli": [64, 71], "longer": [64, 66, 71, 76, 80, 114], "search": [64, 69, 71, 76, 80], "strictli": [64, 71], "runtim": [64, 66, 68, 69, 71, 76, 89, 93, 98, 100, 103, 104, 108, 116], "presenc": [64, 71], "preferenti": [64, 71], "choos": [64, 65, 71, 88], "float64": [64, 71, 76, 77], "refitt": [64, 71, 96], "toggl": [64, 71, 76], "mode": [64, 65, 71, 75, 76, 90, 91, 93, 104, 107], "detail": [64, 65, 67, 71, 88, 89, 94, 96, 111, 114], "natur": [64, 71, 82], "architectur": [64, 66, 69, 71, 76, 96, 110], "amper": [64, 71, 76], "newer": [64, 66, 71, 76], "storag": [64, 71, 91], "use_strong_typ": [64, 71], "strong": [64, 71, 82], "mix": [64, 69, 71], "happen": [64, 65, 71, 88, 98, 101, 112], "sub": [64, 70, 82, 88, 100], "slate": 64, "futur": [64, 65, 71, 76, 77, 114], "occur": [64, 108], "first_output": 64, "subsequ": [64, 96], "second_output": 64, "session": [64, 68, 82, 96, 104], "point": [64, 66, 76, 80, 81, 82, 89, 107, 111], "cover": [64, 93], "benchmark": [64, 70], "automat": [64, 67, 76, 82, 89, 99, 112, 116], "vari": [64, 72, 108, 112], "distribut": [64, 67, 89, 91, 108, 114], "inf": 64, "dynamo_convers": 64, "contribut": 64, "demonstr": [64, 82, 83, 84, 91, 93, 94, 96, 98, 107, 110, 111], "break": [64, 65, 71, 75, 76, 82, 94, 106], "successfulli": [64, 98, 99], "_dynamo": [64, 95, 96, 100, 101, 103, 112], "explain": [64, 65, 69], "veri": [64, 65, 83, 84, 91, 92, 105, 106, 111], "explan": [64, 65], "graph_break_count": 64, "furthermor": 64, "durat": [64, 82], "latter": [64, 75], "logic": [64, 65, 93], "guard": 64, "compos": [65, 88, 91, 93, 107, 111], "variou": [65, 117], "etc": [65, 80, 82, 94, 117], "environ": [65, 68, 111], "research": 65, "few": [65, 66, 76, 93], "nightli": 65, "lower_exampl": 65, "welcom": [65, 89], "finish": 65, "converison": 65, "pleas": [65, 67, 76, 82, 89, 97, 107, 109, 111, 112], "max_batch_s": [65, 72, 111], "2048": [65, 72], "max_workspace_s": [65, 72], "33554432": [65, 72], "explicit_batch_dimens": [65, 72], "lower_precis": [65, 72], "lowerprecis": [65, 72], "verbose_log": [65, 72], "timing_cache_prefix": [65, 72], "save_timing_cach": [65, 72], "cuda_graph_batch_s": [65, 72], "dynamic_batch": [65, 72], "turn": [65, 72, 104], "trtmodul": [65, 72], "otherwis": [65, 66, 72, 96, 114], "implicit": [65, 70, 72, 82], "config": [65, 66, 72, 111], "updat": [65, 66, 67, 71, 72, 76, 94, 99], "dim": [65, 70, 72, 94, 96, 107, 108, 111, 112], "fx2trt_exampl": 65, "acc_trac": 65, "come": [65, 66, 81, 94, 98, 111], "my_pytorch_model": 65, "build_model": 65, "prepar": [65, 111], "acc_mod": 65, "earli": [65, 99], "deprec": [65, 70], "continu": [65, 82, 114], "backward": [65, 75, 94, 116], "vision": [65, 97, 109, 111], "activ": [65, 75, 77, 82, 89, 91, 93, 110, 114, 117], "except": [65, 71, 76], "permut": [65, 70], "transpos": [65, 70, 112], "ll": [65, 96], "inputtensorspec": [65, 72, 76], "experiment": [65, 76, 77], "dataclass": [65, 100], "re": [65, 76, 82, 96, 98, 104, 114], "manual": [65, 76, 81, 82, 99, 108], "sampl": [65, 71, 82, 90, 91, 98, 99, 100, 101, 102, 103, 104, 105, 106, 111], "rand": [65, 89, 96, 98, 99, 100], "from_tensor": [65, 76], "slightli": [65, 66, 94], "promis": 65, "optimize_target_shap": 65, "input_tensor_spec": 65, "shape_rang": [65, 72], "100": [65, 72, 94, 96, 107, 108], "accordingli": [65, 80, 112, 114], "trtinterpreterresult": [65, 72], "namedtupl": 65, "input_nam": [65, 72], "output_nam": [65, 72], "serialized_cach": [65, 72], "bytearrai": [65, 75, 77], "afford": 65, "temporari": [65, 96], "best": [65, 71, 76, 82, 98, 108, 113], "perforamnc": 65, "examin": 65, "suitabl": [65, 93], "force_fp32_output": 65, "strict_type_constraint": 65, "usual": [65, 66, 80], "unless": 65, "certain": [65, 66, 100, 105, 106, 108, 114], "algorithm_selector": 65, "profiling_verbos": 65, "trt_interpreter_result": 65, "64": [65, 76, 90, 106, 107, 112], "25": [65, 72, 89], "runtimeerror": [65, 107], "xxx": 65, "One": [65, 82, 83, 89, 110, 114], "reload_trt_mod": 65, "reload_model_output": 65, "far": [65, 82], "give": [65, 80, 82], "convtert": 65, "scheme": [65, 71, 76], "action": [65, 82], "tensort": [65, 116], "thing": [65, 66, 82], "compar": [65, 71, 76, 90, 99], "vanilla": 65, "mainli": 65, "builtin": 65, "purpos": [65, 110, 111], "acc_op": 65, "leverag": [65, 91], "power": [65, 82, 89, 108, 110], "goe": [65, 82], "whole": 65, "sigmoid": [65, 70], "tensorrt_convert": 65, "acc_ops_sigmoid": 65, "rest": [65, 82, 83], "input_v": [65, 93], "receiv": 65, "region": 65, "add_activ": 65, "get_output": [65, 94], "wherev": 65, "rememb": [65, 66], "mapper": 65, "todo": [65, 67, 80], "logist": 65, "down": [65, 66, 80, 106], "acc_norm": 65, "foo": [65, 82, 83], "register_acc_op": 65, "register_acc_op_map": 65, "this_arg_is_opt": 65, "op_and_target": 65, "arg_replacement_tupl": 65, "rule": [65, 66, 77], "third": [65, 83], "boolean": [65, 76, 93], "matter": [65, 94], "register_custom_acc_mapper_fn": 65, "design": [65, 74, 93, 98, 105, 108, 110, 117], "redund": 65, "throught": 65, "custom_mapp": 65, "_": [65, 82, 94, 107, 108, 113], "foo_kwarg": 65, "inserting_befor": 65, "foo_nod": 65, "meta": [65, 67, 86, 106, 108], "children": 65, "unit": [65, 76], "test_acc_trac": 65, "acc_op_convert": 65, "essenti": 65, "plugin": [65, 94], "yet": [65, 110], "folder": 65, "center": 66, "pypi": 66, "m": [66, 67, 83, 107], "pip": [66, 67, 97, 109, 111], "upload": [66, 111], "x86": [66, 114], "extra": [66, 75, 89, 94, 98], "url": [66, 80, 111], "download": [66, 67, 86, 91, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 111], "whl": [66, 67], "cu118": 66, "cu124": 66, "tarbal": [66, 89, 91], "easiest": [66, 94], "bazelisk": [66, 67], "bazelbuild": [66, 67], "export": [66, 67, 69, 71, 76, 96, 97, 99, 101, 105, 106, 107, 108, 109, 113, 115, 116], "bazel_vers": 66, "path_to_torchtrt_root": 66, "bazelvers": 66, "mkdir": 66, "cd": [66, 111], "curl": [66, 82], "fssl": 66, "o": [66, 82, 111], "dist": 66, "unzip": 66, "bash": 66, "sh": 66, "cp": [66, 67, 94], "usr": [66, 67], "driver": 66, "branch": [66, 67], "4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d": 66, "l46": 66, "pull": [66, 96, 111], "latest": [66, 67, 80], "l53c1": 66, "fact": 66, "reproduc": 66, "l71": 66, "http_archiv": 66, "build_fil": 66, "archiv": [66, 67], "sha256": 66, "strip_prefix": 66, "OR": 66, "TO": [66, 89], "gnu": 66, "tar": [66, 67, 82, 91], "gz": [66, 82, 83, 91], "ld_library_path": 66, "comment": [66, 82], "uncom": 66, "l114c1": 66, "l124c3": 66, "uv": 66, "astral": 66, "project": [66, 81, 86], "simpler": [66, 91], "wheel": [66, 67], "dep": 66, "lighter": 66, "executor": 66, "avoid": [66, 93, 94, 99, 112], "implic": 66, "python_onli": 66, "legaci": [66, 74], "mainten": 66, "torchdynamo": [66, 112, 116], "technolog": [66, 116], "exclud": [66, 94], "speed": [66, 96, 99], "no_torchscript": 66, "dbg": 66, "pre_cxx11_abi": 66, "complic": 66, "incompat": 66, "popular": [66, 81, 97, 105, 106, 109, 110], "ngc": [66, 67, 111], "tabl": [66, 86], "bdist_wheel": 66, "preinstal": 66, "forum": 66, "correctli": [66, 94], "declar": 66, "intend": [66, 100, 101, 102, 103, 104], "microsoft": 66, "2022": [66, 69], "open": [66, 110, 111], "app": 66, "x64": 66, "prompt": [66, 98, 102, 105, 106], "admin": 66, "privileg": 66, "launcher": 66, "chocolatei": 66, "navig": [66, 80], "ninja": 66, "setuptool": 66, "r": [66, 67, 82, 97, 109], "txt": [66, 67, 97, 109], "distutils_use_sdk": 66, "cuda_win": 66, "libtorch_win": 66, "tensorrt_win": 66, "non": [66, 76, 83, 85, 114], "similarli": [66, 96, 104, 114], "ci_workspac": 66, "win": 66, "tmpl": [66, 67], "torchtrtc": [66, 69, 117], "websit": 66, "finder": 66, "dcmake_module_path": 66, "doesn": [66, 82, 88, 89], "dtorch_dir": 66, "dtensorrt_root": 66, "choic": [66, 74], "b": [66, 70, 76, 83, 108, 111], "dcmake_build_typ": 66, "72048": 66, "jp_workspac": [66, 67], "new_local_repositori": 66, "sudo": [66, 67], "home": 66, "unlik": [66, 92], "libtorch_pre_cxx11_abi": 66, "shift": [66, 70, 82], "jetpack": 66, "jetpack_x": 66, "jetpack_5": 66, "drop": [66, 80, 107], "nvida": 67, "ofjetpack": 67, "With": [67, 80, 82, 89, 91, 96, 111], "incorpor": [67, 83], "cudnn": 67, "9": [67, 86, 89, 94, 111], "dlfw": 67, "09": 67, "jetson": [67, 110], "framework": 67, "instal": [67, 69, 86, 89, 97, 109, 111, 114], "kit": 67, "flash": 67, "board": 67, "apt": 67, "show": [67, 80, 82, 96, 102, 108, 110], "dev": 67, "everth": 67, "nvcc": 67, "cmd": 67, "toolkit": [67, 74], "libcusparselt": 67, "lib64": 67, "wget": [67, 111], "cusparselt": 67, "redist": 67, "libcusparse_lt": 67, "sbsa": 67, "xz": 67, "xf": 67, "v1": [67, 98, 102], "arm64": 67, "mv": 67, "chmod": 67, "pypa": 67, "en": [67, 80], "bootstrap": 67, "jp": 67, "v61": 67, "0a0": 67, "872d972e41": 67, "nv24": 67, "08": 67, "17622132": 67, "cp310": 67, "linux_aarch64": 67, "test_requir": 67, "jetpack6": 67, "lanl": 67, "cuda_vers": 67, "grep": 67, "cut": [67, 82, 99], "sed": [67, 83, 85], "torch_install_path": 67, "dirnam": 67, "__file__": 67, "site_package_path": 67, "cuda_hom": 67, "envsubst": 67, "cxx11": [67, 114], "abi": [67, 114], "anywher": 68, "ahead": [68, 69, 89, 98], "ep": [68, 70, 99, 113, 115], "output_format": [68, 76, 115], "input_tensor": [68, 94, 107, 108], "fill": 68, "aot": [69, 89, 97, 98, 99, 109, 116], "integr": [69, 98, 100], "seamlessli": [69, 76], "ecosystem": [69, 116], "hybrid": [69, 71, 76, 77, 116], "advanc": [69, 78, 83, 91, 97, 109], "bert": [69, 78, 97, 109], "triton": [69, 94], "cudagraph": [69, 97, 109], "overload": [69, 97, 109], "mutabl": [69, 97, 109], "diffus": [69, 78, 97, 109], "gpt2": [69, 97, 109], "llama2": [69, 97, 109], "page": [69, 84, 86, 111], "introductori": 69, "blog": [69, 114], "gtc": 69, "2020": [69, 89], "talk": 69, "fall": [69, 76, 94], "2021": 69, "dai": 69, "confer": 69, "_convolut": [70, 89], "stride": [70, 76, 94, 107], "pad": [70, 76, 94, 107], "dilat": 70, "output_pad": 70, "group": [70, 82, 83], "determinist": 70, "cudnn_en": 70, "allow_tf32": 70, "ab": 70, "aco": 70, "acosh": 70, "adaptive_avg_pool1d": 70, "output_s": 70, "adaptive_avg_pool2d": 70, "adaptive_avg_pool3d": 70, "adaptive_max_pool1d": 70, "adaptive_max_pool2d": 70, "adaptive_max_pool3d": 70, "argmax": [70, 108], "keepdim": 70, "argmin": 70, "asin": 70, "asinh": 70, "atan": 70, "atanh": 70, "avg_pool1d": 70, "kernel_s": [70, 94, 107], "ceil_mod": 70, "count_include_pad": 70, "avg_pool2d": 70, "divisor_overrid": 70, "avg_pool3d": 70, "gamma": 70, "var": 70, "momentum": 70, "bitwise_not": 70, "bmm": 70, "ceil": 70, "clamp": 70, "clamp_max": 70, "clamp_min": 70, "constant_pad_nd": 70, "co": [70, 83, 110], "cosh": 70, "cumsum": 70, "tensor_mod": 70, "rounding_mod": 70, "div_": 70, "elu": 70, "scale": [70, 91, 110], "input_scal": 70, "indic": [70, 80, 82, 93, 99, 101, 112, 113], "padding_idx": 70, "eq": [70, 82], "erf": [70, 93], "exp": 70, "expand_a": 70, "fake_quantize_per_channel_affin": 70, "zero_point": 70, "axi": [70, 76], "quant_min": 70, "quant_max": 70, "fake_quantize_per_tensor_affin": 70, "using_int": [70, 89], "start_dim": [70, 89], "end_dim": [70, 89], "floor": 70, "floor_divid": 70, "ge": 70, "gru_cel": 70, "hx": 70, "w_ih": 70, "w_hh": 70, "b_ih": 70, "b_hh": 70, "gt": 70, "hardtanh_": 70, "instance_norm": 70, "running_mean": 70, "running_var": 70, "use_input_stat": 70, "layer_norm": 70, "normalized_shap": 70, "le": 70, "negative_slop": 70, "01": [70, 83, 89, 107], "leaky_relu_": 70, "lstm_cell": 70, "lt": 70, "masked_fil": 70, "mask": [70, 94], "max_pool1d": 70, "max_pool2d": [70, 88, 89], "max_pool3d": 70, "mul_": [70, 93], "narrow": 70, "neg": [70, 98], "norm": 70, "scalaropt_dim": 70, "pixel_shuffl": 70, "upscale_factor": 70, "pow": 70, "tensor_scalar": 70, "expon": 70, "tensor_tensor": 70, "prelu": 70, "prod": [70, 94], "dim_int": 70, "reciproc": 70, "reflection_pad1d": 70, "reflection_pad2d": 70, "relu_": 70, "repeat_interleav": 70, "self_int": 70, "replication_pad1d": 70, "replication_pad2d": 70, "replication_pad3d": 70, "reshap": [70, 94, 111], "roll": 70, "rsub": 70, "scatter": 70, "sigmoid_": 70, "sin": [70, 82], "sinh": 70, "slice": 70, "split_siz": 70, "split_with_s": 70, "sqrt": 70, "squar": 70, "squeez": [70, 110], "sub_": 70, "dim_intlist": 70, "tan": 70, "tanh": [70, 93], "tanh_": [70, 93], "non_block": [70, 107], "memory_format": [70, 76], "prim_devic": 70, "topk": 70, "k": [70, 91, 107], "largest": 70, "dim0": [70, 96], "dim1": 70, "unbind": 70, "unsqueez": 70, "upsample_bilinear2d": 70, "align_corn": 70, "scales_h": 70, "scales_w": 70, "vec": 70, "scale_factor": 70, "upsample_linear1d": 70, "upsample_nearest1d": 70, "upsample_nearest2d": 70, "upsample_nearest3d": 70, "scales_d": 70, "upsample_trilinear3d": 70, "view": [70, 80], "__and__": 70, "__derive_index": 70, "idx": 70, "__getitem__": 70, "__is__": 70, "t1": 70, "t2": 70, "obj": 70, "__isnot__": 70, "__not__": 70, "__or__": 70, "__range_length": 70, "lo": 70, "hi": [70, 82, 83], "__round_to_zero_floordiv": 70, "__xor__": 70, "append": [70, 93, 95, 96, 107, 108], "el": 70, "arang": [70, 94], "pin_memori": 70, "start_step": 70, "copy_": 70, "float_int": 70, "int_float": 70, "floordiv": 70, "is_floating_point": 70, "numel": 70, "l": [70, 107], "9223372036854775807": 70, "requires_grad": 70, "tupleindex": 70, "tup": 70, "exported_program": [71, 76, 115], "arg_input": [71, 76, 93, 99], "kwarg_input": [71, 76, 99], "engine_cache_dir": [71, 95, 96], "engine_cache_s": [71, 95, 96], "custom_engine_cach": [71, 96], "baseenginecach": [71, 96], "int32": [71, 76, 77, 94, 95, 103, 110], "channel_last": [71, 76, 77, 110], "244": [71, 76, 77], "alia": [71, 76], "better": [71, 76, 88, 110, 116], "understand": [71, 76, 112], "convolut": [71, 76, 77, 91, 94, 117], "_c": [71, 76, 77, 92], "oppos": [71, 76, 77], "lean": [71, 76], "spend": [71, 76], "integ": [71, 76, 85], "faster": [71, 76, 95, 96, 110], "parition": [71, 76], "increas": [71, 76, 96, 108], "amount": [71, 76, 108], "defer": [71, 76, 116], "lead": [71, 76, 82, 108, 114], "oversubscript": [71, 76], "hard": [71, 99], "disk": [71, 76, 96], "space": [71, 82, 83, 91], "byte": [71, 75, 76, 77, 94, 96, 108, 110], "1gb": [71, 95, 96], "exce": 71, "oldest": 71, "gear": [71, 91], "toward": [71, 91], "refit_module_weight": [71, 99], "compiled_modul": [71, 99], "new_weight_modul": [71, 99], "verify_output": [71, 99], "use_weight_map_cach": [71, 99], "in_plac": [71, 99], "compmil": 71, "coverag": [71, 94], "min_acc_module_s": 72, "is_aten": 72, "use_experimental_fx_rt": 72, "correctness_atol": 72, "correctness_rtol": 72, "minim": [72, 91, 94], "submodul": [72, 88, 94], "fx2trt": 72, "cpu": [72, 105, 106, 108], "has_batch_dim": 72, "dtyep": 72, "prop": 72, "min_input_shap": 72, "optimized_input_shap": 72, "max_input_shap": 72, "popul": 72, "225": [72, 111], "explicit_precis": 72, "logger_level": 72, "model_trt": 73, "model_torchtrt": 73, "internal_error": 73, "dataloadercalibr": [74, 91], "preprocess": [74, 91, 111], "algo_typ": [74, 91], "calibrationalgo": [74, 91], "cachecalibr": [74, 91], "qualnam": [74, 76], "entropy_calibr": 74, "entropy_calibration_2": [74, 91], "legacy_calibr": 74, "minmax_calibr": 74, "set_multi_device_safe_mod": [75, 114], "_multidevicesafemodecontextmanag": 75, "impact": 75, "suppress": 75, "unsaf": 75, "trt_compiled_modul": 75, "torchtensorrtmodul": [75, 94], "encompass": [75, 77], "simpili": 75, "de": 75, "initi": [75, 76, 82, 99, 100, 101, 103, 104, 105, 106], "scriptmodul": [75, 76, 77, 89, 90, 115, 116], "overridden": [75, 76], "subclass": 75, "although": [75, 82], "recip": [75, 91], "afterward": 75, "former": 75, "care": 75, "hook": 75, "silent": 75, "get_extra_st": 75, "state_dict": [75, 76, 98], "set_extra_st": 75, "picklabl": 75, "pickl": [75, 94, 96], "load_state_dict": [75, 98, 107], "pythontorchtensorrtmodul": 75, "serialized_engin": [75, 77], "_set": [75, 100], "weight_name_map": 75, "trt_modul": 75, "engine_str": 75, "my_modul": 75, "current_devic": 75, "cudagraphs_validate_shap": 75, "versu": 75, "disable_profil": 75, "enable_profil": 75, "iprofil": 75, "spent": 75, "get_layer_info": 75, "request": [76, 89, 111], "decid": 76, "deseri": [76, 77, 89, 94], "retrac": 76, "strict": [76, 114], "valueerror": 76, "mutabletorchtensorrtmodul": [76, 98], "pytorch_model": 76, "regular": 76, "whenev": 76, "refit_gm": 76, "shape_mod": 76, "_shapemod": 76, "interv": 76, "notat": 76, "bound": 76, "torch_tensor": 76, "tracer": 76, "example_tensor": 76, "optimization_profile_field": 76, "classmethod": 76, "disable_memory_format_check": 76, "core_id": 76, "schedul": [76, 111], "use_default": 76, "try_to": 76, "anoth": [76, 82, 83, 88, 90, 99], "typeerror": 76, "unknown": 76, "succe": 76, "float_dtyp": 76, "failur": 76, "bf16": 76, "try_from": [76, 94], "complex128": 76, "16": [76, 86, 88, 89, 90, 101, 104], "brain": 76, "bfloat16": 76, "f64": 76, "f8": 76, "fp8": 76, "float8": 76, "i32": 76, "sign": [76, 111], "i64": 76, "u8": 76, "unsign": 76, "uint8": 76, "trt_dla": 76, "torchtrt_dla": 76, "_from": 76, "torchtrt_dla_ec": 76, "torchtrt_safety_ec": 76, "saefti": 76, "trt_dla_ec": 76, "standalon": [76, 82], "certifi": 76, "tf": 76, "torchtrt_linear": 76, "cdhw32": 76, "thirti": 76, "row": [76, 83], "spatial": 76, "31": [76, 89], "subscript": [76, 82], "chw16": 76, "sixteen": 76, "15": [76, 82, 86], "chw2": 76, "chw32": 76, "chw4": 76, "four": [76, 82, 83], "dhwc": 76, "equivi": 76, "channels_last_3d": 76, "dhwc8": 76, "eight": 76, "dla_hwc4": 76, "imag": [76, 91, 94, 98, 102, 107, 111], "roundup": 76, "elements": 76, "dla_linear": 76, "planar": 76, "hwc": 76, "channels_last": 76, "hwc16": 76, "hwc8": 76, "least": [76, 82, 83], "ishapelay": 77, "check_method_op_support": 77, "seriali": 77, "put_binding_nam": 77, "tensorrtcompilespec": [77, 92], "scriptclass": 77, "0x7efd148567b0": 77, "_jit_to_tensorrt": 77, "00": 78, "000": [78, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "total": [78, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "galleri": [78, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "mem": 78, "torch_compile_advanced_usag": [78, 100], "torch_compile_resnet_exampl": [78, 101], "torch_compile_stable_diffus": [78, 102], "torch_compile_transformers_exampl": [78, 103], "v0": [79, 111], "pytorch_sphinx_them": [80, 87], "conf": [80, 87], "html_theme_opt": 80, "canonical_url": 80, "analytics_id": 80, "logo_onli": 80, "display_vers": 80, "prev_next_buttons_loc": 80, "bottom": 80, "style_external_link": 80, "vcs_pageview_mod": 80, "collapse_navig": 80, "sticky_navig": [80, 84], "navigation_depth": 80, "includehidden": 80, "titles_onli": 80, "canon": 80, "rank": 80, "trail": 80, "slash": 80, "googl": 80, "analyt": 80, "isn": [80, 82, 94], "shown": [80, 82, 89, 113], "sidebar": [80, 86], "button": [80, 82], "icon": [80, 82], "extern": [80, 82, 97, 109], "display_github": 80, "display_gitlab": 80, "gitlab": 80, "bitbucket": 80, "bar": [80, 82], "www": [80, 82, 89, 91, 111], "sphinx": [80, 81, 82, 83, 87, 93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "toctre": 80, "lose": 80, "scroll": [80, 84], "unlimit": 80, "header": [80, 82, 83, 89, 111], "render": 80, "github_url": 80, "bitbucket_url": 80, "gitlab_url": 80, "left": [80, 82], "upon": [80, 100, 103], "rst": [80, 82], "visitor": 80, "revert": 80, "misbuild": 80, "properti": [80, 94], "stick": 80, "screen": 80, "vertic": [80, 82], "too": [80, 82, 83], "sticki": [80, 86], "nav": [80, 86], "At": [81, 93, 99], "django": 81, "payment": 81, "dotpai": 81, "dotpayprovid": 81, "seller_id": 81, "pin": 81, "lock": 81, "lang": 81, "pl": 81, "polish": 81, "gatewai": 81, "transfer": 81, "purchas": 81, "item": [81, 83, 107], "param": 81, "seller": 81, "consult": 81, "ui": 81, "languag": [81, 82, 83, 88, 94, 97, 105, 109, 111], "data_item_1": 81, "emphasi": 82, "hyperlink": 82, "cross": 82, "uri": 82, "web": 82, "anonym": 82, "label": [82, 91, 107, 110, 111], "substitut": 82, "charact": 82, "exceedingli": 82, "ugli": 82, "problem": [82, 106], "problemat": 82, "ext": [82, 83], "autodoc": [82, 83], "demo": [82, 91], "test_py_modul": [82, 86], "my": [82, 105], "role": 82, "pep": 82, "287": 82, "rfc": 82, "2822": 82, "superscript": 82, "gui": 82, "taken": 82, "height": 82, "interfer": 82, "press": 82, "keyboard": 82, "mous": 82, "mmb": 82, "menuselect": 82, "seen": [82, 83], "whitespac": 82, "signific": [82, 94], "strang": 82, "hyphen": 82, "word": [82, 110], "adjust": 82, "width": [82, 110], "browser": 82, "window": 82, "sentenc": [82, 108, 110], "suppli": [82, 99], "258": 82, "equat": 82, "x_": 82, "x_0": 82, "x_1": 82, "x_2": 82, "x_3": 82, "x_4": 82, "nabla": 82, "frac": 82, "theta": 82, "phi": 82, "restructuredtext": [82, 83], "parser": [82, 107], "colon": 82, "indent": 82, "literal_block": 82, "spaces_and_linebreak": 82, "preserv": [82, 88, 91], "markup_process": 82, "Or": 82, "great": [82, 89, 94, 96, 116], "why": [82, 114], "didn": 82, "blank": 82, "align": 82, "permit": 82, "awai": 82, "eric": 82, "orchestra": 82, "leader": 82, "bee": 82, "philosoph": 82, "ipso": 82, "facto": 82, "But": [82, 89, 99, 108], "got": [82, 89], "vi": 82, "entiti": 82, "said": 82, "entir": [82, 116], "ancient": 82, "injuri": 82, "sing": 82, "elk": 82, "bracket": 82, "miss": [82, 89], "brontosaurus": 82, "thin": 82, "thicker": 82, "middl": 82, "That": [82, 89], "mine": 82, "belong": 82, "me": [82, 83], "ann": 82, "begun": 82, "past": 82, "pars": [82, 89], "someurl": 82, "dev0": 82, "e43833d": 82, "caption": [82, 85], "pane": 82, "shell_command": 82, "echo": 82, "did": 82, "window_nam": 82, "session_nam": 82, "shorthand": 82, "some_funct": 82, "highlight": 82, "THE": 82, "heaven": 82, "hexagram": 82, "six": 82, "unbroken": 82, "primal": 82, "light": [82, 115], "spirit": 82, "weak": 82, "essenc": 82, "energi": 82, "unrestrict": 82, "conceiv": 82, "motion": 82, "regard": [82, 116], "basi": 82, "thu": 82, "persist": 82, "dual": 82, "sens": [82, 89], "univers": 82, "world": 82, "men": 82, "express": 82, "deiti": 82, "human": 82, "denot": [82, 94], "holi": 82, "man": [82, 83], "sage": 82, "ruler": 82, "who": 82, "awaken": 82, "utf": [82, 83], "sphinx_rtd_them": [82, 83], "docstr": [82, 83, 90], "dl": 82, "dt": 82, "tag": [82, 111], "tt": 82, "descnam": 82, "descclassnam": 82, "wrote": 82, "anyth": [82, 83, 114], "programm": 82, "myclass": 82, "dothismethod": 82, "flush": 82, "meth": 82, "capit": 82, "flox": 82, "unreferenc": 82, "nonexist": 82, "extrem": 82, "stuff": 82, "mayb": 82, "bold": 82, "ital": 82, "heck": 82, "put": [82, 110], "13": [82, 86], "backlink": 82, "knowledg": 82, "mind": 82, "ey": 82, "thought": 82, "medium": 82, "peopl": 82, "subsect": 82, "interpol": 82, "indirect": 82, "phrase": 82, "docutil": [82, 83], "sourceforg": [82, 83], "ref": 82, "clickabl": 82, "legend": 82, "revis": [82, 83, 98, 102], "revisit": 82, "enhanc": 82, "structuredtext": 82, "wooden": 82, "nickel": 82, "mad": 82, "scientist": 82, "bigger": 82, "bread": 82, "box": [82, 112, 116], "wash": 82, "behind": 82, "ear": 82, "room": 82, "closet": 82, "bathroom": 82, "trash": 82, "sink": 82, "mother": 82, "g_": 82, "mu": 82, "nu": 82, "pi": 82, "t_": 82, "rho_": 82, "servic": 82, "thing1": 82, "thing2": 82, "thing3": 82, "prose": 82, "provok": 82, "mental": 82, "exert": 82, "reader": 82, "discret": 82, "strongli": [82, 108], "advis": 82, "subtitl": 82, "outsid": 82, "often": 82, "besid": 82, "border": 82, "background": [82, 88], "ok": [82, 89], "transmit": 82, "disconnect": 82, "nonetheless": 82, "semant": 82, "blue": [82, 94], "white": 82, "arab": 83, "roman": 83, "upper": 83, "iii": 83, "iv": 83, "classifi": [83, 88, 89, 107, 110], "paragraph": [83, 86], "z": 83, "commonli": 83, "vm": 83, "david": 83, "goodger": 83, "address": [83, 94, 98], "123": 83, "street": 83, "canada": 83, "a1b": 83, "2c3": 83, "contact": 83, "myself": 83, "organ": 83, "humankind": 83, "2012": 83, "03": 83, "19": [83, 86], "53": 83, "0000": 83, "tue": 83, "jan": 83, "progress": 83, "7302": 83, "wish": 83, "redistribut": 83, "reattribut": 83, "sell": 83, "bui": 83, "rent": 83, "leas": 83, "improv": [83, 114], "quot": 83, "excerpt": 83, "collat": 83, "fold": 83, "stapl": 83, "mutil": 83, "anyon": 83, "heart": 83, "bibliograph": 83, "markup": [83, 86], "literal": 83, "yahoo": 83, "oh": 83, "liter": 83, "heh": 83, "child": 83, "beat": 83, "text": [83, 85, 105, 106, 110], "hehe": 83, "kept": 83, "sai": [83, 110], "cackl": 83, "night": 83, "lone": 83, "guangzhou": 83, "destini": 83, "hope": 83, "dream": 83, "forth": 83, "fifth": 83, "sixth": 83, "lorem": [83, 85], "ipsum": [83, 85], "dolor": [83, 85], "sit": [83, 85], "amet": [83, 85], "consectetur": [83, 85], "adipisc": [83, 85], "elit": [83, 85], "donec": [83, 85], "porttitor": [83, 85], "odio": [83, 85], "posuer": [83, 85], "vita": [83, 85], "ornar": [83, 85], "libero": [83, 85], "matti": 83, "loborti": [83, 85], "justo": [83, 85], "vestibulum": [83, 85], "nibh": [83, 85], "aliquet": [83, 85], "feugiat": [83, 85], "sagitti": [83, 85], "nequ": [83, 85], "qui": [83, 85], "eleifend": 83, "dui": [83, 85], "rutrum": [83, 85], "lectu": [83, 85], "suscipit": [83, 85], "letter": [83, 110], "column": 83, "cell": 83, "span": 83, "nam": [83, 85], "mauri": [83, 85], "arcu": [83, 85], "stub": 83, "behav": 84, "area": 84, "interdum": 85, "nec": 85, "finibu": 85, "dictum": 85, "velit": 85, "ut": 85, "eu": 85, "efficitur": 85, "aliquam": 85, "erat": 85, "diam": 85, "gravida": 85, "imperdiet": 85, "tellu": 85, "nisl": 85, "praesent": 85, "eget": 85, "elementum": 85, "rhoncu": 85, "tincidunt": 85, "suspendiss": 85, "volutpat": 85, "scelerisqu": 85, "tristiqu": 85, "aenean": 85, "condimentum": 85, "risu": 85, "accumsan": 85, "laoreet": 85, "maximu": 85, "sapien": 85, "ligula": 85, "fringilla": 85, "commodo": 85, "proin": 85, "et": 85, "pharetra": 85, "etiam": 85, "turpi": 85, "ant": 85, "luctu": 85, "vel": 85, "malesuada": 85, "dignissim": 85, "mi": 85, "nunc": 85, "augu": 85, "sem": 85, "cursu": 85, "nulla": 85, "pellentesqu": 85, "habit": 85, "morbi": 85, "senectu": 85, "netu": 85, "fame": 85, "ac": 85, "egesta": 85, "placerat": 85, "tortor": 85, "iaculi": 85, "venenati": 85, "cra": 85, "puru": 85, "ero": 85, "vehicula": 85, "fusc": 85, "auctor": 85, "phasellu": 85, "est": 85, "viverra": 85, "conval": 85, "faucibu": 85, "vulput": 85, "feli": 85, "sodal": 85, "maecena": 85, "congu": 85, "semper": 85, "enim": 85, "blandit": 85, "sollicitudin": 85, "urna": 85, "orci": 85, "lacu": 85, "quisqu": 85, "facilisi": 85, "hendrerit": 85, "curabitur": 85, "variu": 85, "bibendum": 85, "massa": 85, "magna": 85, "tempu": 85, "metu": 85, "nisi": 85, "pretium": 85, "leo": 85, "euismod": 85, "ultric": 85, "dapibu": 85, "lacinia": 85, "vivamu": 85, "molesti": 85, "hac": 85, "habitass": 85, "platea": 85, "dictumst": 85, "git": 86, "content": [86, 91, 111], "changelog": 86, "math": 86, "14": [86, 95, 103, 111], "17": 86, "18": [86, 89, 98], "submenu": 86, "symlink": 87, "subtre": 87, "_theme": 87, "html_theme": 87, "html_theme_path": 87, "optimiz": 88, "tutori": [88, 91, 93, 94, 96, 98, 99], "beginn": 88, "intro_to_torchscript_tutori": 88, "briefli": 88, "lenet": [88, 89], "lenetfeatextractor": 88, "conv1": [88, 89], "conv2d": [88, 94, 107], "conv2": [88, 89], "lenetclassifi": 88, "fc1": [88, 89], "120": [88, 89], "fc2": [88, 89], "84": [88, 89], "fc3": [88, 89], "feat": [88, 89], "obvious": 88, "pathwai": 88, "input_data": [88, 90], "traced_model": 88, "pick": [88, 113], "script_model": [88, 92], "perspect": 88, "___torch_mangle_10": 88, "129": 88, "___torch_mangle_9": 88, "119": 88, "___torch_mangle_5": 88, "137": 88, "callmethod": 88, "138": 88, "38": 88, "39": 88, "torch_script_modul": [88, 89], "in_tensor": 88, "fly": 88, "lenet_script": [88, 89], "haven": 89, "acquir": 89, "dyanmo": 89, "almost": [89, 116], "trt_lenet_script": 89, "apr": 89, "56": 89, "04": [89, 111], "credit": 89, "stop": 89, "argc": 89, "argv": 89, "cerr": 89, "cout": 89, "even": [89, 98], "cppdoc": 89, "pretti": 89, "fashion": [89, 110], "enable_precis": 89, "And": 89, "convertgraphtotrtengin": 89, "engine_converted_from_jit": 89, "close": [89, 93], "saw": 89, "576": 89, "346": 89, "539": 89, "0464": 89, "0383": 89, "0678": 89, "0932": 89, "1045": 89, "0805": 89, "0435": 89, "0818": 89, "0208": 89, "0358": 89, "cudafloattyp": 89, "0530": 89, "1691": 89, "2802": 89, "1502": 89, "1056": 89, "1549": 89, "input0": [89, 90], "1063": 89, "input1": [89, 90], "input2": 89, "28": 89, "29": 89, "33": 89, "35": 89, "36": 89, "37": 89, "compilegraph": [89, 91], "transform": [89, 91, 95, 97, 99, 103, 105, 106, 107, 108, 109, 111, 115], "laid": 89, "translat": [89, 99], "aren": 89, "techniqu": [89, 91, 106, 114], "checkmethodoperatorsupport": 89, "modular": 89, "ship": [89, 114], "exhaust": 89, "109": 89, "addlay": 89, "yourself": 89, "question": [89, 93], "outself": 89, "flatten_convert": 89, "unwraptoint": 89, "in_shap": 89, "tovec": 89, "out_shap": 89, "shuffl": [89, 91, 107], "addshuffl": 89, "setreshapedimens": 89, "todim": 89, "extens": [89, 116], "ctype": 89, "cdll": 89, "contributor": 89, "upstream": 89, "pr": 89, "usecas": 90, "sole": [90, 91, 116], "individu": 90, "accuraci": [91, 110], "loss": [91, 110], "infrastructur": [91, 111], "streamlin": 91, "expos": [91, 94], "cpp_frontend": 91, "loading_data_recip": 91, "cifar10": [91, 107], "cstddef": 91, "ktrain": 91, "ktest": 91, "un": 91, "cs": 91, "toronto": 91, "edu": 91, "kriz": 91, "cifar": 91, "is_train": 91, "trim": 91, "use_subset": 91, "new_siz": 91, "mode_": 91, "images_": 91, "targets_": 91, "calibration_dataset": 91, "data_dir": 91, "320": 91, "4914": [91, 107], "4822": [91, 107], "4465": [91, 107], "2023": [91, 107], "1994": [91, 107], "2010": [91, 107], "dataloaderopt": 91, "worker": 91, "virtual": 91, "input_shap": [91, 117], "compile_spec": [91, 101, 117], "kf16": [91, 117], "ki8": 91, "vgg16": [91, 107], "testing_dataset": [91, 107], "totensor": [91, 107, 111], "testing_dataload": [91, 107], "num_work": [91, 107], "vgg": [91, 107], "test_ptq_dataloader_calibr": 91, "test_ptq_trt_calibr": 91, "krizhevski": 91, "hinton": 91, "2009": 91, "tini": 91, "simonyan": 91, "zisserman": 91, "2014": 91, "recognit": [91, 110], "arxiv": 91, "preprint": 91, "1409": 91, "1556": 91, "_jit_to_backend": 92, "mobilenet_v2": 92, "pretrain": [92, 96, 98, 101, 104, 110, 111], "gelu": 93, "sy": 93, "approxim": 93, "suppos": 93, "my_mod": 93, "ex_input": [93, 94], "baselin": 93, "my_standard_gelu": 93, "supports_dynamic_shap": 93, "supersed": 93, "converterprior": 93, "vers": 93, "prior": [93, 96, 112, 114], "distinct": 93, "prepend": 93, "candid": 93, "primit": 93, "compiler_ir": 93, "boilerpl": 93, "focu": [93, 98], "interoper": 93, "aten_ops_gelu": 93, "sourceir": 93, "cheap": 93, "unqiu": 93, "op_count": 93, "get_op_count": 93, "nonloc": 93, "elementwis": 93, "source_ir": 93, "lhs_val": 93, "rhs_val": 93, "x_7": 93, "x_8": 93, "79788456080000003": 93, "x_9": 93, "044714999999999998": 93, "x_10": 93, "x_11": 93, "x_12": 93, "x_13": 93, "x_14": 93, "x_15": 93, "my_custom_gelu": 93, "allclos": [93, 98, 99], "my_mod_erf": 93, "my_gelu_erf": 93, "notic": 93, "minut": [93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "converter_overload": 93, "jupyt": [93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "ipynb": [93, 94, 95, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "cost": [94, 96, 99, 114], "perhap": 94, "overhead": [94, 108, 114], "sake": 94, "circular": 94, "red": 94, "green": 94, "twice": 94, "written": 94, "openai": 94, "formal": 94, "tl": 94, "custom_op": 94, "circ_pad_kernel": 94, "all_pads_0": 94, "all_pads_2": 94, "all_pads_4": 94, "all_pads_6": 94, "orig_dims_0": 94, "orig_dims_1": 94, "orig_dims_2": 94, "orig_dims_3": 94, "y_shape_1": 94, "y_shape_2": 94, "y_shape_3": 94, "x_len": 94, "y_len": 94, "block_siz": 94, "pid": 94, "program_id": 94, "mask_i": 94, "i3": 94, "i2": 94, "i1": 94, "i0": 94, "j0": 94, "j1": 94, "j2": 94, "j3": 94, "load_idx": 94, "mask_x": 94, "launch": [94, 111], "torchtrt_ex": 94, "triton_circular_pad": 94, "mutates_arg": 94, "out_dim": 94, "tolist": 94, "all_pad": 94, "zero": 94, "orig_dim": 94, "blocksiz": 94, "256": [94, 107, 108, 111], "numblock": 94, "tracabl": 94, "prerequisit": 94, "fake": 94, "real": 94, "faketensor": 94, "register_fak": 94, "autograd": 94, "beyond": 94, "register_autograd": 94, "padded_x": 94, "my_model": 94, "2604": 94, "4232": 94, "3041": 94, "0833": 94, "2461": 94, "1270": 94, "2450": 94, "4079": 94, "2887": 94, "2828": 94, "0373": 94, "0332": 94, "3143": 94, "6344": 94, "5638": 94, "1867": 94, "5068": 94, "4363": 94, "7937": 94, "3488": 94, "1350": 94, "7966": 94, "3517": 94, "1379": 94, "5537": 94, "1088": 94, "8950": 94, "0550": 94, "6163": 94, "0109": 94, "5245": 94, "9632": 94, "5686": 94, "3775": 94, "8162": 94, "4216": 94, "4311": 94, "1649": 94, "2091": 94, "3668": 94, "1006": 94, "1447": 94, "0352": 94, "7689": 94, "8131": 94, "_run_on_gpu_0": 94, "_run_on_acc_1": 94, "dry": 94, "50": [94, 110], "count": 94, "__": 94, "were": [94, 99, 114], "aggreg": 94, "stat": 94, "latenc": [94, 108, 114], "abstractli": 94, "pkl": [94, 98], "cupi": 94, "gap": 94, "prealloc": 94, "circularpaddingplugin": 94, "ipluginv2dynamicext": 94, "field_collect": 94, "pluginfieldcollect": 94, "x_shape": 94, "num_output": 94, "plugin_namespac": 94, "plugin_typ": 94, "plugin_vers": 94, "assert": [94, 98, 99], "get_output_datatyp": 94, "input_typ": 94, "get_output_dimens": 94, "output_index": 94, "dimsexpr": 94, "exprbuild": 94, "iexprbuild": 94, "output_dim": 94, "dimensionoper": 94, "configure_plugin": 94, "inp": 94, "dynamicplugintensordesc": 94, "x_dim": 94, "desc": 94, "supports_format_combin": 94, "po": 94, "in_out": 94, "plugintensordesc": 94, "num_input": 94, "enqueu": 94, "input_desc": 94, "output_desc": 94, "in_dtyp": 94, "a_mem": 94, "unownedmemori": 94, "items": 94, "c_mem": 94, "a_ptr": 94, "memorypoint": 94, "c_ptr": 94, "a_d": 94, "memptr": 94, "c_d": 94, "a_t": 94, "as_tensor": 94, "c_t": 94, "cloned_plugin": 94, "__dict__": 94, "circularpaddingplugincr": 94, "iplugincr": 94, "field_nam": 94, "pluginfield": 94, "pluginfieldtyp": 94, "create_plugin": 94, "pluginfieldcollection_": 94, "deserialize_plugin": 94, "pads_dict": 94, "creator": 94, "trt_plugin_registri": 94, "get_plugin_registri": 94, "register_cr": 94, "untyp": 94, "get_trt_tensor": 94, "set_layer_nam": 94, "recal": 94, "intlist": 94, "circular_padding_convert": 94, "retriev": 94, "elsewher": 94, "plugin_registri": 94, "plugin_cr": 94, "get_plugin_cr": 94, "field_config": 94, "eventu": 94, "freez": 94, "_input": 94, "add_plugin_v2": 94, "circular_padding_plugin": 94, "_run_on_acc_0": 94, "grad_fn": 94, "subbackward0": 94, "custom_kernel_plugin": 94, "engine_caching_exampl": [95, 96], "remove_timing_cach": [95, 96], "bertmodel": [95, 103], "random": [95, 96, 98, 99, 108], "seed": [95, 96, 98, 99], "manual_se": [95, 96, 98, 99], "from_pretrain": [95, 98, 102, 103, 105, 106, 108], "uncas": [95, 103, 110], "return_dict": 95, "randint": [95, 103, 108], "compile_bert": 95, "enable_tim": [95, 96], "1st": [95, 96], "measur": [95, 96, 108], "2nd": [95, 96], "3rd": [95, 96], "slower": [95, 96], "messur": [95, 96], "compilation_kwarg": [95, 103], "torch_trt_bert_engine_cach": 95, "30": [95, 96, 98, 99, 101, 103, 113], "synchron": [95, 96, 108], "elapsed_tim": [95, 96], "millisecond": 95, "__name__": [95, 100, 103], "__main__": [95, 100, 103], "engine_caching_bert_exampl": 95, "paid": 96, "upfront": 96, "invalid": 96, "repeatedli": 96, "mitig": 96, "explor": 96, "resnet18": [96, 98, 99, 101, 104], "torch_trt": [96, 98, 99], "_default": 96, "_engine_cach": 96, "flexibl": [96, 116], "histor": 96, "barrier": 96, "reconstruct": 96, "ti": 96, "hash": 96, "magnitud": 96, "torch_compil": [96, 100, 101, 103, 104, 112, 116], "compiled_model": 96, "ms": [96, 108], "dynamo_compil": 96, "example_input": 96, "200": 96, "dynamic_shap": [96, 112], "remot": 96, "systen": 96, "agnost": 96, "implent": 96, "ramenginecach": 96, "held": 96, "engine_cach": 96, "torch_compile_my_cach": 96, "_torch_export_gpt2": [97, 109], "_torch_export_llama2": [97, 109], "straightforward": 98, "especi": 98, "hug": [98, 105, 106], "face": [98, 105, 106], "difficult": 98, "ever": 98, "walk": [98, 99, 105], "lora": [98, 99], "use_python": 98, "mutable_modul": 98, "model2": [98, 99], "expected_output": [98, 99], "refitted_output": [98, 99], "reload": [98, 116], "checkpoint": [98, 107], "civitai": 98, "12597": 98, "moxin": 98, "diffusionpipelin": [98, 102], "no_grad": [98, 105, 106, 107, 108], "model_id": [98, 102], "runwayml": 98, "hous": 98, "forest": 98, "shuimobysim": 98, "wuchangshuo": 98, "qualiti": 98, "worst": 98, "lowr": 98, "cloudi": 98, "watermark": 98, "pipe": [98, 102], "torch_dtyp": [98, 102], "unet": [98, 102], "negative_prompt": 98, "num_inference_step": 98, "without_lora_mut": 98, "jpg": [98, 111], "procedur": 98, "load_lora_weight": 98, "stablediffusionapi": 98, "load_lora_embed": 98, "weight_nam": 98, "safetensor": 98, "adapter_nam": 98, "lora1": 98, "set_adapt": 98, "adapter_weight": 98, "fuse_lora": 98, "unload_lora_weight": 98, "with_lora_mut": 98, "mutable_torchtrt_module_exampl": 98, "expens": 99, "involv": 99, "occasion": [99, 100, 103], "adapt": 99, "infeas": 99, "focus": 99, "mostli": 99, "recogn": 99, "behalf": 99, "init": [99, 107], "sett": 99, "randomli": 99, "exp_program2": 99, "compiled_trt_ep": 99, "new_trt_gm": 99, "accomplish": 99, "gaurente": 99, "attempt": [99, 107, 112], "rebuild": 99, "heurist": 99, "refit_engine_exampl": 99, "x_out": 100, "y_out": 100, "x_y_out": 100, "invoc": 100, "sample_inputs_half": 100, "model_half": 100, "backend_kwarg": 100, "optimized_model_custom": 100, "exit": [100, 103, 111], "2052": [100, 103], "compile_engine_and_inf": [100, 103], "new_input": [101, 103], "new_output": [101, 103], "new_batch_size_input": 101, "new_batch_size_output": 101, "inputs_bs8": 101, "mark_dynam": [101, 112], "outputs_bs8": 101, "No": [101, 112], "inputs_bs12": 101, "outputs_bs12": 101, "compvi": 102, "majest": 102, "castl": 102, "cloud": 102, "majestic_castl": 102, "png": 102, "enable_cudagraph": [104, 114], "out_trt": 104, "set_cudagraphs_mod": [104, 114], "inputs_2": 104, "inputs_3": 104, "out_trt_2": 104, "out_trt_3": 104, "torch_export_cudagraph": 104, "automodelforcausallm": [105, 106, 108], "autotoken": [105, 106], "export_llm": [105, 106, 108], "max_token": [105, 106, 108], "kv_cach": [105, 106], "token": [105, 106, 110], "pad_token_id": 105, "eos_token_id": [105, 106], "attn_implement": [105, 106, 108], "eager": [105, 106, 108], "enjoi": 105, "cute": 105, "dog": 105, "model_input": [105, 106], "return_tensor": [105, 106], "input_id": [105, 106], "regress": [105, 106], "huggingfac": [105, 106, 110], "pyt_gen_token": [105, 106], "gpt2_ep": 105, "max_seq_len": [105, 106, 108], "trt_gen_token": [105, 106], "skip_special_token": [105, 106], "parallel": 105, "paradigm": 105, "torch_export_gpt2": 105, "llama_path": [106, 108], "llama": [106, 108], "7b": [106, 108], "chat": [106, 108], "hf": [106, 108], "llama2_ep": [106, 108], "batch_decod": 106, "clean_up_tokenization_spac": 106, "solv": [106, 111], "smaller": [106, 110], "subproblem": 106, "torch_export_llama2": 106, "argpars": 107, "modelopt": 107, "mtq": 107, "export_torch_mod": 107, "layer_spec": 107, "num_class": 107, "1000": [107, 108, 111], "init_weight": 107, "in_channel": 107, "pool": [107, 117], "maxpool2d": 107, "batchnorm2d": 107, "sequenti": 107, "avgpool": 107, "adaptiveavgpool2d": 107, "4096": 107, "dropout": 107, "_initialize_weight": 107, "kaiming_normal_": 107, "fan_out": 107, "nonlinear": 107, "constant_": 107, "elif": 107, "normal_": 107, "vgg16_cfg": 107, "128": [107, 108], "argumentpars": 107, "add_argu": 107, "ckpt": 107, "parse_arg": 107, "model_state_dict": 107, "device_count": 107, "ordereddict": 107, "new_state_dict": 107, "forget": 107, "training_dataset": 107, "randomcrop": 107, "randomhorizontalflip": 107, "training_dataload": 107, "drop_last": 107, "crit": 107, "crossentropyloss": 107, "calibrate_loop": 107, "pred": 107, "5f": 107, "acc": 107, "2f": 107, "quantize_typ": 107, "quant_cfg": 107, "int8_default_cfg": 107, "fp8_default_cfg": 107, "forward_loop": 107, "qdq": 107, "incomplet": 107, "functionaltensor": 107, "functionaltensormod": 107, "_trace": 107, "_export": 107, "float8_e4m3fn": 107, "class_prob": 107, "class_pr": 107, "test_prob": 107, "test_pr": 107, "test_loss": 107, "test_acc": 107, "vgg16_ptq": 107, "overcom": 108, "throughput": 108, "sometim": [108, 112], "outweigh": 108, "slowdown": 108, "hardwar": [108, 117], "experi": 108, "balanc": 108, "timeit": 108, "time_gener": 108, "output_seq_length": 108, "seq_len": [108, 112], "llm": 108, "input_seq": 108, "start_tim": 108, "default_tim": 108, "inputs_copi": 108, "decod": 108, "logit": 108, "next_token_logit": 108, "next_token": 108, "end_tim": 108, "time_mean_m": 108, "isl": 108, "osl": 108, "warm": 108, "solut": 108, "insight": 108, "weight_streaming_ctx": 108, "weight_stream": 108, "mean_lat": 108, "percentag": 108, "weight_budget_pct": 108, "device_budget": 108, "total_device_budget": 108, "permiss": 108, "equal": 108, "proportion": 108, "streamabl": 108, "streamable_budget": 108, "requested_budget": 108, "get_automatic_weight_streaming_budget": 108, "weight_streaming_exampl": 108, "_rendered_examples_python": 109, "_rendered_examples_jupyt": 109, "acoust": 110, "speech": 110, "quartznet": 110, "contextnet": 110, "subword": 110, "piec": 110, "excit": 110, "se": 110, "audio": 110, "transcrib": 110, "speedup": 110, "obtain": [110, 115], "feedforward": 110, "cnn": 110, "uniformli": 110, "resolut": 110, "highli": [110, 111], "compound": 110, "coeffici": 110, "b0": 110, "corpu": 110, "english": 110, "supervis": 110, "walkthrough": 110, "overal": 110, "adopt": 110, "mobilenetv2": 110, "classif": 110, "imagenet": 110, "imagenett": 110, "qat": 110, "simul": 110, "hand": 111, "consider": 111, "concurr": 111, "grpc": 111, "aforement": 111, "familiar": 111, "resnet50": 111, "torchhub": 111, "docker": 111, "login": 111, "xx": 111, "yy": 111, "mm": 111, "publish": 111, "22": 111, "pwd": 111, "scratch_spac": 111, "nvcr": 111, "py3": 111, "proce": 111, "hub": 111, "_validate_not_a_forked_repo": 111, "suggest": 111, "simplest": 111, "model_repositori": 111, "pbtxt": 111, "pytorch_libtorch": 111, "input__0": 111, "data_typ": 111, "type_fp32": 111, "output__0": 111, "exact": 111, "encourag": 111, "rm": 111, "8000": 111, "8001": 111, "8002": 111, "the_model_repositori": 111, "tritonserv": 111, "spin": 111, "proceed": 111, "flesh": 111, "img1": 111, "hakaimagazin": 111, "wp": 111, "gulf": 111, "bird": 111, "attrdict": 111, "pyindex": 111, "tritoncli": 111, "jump": 111, "firstli": 111, "resiz": 111, "pil": 111, "httpclient": 111, "triton_to_np_dtyp": 111, "rn50_preprocess": 111, "img_path": 111, "img": 111, "centercrop": 111, "485": 111, "456": 111, "406": 111, "229": 111, "transformed_img": 111, "inferenceservercli": 111, "localhost": 111, "secondli": 111, "inferinput": 111, "set_data_from_numpi": 111, "binary_data": 111, "inferrequestedoutput": 111, "class_count": 111, "lastli": 111, "send": 111, "model_nam": 111, "inference_output": 111, "as_numpi": 111, "468750": 111, "90": 111, "523438": 111, "92": 111, "664062": 111, "429688": 111, "136": 111, "234375": 111, "confidence_scor": 111, "classification_index": 111, "eagerli": 112, "swap": 112, "exactli": 112, "_tracer": 112, "queri": 112, "attn_weight": 112, "compiler_dynamic_shap": 112, "inputs_bs2": 112, "mymodul": 113, "linear1": 113, "linear2": 113, "linear3": 113, "40": 113, "__myl_mulsum_myl0_0": 113, "layertyp": 113, "kgen": 113, "__mye116_dconst": 113, "__myln_k_arg__bb1_2": 113, "tacticnam": 113, "__myl_mulsum_0xfa6c1858aea1b13b03f90165d7149ec6": 113, "streamid": 113, "__myl_addresmulsum_myl0_1": 113, "__mye131_dconst": 113, "addmm_constant_0": 113, "addmm_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 113, "__myln_k_arg__bb1_3": 113, "__myl_addresmulsum_0xb3915d7ebfe48be45b6d49083479e12f": 113, "__myl_addresmulsumadd_myl0_2": 113, "__mye146_dconst": 113, "addmm_2_constant_0": 113, "addmm_2_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 113, "addmm_1_constant_0": 113, "addmm_1_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 113, "__myl_addresmulsumadd_0xcdd0085ad25f5f45ac5fafb72acbffd6": 113, "__myl_mulsumaddcas_myl0_0": 113, "__mye112_dconst": 113, "__myl_mulsumaddcas_0xacf8f5dd9be2f3e7bb09cdddeac6c936": 113, "__myl_resmulsumaddcas_myl0_1": 113, "__mye127_dconst": 113, "addmm_1_add_broadcast_to_same_shape_lhs_broadcast_constanthalf": 113, "__myl_resmulsumaddcas_0x5a3b318b5a1c97b7d5110c0291481337": 113, "__myl_resmulsumadd_myl0_2": 113, "__mye142_dconst": 113, "__myl_resmulsumadd_0x3fad91127c640fd6db771aa9cde67db0": 113, "libtorchtrt_runtim": 114, "dl_open": 114, "ld_preload": 114, "load_librari": 114, "wl": 114, "ltorchtrt": 114, "torchtrt_runtime_exampl": 114, "libtorchtrt_plugin": 114, "neglig": 114, "thread": 114, "alert": 114, "switch": 114, "mismatch": 114, "crash": 114, "sacrif": 114, "incur": 114, "intens": 114, "trt_ep": 115, "stai": 115, "trt_t": 115, "ergonom": 116, "deleg": 116, "believ": 116, "amen": 116, "artifact": 116, "pack": 116, "year": 116, "superset": 116, "codebas": 116, "immedi": 116, "traceabl": 116, "scriptabl": 116, "neural": 117, "deconvolut": 117, "scripted_model": 117}, "objects": {"": [[5, 0, 1, "c.STR", "STR"], [9, 0, 1, "c.TORCHTRT_API", "TORCHTRT_API"], [11, 0, 1, "c.TORCHTRT_HIDDEN", "TORCHTRT_HIDDEN"], [7, 0, 1, "c.TORCH_TENSORRT_MAJOR_VERSION", "TORCH_TENSORRT_MAJOR_VERSION"], [8, 0, 1, "c.TORCH_TENSORRT_MINOR_VERSION", "TORCH_TENSORRT_MINOR_VERSION"], [6, 0, 1, "c.TORCH_TENSORRT_PATCH_VERSION", "TORCH_TENSORRT_PATCH_VERSION"], [12, 0, 1, "c.TORCH_TENSORRT_VERSION", "TORCH_TENSORRT_VERSION"], [10, 0, 1, "c.XSTR", "XSTR"], [0, 1, 1, "_CPPv4N14torch_tensorrt8DataTypeE", "torch_tensorrt::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value", "torch_tensorrt::DataType::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE", "torch_tensorrt::DataType::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEv", "torch_tensorrt::DataType::DataType"], [0, 3, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value", "torch_tensorrt::DataType::DataType::t"], [0, 3, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE", "torch_tensorrt::DataType::DataType::t"], [0, 4, 1, "_CPPv4N14torch_tensorrt8DataType5ValueE", "torch_tensorrt::DataType::Value"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kBoolE", "torch_tensorrt::DataType::Value::kBool"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kCharE", "torch_tensorrt::DataType::Value::kChar"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value7kDoubleE", "torch_tensorrt::DataType::Value::kDouble"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value6kFloatE", "torch_tensorrt::DataType::Value::kFloat"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kHalfE", "torch_tensorrt::DataType::Value::kHalf"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value4kIntE", "torch_tensorrt::DataType::Value::kInt"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kLongE", "torch_tensorrt::DataType::Value::kLong"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE", "torch_tensorrt::DataType::Value::kUnknown"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kBoolE", "torch_tensorrt::DataType::kBool"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kCharE", "torch_tensorrt::DataType::kChar"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value7kDoubleE", "torch_tensorrt::DataType::kDouble"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value6kFloatE", "torch_tensorrt::DataType::kFloat"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kHalfE", "torch_tensorrt::DataType::kHalf"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value4kIntE", "torch_tensorrt::DataType::kInt"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kLongE", "torch_tensorrt::DataType::kLong"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE", "torch_tensorrt::DataType::kUnknown"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypecv5ValueEv", "torch_tensorrt::DataType::operator Value"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataTypecvbEv", "torch_tensorrt::DataType::operator bool"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeneE8DataType", "torch_tensorrt::DataType::operator!="], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE", "torch_tensorrt::DataType::operator!="], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeneE8DataType", "torch_tensorrt::DataType::operator!=::other"], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE", "torch_tensorrt::DataType::operator!=::other"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType", "torch_tensorrt::DataType::operator=="], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE", "torch_tensorrt::DataType::operator=="], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType", "torch_tensorrt::DataType::operator==::other"], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE", "torch_tensorrt::DataType::operator==::other"], [46, 1, 1, "_CPPv4N14torch_tensorrt6DeviceE", "torch_tensorrt::Device"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device6DeviceEv", "torch_tensorrt::Device::Device"], [1, 1, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypeE", "torch_tensorrt::Device::DeviceType"], [46, 1, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypeE", "torch_tensorrt::Device::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [1, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [46, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [46, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [1, 4, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE", "torch_tensorrt::Device::DeviceType::Value"], [46, 4, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE", "torch_tensorrt::Device::DeviceType::Value"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::Value::kDLA"], [46, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::Value::kDLA"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::Value::kGPU"], [46, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::Value::kGPU"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::kDLA"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::kGPU"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv", "torch_tensorrt::Device::DeviceType::operator Value"], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv", "torch_tensorrt::Device::DeviceType::operator Value"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv", "torch_tensorrt::Device::DeviceType::operator bool"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv", "torch_tensorrt::Device::DeviceType::operator bool"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!="], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!="], [1, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!=::other"], [46, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!=::other"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator=="], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator=="], [1, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator==::other"], [46, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator==::other"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device18allow_gpu_fallbackE", "torch_tensorrt::Device::allow_gpu_fallback"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device11device_typeE", "torch_tensorrt::Device::device_type"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device8dla_coreE", "torch_tensorrt::Device::dla_core"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device6gpu_idE", "torch_tensorrt::Device::gpu_id"], [17, 4, 1, "_CPPv4N14torch_tensorrt16EngineCapabilityE", "torch_tensorrt::EngineCapability"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE", "torch_tensorrt::EngineCapability::kDLA_STANDALONE"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE", "torch_tensorrt::EngineCapability::kSAFETY"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE", "torch_tensorrt::EngineCapability::kSTANDARD"], [47, 1, 1, "_CPPv4N14torch_tensorrt11GraphInputsE", "torch_tensorrt::GraphInputs"], [47, 6, 1, "_CPPv4N14torch_tensorrt11GraphInputs15input_signatureE", "torch_tensorrt::GraphInputs::input_signature"], [47, 6, 1, "_CPPv4N14torch_tensorrt11GraphInputs6inputsE", "torch_tensorrt::GraphInputs::inputs"], [48, 1, 1, "_CPPv4N14torch_tensorrt5InputE", "torch_tensorrt::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEv", "torch_tensorrt::Input::Input"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE", "torch_tensorrt::Input::Input::tensor"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input5dtypeE", "torch_tensorrt::Input::dtype"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input6formatE", "torch_tensorrt::Input::format"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9max_shapeE", "torch_tensorrt::Input::max_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9min_shapeE", "torch_tensorrt::Input::min_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9opt_shapeE", "torch_tensorrt::Input::opt_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input5shapeE", "torch_tensorrt::Input::shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input13tensor_domainE", "torch_tensorrt::Input::tensor_domain"], [2, 1, 1, "_CPPv4N14torch_tensorrt12TensorFormatE", "torch_tensorrt::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEv", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 3, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value", "torch_tensorrt::TensorFormat::TensorFormat::t"], [2, 3, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE", "torch_tensorrt::TensorFormat::TensorFormat::t"], [2, 4, 1, "_CPPv4N14torch_tensorrt12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::Value"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE", "torch_tensorrt::TensorFormat::Value::kChannelsLast"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE", "torch_tensorrt::TensorFormat::Value::kContiguous"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE", "torch_tensorrt::TensorFormat::Value::kUnknown"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE", "torch_tensorrt::TensorFormat::kChannelsLast"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE", "torch_tensorrt::TensorFormat::kContiguous"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE", "torch_tensorrt::TensorFormat::kUnknown"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatcv5ValueEv", "torch_tensorrt::TensorFormat::operator Value"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormatcvbEv", "torch_tensorrt::TensorFormat::operator bool"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat", "torch_tensorrt::TensorFormat::operator!="], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator!="], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat", "torch_tensorrt::TensorFormat::operator!=::other"], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator!=::other"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat", "torch_tensorrt::TensorFormat::operator=="], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator=="], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat", "torch_tensorrt::TensorFormat::operator==::other"], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator==::other"], [37, 2, 1, "_CPPv4N14torch_tensorrt15dump_build_infoEv", "torch_tensorrt::dump_build_info"], [35, 2, 1, "_CPPv4N14torch_tensorrt14get_build_infoEv", "torch_tensorrt::get_build_info"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE", "torch_tensorrt::kDLA_STANDALONE"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE", "torch_tensorrt::kSAFETY"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE", "torch_tensorrt::kSTANDARD"], [16, 4, 1, "_CPPv4N14torch_tensorrt7logging5LevelE", "torch_tensorrt::logging::Level"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE", "torch_tensorrt::logging::Level::kDEBUG"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kERRORE", "torch_tensorrt::logging::Level::kERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE", "torch_tensorrt::logging::Level::kGRAPH"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level5kINFOE", "torch_tensorrt::logging::Level::kINFO"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE", "torch_tensorrt::logging::Level::kINTERNAL_ERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE", "torch_tensorrt::logging::Level::kWARNING"], [24, 2, 1, "_CPPv4N14torch_tensorrt7logging24get_is_colored_output_onEv", "torch_tensorrt::logging::get_is_colored_output_on"], [22, 2, 1, "_CPPv4N14torch_tensorrt7logging18get_logging_prefixEv", "torch_tensorrt::logging::get_logging_prefix"], [23, 2, 1, "_CPPv4N14torch_tensorrt7logging24get_reportable_log_levelEv", "torch_tensorrt::logging::get_reportable_log_level"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE", "torch_tensorrt::logging::kDEBUG"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kERRORE", "torch_tensorrt::logging::kERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE", "torch_tensorrt::logging::kGRAPH"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level5kINFOE", "torch_tensorrt::logging::kINFO"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE", "torch_tensorrt::logging::kINTERNAL_ERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE", "torch_tensorrt::logging::kWARNING"], [26, 2, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log"], [26, 3, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log::lvl"], [26, 3, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log::msg"], [27, 2, 1, "_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb", "torch_tensorrt::logging::set_is_colored_output_on"], [27, 3, 1, "_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb", "torch_tensorrt::logging::set_is_colored_output_on::colored_output_on"], [28, 2, 1, "_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE", "torch_tensorrt::logging::set_logging_prefix"], [28, 3, 1, "_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE", "torch_tensorrt::logging::set_logging_prefix::prefix"], [25, 2, 1, "_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level", "torch_tensorrt::logging::set_reportable_log_level"], [25, 3, 1, "_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level", "torch_tensorrt::logging::set_reportable_log_level::lvl"], [3, 1, 1, "_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE", "torch_tensorrt::ptq::Int8CacheCalibrator"], [3, 7, 1, "_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE", "torch_tensorrt::ptq::Int8CacheCalibrator::Algorithm"], [3, 2, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE", "torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator"], [3, 3, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE", "torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator::cache_file_path"], [3, 2, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibratorcvPN8nvinfer115IInt8CalibratorEEv", "torch_tensorrt::ptq::Int8CacheCalibrator::operator nvinfer1::IInt8Calibrator*"], [4, 1, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator"], [4, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator::Algorithm"], [4, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator::DataLoaderUniquePtr"], [4, 2, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::cache_file_path"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::dataloader"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::use_cache"], [4, 2, 1, "_CPPv4N14torch_tensorrt3ptq14Int8CalibratorcvPN8nvinfer115IInt8CalibratorEEv", "torch_tensorrt::ptq::Int8Calibrator::operator nvinfer1::IInt8Calibrator*"], [29, 2, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator"], [29, 7, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator::Algorithm"], [29, 3, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator::cache_file_path"], [30, 2, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator"], [30, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::Algorithm"], [30, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::DataLoader"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::cache_file_path"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::dataloader"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::use_cache"], [36, 2, 1, "_CPPv4N14torch_tensorrt10set_deviceEKi", "torch_tensorrt::set_device"], [36, 3, 1, "_CPPv4N14torch_tensorrt10set_deviceEKi", "torch_tensorrt::set_device::gpu_id"], [49, 1, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpecE", "torch_tensorrt::torchscript::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::input_signature"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::inputs"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec19allow_shape_tensorsE", "torch_tensorrt::torchscript::CompileSpec::allow_shape_tensors"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec10capabilityE", "torch_tensorrt::torchscript::CompileSpec::capability"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec5debugE", "torch_tensorrt::torchscript::CompileSpec::debug"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec6deviceE", "torch_tensorrt::torchscript::CompileSpec::device"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec12disable_tf32E", "torch_tensorrt::torchscript::CompileSpec::disable_tf32"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec20dla_global_dram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_global_dram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec19dla_local_dram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_local_dram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec13dla_sram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_sram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec18enabled_precisionsE", "torch_tensorrt::torchscript::CompileSpec::enabled_precisions"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec12graph_inputsE", "torch_tensorrt::torchscript::CompileSpec::graph_inputs"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14min_block_sizeE", "torch_tensorrt::torchscript::CompileSpec::min_block_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec20num_avg_timing_itersE", "torch_tensorrt::torchscript::CompileSpec::num_avg_timing_iters"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14ptq_calibratorE", "torch_tensorrt::torchscript::CompileSpec::ptq_calibrator"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec5refitE", "torch_tensorrt::torchscript::CompileSpec::refit"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec24require_full_compilationE", "torch_tensorrt::torchscript::CompileSpec::require_full_compilation"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14sparse_weightsE", "torch_tensorrt::torchscript::CompileSpec::sparse_weights"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec22torch_executed_modulesE", "torch_tensorrt::torchscript::CompileSpec::torch_executed_modules"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec18torch_executed_opsE", "torch_tensorrt::torchscript::CompileSpec::torch_executed_ops"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec24truncate_long_and_doubleE", "torch_tensorrt::torchscript::CompileSpec::truncate_long_and_double"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14workspace_sizeE", "torch_tensorrt::torchscript::CompileSpec::workspace_size"], [31, 2, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support"], [31, 3, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support::method_name"], [31, 3, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support::module"], [32, 2, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile"], [32, 3, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile::info"], [32, 3, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile::module"], [34, 2, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::info"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::method_name"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::module"], [33, 2, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::device"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::engine"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::input_binding_names"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::output_binding_names"], [76, 8, 0, "-", "torch_tensorrt"]], "torch_tensorrt": [[76, 9, 1, "", "Device"], [76, 9, 1, "", "DeviceType"], [76, 9, 1, "", "EngineCapability"], [76, 9, 1, "", "Input"], [76, 9, 1, "", "MutableTorchTensorRTModule"], [76, 12, 1, "", "compile"], [76, 12, 1, "", "convert_method_to_trt_engine"], [76, 9, 1, "", "dtype"], [115, 8, 0, "-", "dynamo"], [72, 8, 0, "-", "fx"], [76, 12, 1, "", "load"], [73, 8, 0, "-", "logging"], [76, 9, 1, "", "memory_format"], [75, 8, 0, "-", "runtime"], [76, 12, 1, "", "save"], [77, 8, 0, "-", "ts"]], "torch_tensorrt.Device": [[76, 10, 1, "", "__init__"], [76, 11, 1, "", "device_type"], [76, 11, 1, "", "dla_core"], [76, 11, 1, "", "gpu_id"]], "torch_tensorrt.DeviceType": [[76, 11, 1, "", "DLA"], [76, 11, 1, "", "GPU"], [76, 11, 1, "", "UNKNOWN"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.EngineCapability": [[76, 11, 1, "", "DLA_STANDALONE"], [76, 11, 1, "", "SAFETY"], [76, 11, 1, "", "STANDARD"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.Input": [[76, 10, 1, "", "__init__"], [76, 11, 1, "", "dtype"], [76, 10, 1, "", "example_tensor"], [76, 11, 1, "", "format"], [76, 10, 1, "", "from_tensor"], [76, 10, 1, "", "from_tensors"]], "torch_tensorrt.MutableTorchTensorRTModule": [[76, 10, 1, "", "__init__"], [76, 10, 1, "", "compile"], [76, 10, 1, "", "refit_gm"]], "torch_tensorrt.dtype": [[76, 11, 1, "", "b"], [76, 11, 1, "", "bf16"], [76, 11, 1, "", "f16"], [76, 11, 1, "", "f32"], [76, 11, 1, "", "f64"], [76, 11, 1, "", "f8"], [76, 11, 1, "", "i32"], [76, 11, 1, "", "i64"], [76, 11, 1, "", "i8"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"], [76, 11, 1, "", "u8"], [76, 11, 1, "", "unknown"]], "torch_tensorrt.dynamo": [[71, 9, 1, "", "CompilationSettings"], [71, 12, 1, "", "compile"], [71, 12, 1, "", "export"], [71, 12, 1, "", "refit_module_weights"], [71, 12, 1, "", "trace"]], "torch_tensorrt.fx": [[72, 9, 1, "", "InputTensorSpec"], [72, 9, 1, "", "TRTInterpreter"], [72, 9, 1, "", "TRTInterpreterResult"], [72, 9, 1, "", "TRTModule"], [72, 12, 1, "", "compile"]], "torch_tensorrt.logging": [[73, 9, 1, "", "debug"], [73, 9, 1, "", "errors"], [73, 9, 1, "", "graphs"], [73, 9, 1, "", "info"], [73, 9, 1, "", "internal_errors"], [73, 9, 1, "", "warnings"]], "torch_tensorrt.memory_format": [[76, 11, 1, "", "cdhw32"], [76, 11, 1, "", "chw16"], [76, 11, 1, "", "chw2"], [76, 11, 1, "", "chw32"], [76, 11, 1, "", "chw4"], [76, 11, 1, "", "dhwc"], [76, 11, 1, "", "dhwc8"], [76, 11, 1, "", "dla_hwc4"], [76, 11, 1, "", "dla_linear"], [76, 11, 1, "", "hwc"], [76, 11, 1, "", "hwc16"], [76, 11, 1, "", "hwc8"], [76, 11, 1, "", "linear"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.runtime": [[75, 9, 1, "", "PythonTorchTensorRTModule"], [75, 9, 1, "", "TorchTensorRTModule"], [75, 12, 1, "", "set_multi_device_safe_mode"]], "torch_tensorrt.runtime.PythonTorchTensorRTModule": [[75, 10, 1, "", "__init__"], [75, 10, 1, "", "cudagraphs_validate_shapes"], [75, 10, 1, "", "disable_profiling"], [75, 10, 1, "", "enable_profiling"], [75, 10, 1, "", "forward"], [75, 10, 1, "", "get_layer_info"]], "torch_tensorrt.runtime.TorchTensorRTModule": [[75, 10, 1, "", "__init__"], [75, 10, 1, "", "forward"], [75, 10, 1, "", "get_extra_state"], [75, 10, 1, "", "set_extra_state"]], "torch_tensorrt.ts": [[77, 12, 1, "", "TensorRTCompileSpec"], [77, 12, 1, "", "check_method_op_support"], [77, 12, 1, "", "compile"], [77, 12, 1, "", "convert_method_to_trt_engine"], [77, 12, 1, "", "embed_engine_in_new_module"], [74, 8, 0, "-", "ptq"]], "torch_tensorrt.ts.ptq": [[74, 9, 1, "", "CacheCalibrator"], [74, 9, 1, "", "CalibrationAlgo"], [74, 9, 1, "", "DataLoaderCalibrator"]], "torch_tensorrt.ts.ptq.CalibrationAlgo": [[74, 11, 1, "", "ENTROPY_CALIBRATION"], [74, 11, 1, "", "ENTROPY_CALIBRATION_2"], [74, 11, 1, "", "LEGACY_CALIBRATION"], [74, 11, 1, "", "MINMAX_CALIBRATION"]]}, "objtypes": {"0": "c:macro", "1": "cpp:class", "2": "cpp:function", "3": "cpp:functionParam", "4": "cpp:enum", "5": "cpp:enumerator", "6": "cpp:member", "7": "cpp:templateParam", "8": "py:module", "9": "py:class", "10": "py:method", "11": "py:attribute", "12": "py:function"}, "objnames": {"0": ["c", "macro", "C macro"], "1": ["cpp", "class", "C++ class"], "2": ["cpp", "function", "C++ function"], "3": ["cpp", "functionParam", "C++ function parameter"], "4": ["cpp", "enum", "C++ enum"], "5": ["cpp", "enumerator", "C++ enumerator"], "6": ["cpp", "member", "C++ member"], "7": ["cpp", "templateParam", "C++ template parameter"], "8": ["py", "module", "Python module"], "9": ["py", "class", "Python class"], "10": ["py", "method", "Python method"], "11": ["py", "attribute", "Python attribute"], "12": ["py", "function", "Python function"]}, "titleterms": {"class": [0, 1, 2, 3, 4, 20, 21, 38, 40, 41, 50, 71, 72, 74, 75, 76], "datatyp": 0, "document": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 46, 47, 48, 49, 61, 69, 85, 86], "devic": [1, 46, 114], "devicetyp": 1, "nest": [1, 46], "relationship": [1, 3, 4, 46, 48], "tensorformat": 2, "templat": [3, 4, 29, 30], "int8cachecalibr": 3, "inherit": [3, 4, 48], "base": [3, 4, 48, 80], "type": [3, 4, 46, 48, 54], "int8calibr": 4, "defin": [5, 6, 7, 8, 9, 10, 11, 12, 19, 50, 107], "str": 5, "torch_tensorrt_patch_vers": 6, "torch_tensorrt_major_vers": 7, "torch_tensorrt_minor_vers": 8, "torchtrt_api": 9, "xstr": 10, "torchtrt_hidden": 11, "torch_tensorrt_vers": 12, "directori": [13, 14, 15, 51], "cpp": [13, 18, 19, 20, 21, 56], "subdirectori": [13, 14], "includ": [14, 18, 19, 20, 21], "torch_tensorrt": [15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 45, 67, 71, 72, 73, 74, 75, 76, 77, 101, 103, 104, 116], "file": [15, 18, 19, 20, 21, 42, 43, 44, 45, 50, 51], "enum": [16, 17, 38, 39, 50, 74, 76], "level": [16, 80, 82, 83], "enginecap": 17, "log": [18, 22, 23, 24, 25, 26, 27, 28, 39, 42, 73], "h": [18, 19, 20, 21, 42, 43, 44, 45, 56], "content": [18, 19, 20, 21, 38, 39, 40, 41, 80, 81, 82, 83, 84, 85], "definit": [18, 19, 20, 21, 83, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "By": [18, 19], "namespac": [18, 19, 20, 21, 38, 39, 40, 41, 50], "macro": [19, 43], "ptq": [20, 29, 30, 40, 44, 74, 91, 107], "function": [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 50, 61, 71, 72, 75, 76, 77, 107], "get_logging_prefix": 22, "get_reportable_log_level": 23, "get_is_colored_output_on": 24, "set_reportable_log_level": 25, "set_is_colored_output_on": 27, "set_logging_prefix": 28, "make_int8_cache_calibr": 29, "make_int8_calibr": 30, "torchscript": [31, 32, 33, 34, 41, 60, 66, 69, 88, 89, 92, 115, 116], "check_method_operator_support": 31, "compil": [32, 57, 59, 63, 64, 66, 68, 69, 89, 96, 99, 100, 101, 102, 103, 104, 105, 106, 108, 110, 112, 113, 115, 116], "embed_engine_in_new_modul": 33, "convert_method_to_trt_engin": 34, "get_build_info": 35, "set_devic": 36, "dump_build_info": 37, "program": [42, 43, 44, 45, 63, 99, 114], "list": [42, 43, 44, 45, 83], "struct": [46, 47, 48, 49, 50], "graphinput": 47, "input": [48, 101, 103], "compilespec": 49, "torch": [50, 61, 63, 64, 65, 66, 68, 69, 89, 90, 92, 93, 94, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 109, 110, 111, 112, 113, 114, 115, 116], "tensorrt": [50, 58, 61, 63, 64, 65, 66, 69, 89, 90, 92, 93, 94, 98, 99, 105, 106, 107, 109, 110, 111, 112, 113, 114, 115, 116], "c": [50, 61, 66, 68, 69, 89, 91, 110], "api": [50, 51, 61, 66, 69], "hierarchi": 50, "full": [50, 51], "torchtrtc": [52, 89], "convers": [53, 57, 59, 60], "phase": [53, 55, 56, 57, 58, 59], "node": 53, "evalu": [53, 54, 70], "convert": [53, 54, 60, 65, 70, 89, 93], "write": [54, 60, 62, 93, 94], "dynamo": [54, 62, 69, 71, 105, 106, 115, 116], "implement": [54, 93], "registr": 54, "capabl": 54, "valid": 54, "contract": [54, 60], "exampl": [54, 62, 82, 84], "convolut": 54, "oper": [54, 64, 70, 89, 94], "decomposit": 54, "addmm": [54, 55], "lower": [55, 57, 59, 62], "pass": [55, 62], "us": [55, 61, 89, 90, 92, 93, 94, 100, 101, 102, 103, 104, 105, 106, 107, 110, 112], "eliminatecommonsubexpress": 55, "elimin": 55, "dead": 55, "code": [55, 69, 82], "except": 55, "Or": 55, "pattern": 55, "redund": 55, "guard": 55, "freez": 55, "modul": [55, 88, 89, 98, 116], "fuse": 55, "branch": 55, "linear": 55, "flatten": 55, "graph": [55, 58, 116], "tupl": 55, "fallback": [55, 56], "peephol": 55, "optim": [55, 68, 111], "remov": 55, "contigu": 55, "dropout": 55, "To": 55, "unpack": 55, "logsoftmax": 55, "unrol": 55, "loop": [55, 107], "replac": [55, 82], "tile": 55, "repeat": 55, "partit": [56, 57, 59], "partitoninfo": 56, "segmentedblock": 56, "shape_analysi": 56, "automat": [56, 108], "depend": [56, 66, 97, 109], "awar": [56, 110], "runtim": [57, 58, 59, 75, 114], "background": [58, 60], "engin": [58, 65, 94, 95, 96], "executor": 58, "op": [58, 65, 94], "construct": 58, "result": 58, "serial": [58, 64, 68], "deseri": 58, "abi": [58, 66], "version": [58, 66], "format": [58, 116], "system": [59, 66], "overview": [59, 67], "what": 60, "guarante": 60, "respons": 60, "context": [60, 80, 108], "arg": [60, 81], "weight": [60, 99, 107, 108], "other": 60, "advic": 60, "link": [61, 82], "develop": 61, "avail": 61, "layer": 61, "expect": 61, "dimens": 61, "python": [61, 66, 68, 69, 88, 90, 91], "sometim": 61, "easier": 61, "read": 61, "pytorch": [61, 65, 69, 92, 94, 105, 106, 110], "native_op": 61, "ir": [61, 115, 116], "aten": 62, "basic": 62, "requir": 62, "regist": [62, 89], "export": [63, 68, 104, 112], "customiz": [63, 64], "set": [63, 64, 98, 100, 104, 111], "under": [63, 89, 112], "hood": [63, 89, 112], "trace": 63, "backend": [64, 101, 102, 103, 105, 106], "kei": 64, "featur": 64, "custom": [64, 89, 93, 94, 96, 100, 112], "usag": [64, 99, 100], "after": 64, "model": [64, 65, 69, 94, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 113, 115], "perform": 64, "coverag": 64, "feasibl": 64, "dynam": [64, 101, 110, 112], "shape": [64, 101, 110, 112], "support": [64, 70], "recompil": [64, 101], "condit": 64, "fx": [65, 69, 72, 110, 116], "frontend": [65, 66, 69, 92, 110, 116], "user": [65, 69], "guid": [65, 69], "acc": 65, "tracer": 65, "fx2trt": 65, "how": [65, 80, 91], "add": 65, "miss": 65, "instal": [66, 87], "precompil": 66, "binari": 66, "specif": 66, "cuda": [66, 100, 103], "nightli": 66, "build": [66, 67, 80, 111], "onli": 66, "from": [66, 92], "sourc": 66, "linux": 66, "packag": [66, 114], "addit": 66, "option": [66, 68, 80, 81, 83, 101, 103, 108, 116], "distribut": 66, "No": 66, "librari": [66, 114], "standalon": 66, "releas": 66, "debug": 66, "pre": [66, 107], "cxx11": 66, "choos": 66, "right": 66, "window": 66, "step": [66, 68, 111], "advanc": [66, 99, 100], "setup": 66, "troubleshoot": 66, "altern": 66, "cmake": 66, "nativ": 66, "aarch64": 66, "jetson": 66, "prerequisit": [66, 67], "environ": 66, "cli": [66, 69], "jetpack": 67, "6": [67, 84], "1": [67, 68, 84, 111], "quick": 68, "start": [68, 69], "2": [68, 84, 85, 111], "deploi": [68, 107, 110, 114], "deploy": 68, "In": [69, 99], "framework": 69, "infer": [69, 101, 102, 103, 104, 107, 111], "nvidia": 69, "gpu": 69, "get": 69, "tutori": [69, 109], "zoo": [69, 97, 109], "contributor": 69, "indic": 69, "legaci": [69, 110, 116], "further": 69, "inform": 69, "current": 70, "through": 70, "ts": [74, 77, 116], "submodul": 76, "comput": 78, "time": [78, 116], "changelog": 79, "configur": 80, "project": 80, "wide": 80, "html": 80, "theme": [80, 86], "toc": 80, "page": 80, "tabl": [80, 81, 82, 83, 84, 85], "mod": 81, "test_py_modul": 81, "gener": [81, 105, 106], "index": 81, "paramet": 81, "data": 81, "paragraph": [82, 85], "markup": 82, "inlin": 82, "math": 82, "meta": 82, "block": 82, "liter": 82, "line": 82, "quot": 82, "doctest": 82, "emphas": 82, "number": [82, 83], "sidebar": 82, "ch": 82, "ien": 82, "The": [82, 89], "creativ": 82, "A": 82, "refer": 82, "footnot": 82, "citat": [82, 91], "glossari": 82, "target": 82, "direct": 82, "center": 82, "text": 82, "imag": [82, 83], "figur": 82, "admonit": 82, "And": 82, "wai": 82, "topic": 82, "rubric": 82, "titl": 82, "compound": 82, "download": [82, 87], "enumer": 83, "field": 83, "bullet": 83, "second": 83, "But": 83, "deeper": 83, "down": 83, "rabbit": 83, "hole": 83, "hlist": 83, "grid": 83, "giant": 83, "can": 83, "have": 83, "caption": [83, 86], "like": 83, "thi": [83, 86], "one": 83, "long": [84, 86], "sticki": 84, "nav": 84, "menu": [84, 86], "3": [84, 111], "4": 84, "5": 84, "7": 84, "8": 84, "9": 84, "10": 84, "11": 84, "12": 84, "13": 84, "14": 84, "15": 84, "16": 84, "17": 84, "18": 84, "19": 84, "20": 84, "submenu": 84, "subsubmenu": 84, "structur": 85, "element": 85, "section": 85, "subsect": 85, "subsubsect": 85, "demo": 86, "an": 86, "incred": 86, "via": 87, "git": 87, "creat": [88, 91], "work": [88, 89], "save": [88, 98, 115], "disk": 88, "quickstart": 89, "unsupport": 89, "post": 91, "train": [91, 107, 110], "quantiz": [91, 107, 110], "your": [91, 111], "own": 91, "applic": 91, "directli": 92, "overload": 93, "metadata": 93, "our": [93, 94], "kernel": 94, "within": 94, "test": 94, "wrap": 94, "insert": 94, "cach": [95, 96, 99], "bert": [95, 103, 110], "jit": [96, 112], "aot": [96, 112], "mutabl": 98, "initi": 98, "make": [98, 99], "modif": 98, "stabl": [98, 102], "diffus": [98, 102], "huggingfac": 98, "refit": 99, "new": 99, "standard": 99, "workflow": 99, "import": [99, 100, 101, 102, 103, 104, 105, 106, 107, 108], "refitt": 99, "pretrain": 99, "map": 99, "place": 99, "default": [100, 104], "cleanup": [100, 103], "driver": [100, 103], "error": [100, 103], "note": [100, 103], "resnet": 101, "argument": [101, 103], "avoid": 101, "specifi": 101, "befor": 101, "trt": 101, "cudagraph": [104, 114], "integr": 104, "gpt2": 105, "output": [105, 106], "decod": [105, 106], "sentenc": [105, 106], "llama2": 106, "load": [107, 115], "dataset": 107, "loss": 107, "calibr": 107, "tune": 107, "fp8": 107, "stream": 108, "run": 108, "budget": 108, "size": 108, "manag": 108, "notebook": 110, "citrinet": 110, "efficientnet": 110, "mask": 110, "languag": 110, "mlm": 110, "hug": 110, "face": 110, "transform": 110, "acceler": 110, "serv": [110, 111], "resnet50": 110, "lenet": 110, "deep": 110, "learn": 110, "object": 110, "detect": 110, "ssd": 110, "int8": 110, "triton": 111, "up": 111, "server": 111, "client": 111, "queri": 111, "constraint": 112, "mix": 113, "precis": 113, "libtorchtrt": 114, "so": 114, "plugin": 114, "multi": 114, "safe": 114, "mode": 114, "exportedprogram": 115, "b": 115, "explain": 116, "just": 116, "accept": 116, "return": 116, "ahead": 116, "dla": 117}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx.ext.intersphinx": 1, "sphinx.ext.todo": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}})
\ No newline at end of file
+Search.setIndex({"docnames": ["_cpp_api/classtorch__tensorrt_1_1DataType", "_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType", "_cpp_api/classtorch__tensorrt_1_1TensorFormat", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator", "_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502", "_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268", "_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e", "_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827", "_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b", "_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da", "_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59", "_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883", "_cpp_api/dir_cpp", "_cpp_api/dir_cpp_include", "_cpp_api/dir_cpp_include_torch_tensorrt", "_cpp_api/enum_namespacetorch__tensorrt_1_1logging_1a130f65408ad8cbaee060f05e8db69558", "_cpp_api/enum_namespacetorch__tensorrt_1a3fbe5d72e4fc624dbd038853079620eb", "_cpp_api/file_cpp_include_torch_tensorrt_logging.h", "_cpp_api/file_cpp_include_torch_tensorrt_macros.h", "_cpp_api/file_cpp_include_torch_tensorrt_ptq.h", "_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0593f776f469c20469e2f729fc7861a3", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0c012cb374addd90eb1f42eaec570650", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a56e110feaaba2c3fd44bd201fd21a76a", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a7cb50492421ea9de4e3db895819df6f2", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ac46ac0901cb97e3ae6e93b45f24e90b8", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ad2efd47b6c3689e58ccc595680579ae5", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1af8f3443813315af7901903d25dd495cc", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a226e3c83379d1012cde8578c1c86b16c", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a6186e305f47c1d94b6130ef6c7f7e178", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a5b405fd3bf3c8fc2e2a54cbbab979797", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a6e19490a08fb1553c9dd347a5ae79db9", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a81f9783517335dda877d8cfcf38987c9", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1ae8d56472106eeef37fbe51ff7f40c9b2", "_cpp_api/function_namespacetorch__tensorrt_1ac4ab8313ae72c2c899ea31548b528528", "_cpp_api/function_namespacetorch__tensorrt_1ad1acd06eaeaffbbcf6e7ebf426891384", "_cpp_api/function_namespacetorch__tensorrt_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1", "_cpp_api/namespace_torch_tensorrt", "_cpp_api/namespace_torch_tensorrt__logging", "_cpp_api/namespace_torch_tensorrt__ptq", "_cpp_api/namespace_torch_tensorrt__torchscript", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h", "_cpp_api/structtorch__tensorrt_1_1Device", "_cpp_api/structtorch__tensorrt_1_1GraphInputs", "_cpp_api/structtorch__tensorrt_1_1Input", "_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec", "_cpp_api/torch_tensort_cpp", "_cpp_api/unabridged_orphan", "cli/torchtrtc", "contributors/conversion", "contributors/dynamo_converters", "contributors/lowering", "contributors/partitioning", "contributors/phases", "contributors/runtime", "contributors/system_overview", "contributors/ts_converters", "contributors/useful_links", "contributors/writing_dynamo_aten_lowering_passes", "dynamo/dynamo_export", "dynamo/torch_compile", "fx/getting_started_with_fx_path", "getting_started/installation", "getting_started/jetpack", "getting_started/quick_start", "index", "indices/supported_ops", "py_api/dynamo", "py_api/fx", "py_api/logging", "py_api/ptq", "py_api/runtime", "py_api/torch_tensorrt", "py_api/ts", "sg_execution_times", "src/pytorch-sphinx-theme/docs/changelog", "src/pytorch-sphinx-theme/docs/configuring", "src/pytorch-sphinx-theme/docs/demo/api", "src/pytorch-sphinx-theme/docs/demo/demo", "src/pytorch-sphinx-theme/docs/demo/lists_tables", "src/pytorch-sphinx-theme/docs/demo/long", "src/pytorch-sphinx-theme/docs/demo/structure", "src/pytorch-sphinx-theme/docs/index", "src/pytorch-sphinx-theme/docs/installing", "ts/creating_torchscript_module_in_python", "ts/getting_started_with_cpp_api", "ts/getting_started_with_python_api", "ts/ptq", "ts/torchscript_frontend_from_pytorch", "tutorials/_rendered_examples/dynamo/converter_overloading", "tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows", "tutorials/_rendered_examples/dynamo/custom_kernel_plugins", "tutorials/_rendered_examples/dynamo/engine_caching_bert_example", "tutorials/_rendered_examples/dynamo/engine_caching_example", "tutorials/_rendered_examples/dynamo/index", "tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example", "tutorials/_rendered_examples/dynamo/refit_engine_example", "tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage", "tutorials/_rendered_examples/dynamo/torch_compile_resnet_example", "tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion", "tutorials/_rendered_examples/dynamo/torch_compile_transformers_example", "tutorials/_rendered_examples/dynamo/torch_export_cudagraphs", "tutorials/_rendered_examples/dynamo/torch_export_gpt2", "tutorials/_rendered_examples/dynamo/torch_export_llama2", "tutorials/_rendered_examples/dynamo/vgg16_ptq", "tutorials/_rendered_examples/dynamo/weight_streaming_example", "tutorials/_rendered_examples/index", "tutorials/notebooks", "tutorials/serving_torch_tensorrt_with_triton", "user_guide/dynamic_shapes", "user_guide/mixed_precision", "user_guide/runtime", "user_guide/saving_models", "user_guide/torch_tensorrt_explained", "user_guide/using_dla"], "filenames": ["_cpp_api/classtorch__tensorrt_1_1DataType.rst", "_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.rst", "_cpp_api/classtorch__tensorrt_1_1TensorFormat.rst", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.rst", "_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.rst", "_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.rst", "_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.rst", "_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.rst", "_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.rst", "_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.rst", "_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.rst", "_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.rst", "_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.rst", "_cpp_api/dir_cpp.rst", "_cpp_api/dir_cpp_include.rst", "_cpp_api/dir_cpp_include_torch_tensorrt.rst", "_cpp_api/enum_namespacetorch__tensorrt_1_1logging_1a130f65408ad8cbaee060f05e8db69558.rst", "_cpp_api/enum_namespacetorch__tensorrt_1a3fbe5d72e4fc624dbd038853079620eb.rst", "_cpp_api/file_cpp_include_torch_tensorrt_logging.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_macros.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_ptq.h.rst", "_cpp_api/file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0593f776f469c20469e2f729fc7861a3.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a0c012cb374addd90eb1f42eaec570650.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a56e110feaaba2c3fd44bd201fd21a76a.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1a7cb50492421ea9de4e3db895819df6f2.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ac46ac0901cb97e3ae6e93b45f24e90b8.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1ad2efd47b6c3689e58ccc595680579ae5.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1logging_1af8f3443813315af7901903d25dd495cc.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a226e3c83379d1012cde8578c1c86b16c.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1ptq_1a6186e305f47c1d94b6130ef6c7f7e178.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a5b405fd3bf3c8fc2e2a54cbbab979797.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a6e19490a08fb1553c9dd347a5ae79db9.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1a81f9783517335dda877d8cfcf38987c9.rst", "_cpp_api/function_namespacetorch__tensorrt_1_1torchscript_1ae8d56472106eeef37fbe51ff7f40c9b2.rst", "_cpp_api/function_namespacetorch__tensorrt_1ac4ab8313ae72c2c899ea31548b528528.rst", "_cpp_api/function_namespacetorch__tensorrt_1ad1acd06eaeaffbbcf6e7ebf426891384.rst", "_cpp_api/function_namespacetorch__tensorrt_1ad6a4ee8ca6c8f6e5519eb1128ec7f4a1.rst", "_cpp_api/namespace_torch_tensorrt.rst", "_cpp_api/namespace_torch_tensorrt__logging.rst", "_cpp_api/namespace_torch_tensorrt__ptq.rst", "_cpp_api/namespace_torch_tensorrt__torchscript.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_logging.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_macros.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_ptq.h.rst", "_cpp_api/program_listing_file_cpp_include_torch_tensorrt_torch_tensorrt.h.rst", "_cpp_api/structtorch__tensorrt_1_1Device.rst", "_cpp_api/structtorch__tensorrt_1_1GraphInputs.rst", "_cpp_api/structtorch__tensorrt_1_1Input.rst", "_cpp_api/structtorch__tensorrt_1_1torchscript_1_1CompileSpec.rst", "_cpp_api/torch_tensort_cpp.rst", "_cpp_api/unabridged_orphan.rst", "cli/torchtrtc.rst", "contributors/conversion.rst", "contributors/dynamo_converters.rst", "contributors/lowering.rst", "contributors/partitioning.rst", "contributors/phases.rst", "contributors/runtime.rst", "contributors/system_overview.rst", "contributors/ts_converters.rst", "contributors/useful_links.rst", "contributors/writing_dynamo_aten_lowering_passes.rst", "dynamo/dynamo_export.rst", "dynamo/torch_compile.rst", "fx/getting_started_with_fx_path.rst", "getting_started/installation.rst", "getting_started/jetpack.rst", "getting_started/quick_start.rst", "index.rst", "indices/supported_ops.rst", "py_api/dynamo.rst", "py_api/fx.rst", "py_api/logging.rst", "py_api/ptq.rst", "py_api/runtime.rst", "py_api/torch_tensorrt.rst", "py_api/ts.rst", "sg_execution_times.rst", "src/pytorch-sphinx-theme/docs/changelog.rst", "src/pytorch-sphinx-theme/docs/configuring.rst", "src/pytorch-sphinx-theme/docs/demo/api.rst", "src/pytorch-sphinx-theme/docs/demo/demo.rst", "src/pytorch-sphinx-theme/docs/demo/lists_tables.rst", "src/pytorch-sphinx-theme/docs/demo/long.rst", "src/pytorch-sphinx-theme/docs/demo/structure.rst", "src/pytorch-sphinx-theme/docs/index.rst", "src/pytorch-sphinx-theme/docs/installing.rst", "ts/creating_torchscript_module_in_python.rst", "ts/getting_started_with_cpp_api.rst", "ts/getting_started_with_python_api.rst", "ts/ptq.rst", "ts/torchscript_frontend_from_pytorch.rst", "tutorials/_rendered_examples/dynamo/converter_overloading.rst", "tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.rst", "tutorials/_rendered_examples/dynamo/custom_kernel_plugins.rst", "tutorials/_rendered_examples/dynamo/engine_caching_bert_example.rst", "tutorials/_rendered_examples/dynamo/engine_caching_example.rst", "tutorials/_rendered_examples/dynamo/index.rst", "tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.rst", "tutorials/_rendered_examples/dynamo/refit_engine_example.rst", "tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.rst", "tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.rst", "tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.rst", "tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.rst", "tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.rst", "tutorials/_rendered_examples/dynamo/torch_export_gpt2.rst", "tutorials/_rendered_examples/dynamo/torch_export_llama2.rst", "tutorials/_rendered_examples/dynamo/vgg16_ptq.rst", "tutorials/_rendered_examples/dynamo/weight_streaming_example.rst", "tutorials/_rendered_examples/index.rst", "tutorials/notebooks.rst", "tutorials/serving_torch_tensorrt_with_triton.rst", "user_guide/dynamic_shapes.rst", "user_guide/mixed_precision.rst", "user_guide/runtime.rst", "user_guide/saving_models.rst", "user_guide/torch_tensorrt_explained.rst", "user_guide/using_dla.rst"], "titles": ["Class DataType", "Class Device::DeviceType", "Class TensorFormat", "Template Class Int8CacheCalibrator", "Template Class Int8Calibrator", "Define STR", "Define TORCH_TENSORRT_PATCH_VERSION", "Define TORCH_TENSORRT_MAJOR_VERSION", "Define TORCH_TENSORRT_MINOR_VERSION", "Define TORCHTRT_API", "Define XSTR", "Define TORCHTRT_HIDDEN", "Define TORCH_TENSORRT_VERSION", "Directory cpp", "Directory include", "Directory torch_tensorrt", "Enum Level", "Enum EngineCapability", "File logging.h", "File macros.h", "File ptq.h", "File torch_tensorrt.h", "Function torch_tensorrt::logging::get_logging_prefix", "Function torch_tensorrt::logging::get_reportable_log_level", "Function torch_tensorrt::logging::get_is_colored_output_on", "Function torch_tensorrt::logging::set_reportable_log_level", "Function torch_tensorrt::logging::log", "Function torch_tensorrt::logging::set_is_colored_output_on", "Function torch_tensorrt::logging::set_logging_prefix", "Template Function torch_tensorrt::ptq::make_int8_cache_calibrator", "Template Function torch_tensorrt::ptq::make_int8_calibrator", "Function torch_tensorrt::torchscript::check_method_operator_support", "Function torch_tensorrt::torchscript::compile", "Function torch_tensorrt::torchscript::embed_engine_in_new_module", "Function torch_tensorrt::torchscript::convert_method_to_trt_engine", "Function torch_tensorrt::get_build_info", "Function torch_tensorrt::set_device", "Function torch_tensorrt::dump_build_info", "Namespace torch_tensorrt", "Namespace torch_tensorrt::logging", "Namespace torch_tensorrt::ptq", "Namespace torch_tensorrt::torchscript", "Program Listing for File logging.h", "Program Listing for File macros.h", "Program Listing for File ptq.h", "Program Listing for File torch_tensorrt.h", "Struct Device", "Struct GraphInputs", "Struct Input", "Struct CompileSpec", "Torch-TensorRT C++ API", "Full API", "torchtrtc", "Conversion Phase", "Writing Dynamo Converters", "Lowering Phase", "Partitioning Phase", "Compiler Phases", "Runtime Phase", "System Overview", "Writing TorchScript Converters", "Useful Links for Torch-TensorRT Development", "Writing Dynamo ATen Lowering Passes", "Compiling Exported Programs with Torch-TensorRT", "TensorRT Backend for torch.compile", "Torch-TensorRT (FX Frontend) User Guide", "Installation", "Overview", "Quick Start", "Torch-TensorRT", "Operators Supported", "torch_tensorrt.dynamo", "torch_tensorrt.fx", "torch_tensorrt.logging", "torch_tensorrt.ts.ptq", "torch_tensorrt.runtime", "torch_tensorrt", "torch_tensorrt.ts", "Computation times", "Changelog", "Configuration", "5. :mod:`test_py_module`", "3. Paragraph Level Markup", "4. Lists & Tables", "1. Long Sticky Nav", "1. Structural Elements", "<no title>", "Installation", "Creating a TorchScript Module", "Using Torch-TensorRT in C++", "Using Torch-TensorRT in Python", "Post Training Quantization (PTQ)", "Using Torch-TensorRT TorchScript Frontend Directly From PyTorch", "Overloading Torch-TensorRT Converters with Custom Converters", "Cross runtime compilation for windows example", "Using Custom Kernels within TensorRT Engines with Torch-TensorRT", "Engine Caching (BERT)", "Engine Caching", "Dependencies", "Mutable Torch TensorRT Module", "Refitting Torch-TensorRT Programs with New Weights", "Torch Compile Advanced Usage", "Compiling ResNet with dynamic shapes using the torch.compile backend", "Compiling Stable Diffusion model using the torch.compile backend", "Compiling BERT using the torch.compile backend", "Torch Export with Cudagraphs", "Compiling GPT2 using the dynamo backend", "Compiling Llama2 using the dynamo backend", "Deploy Quantized Models using Torch-TensorRT", "Weight Streaming", "Torch-TensorRT Tutorials", "Legacy notebooks", "Serving a Torch-TensorRT model with Triton", "Dynamic shapes with Torch-TensorRT", "Compile Mixed Precision models with Torch-TensorRT", "Deploying Torch-TensorRT Programs", "Saving models compiled with Torch-TensorRT", "Torch-TensorRT Explained", "DLA"], "terms": {"defin": [0, 1, 2, 3, 4, 33, 43, 46, 47, 48, 49, 51, 52, 54, 65, 68, 75, 76, 80, 88, 89, 90, 91, 93, 95, 97, 101, 104, 105, 106, 107, 111], "file": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 46, 47, 48, 49, 52, 54, 56, 58, 59, 64, 65, 66, 67, 68, 71, 72, 74, 76, 77, 78, 80, 81, 83, 87, 89, 91, 94, 112, 113, 116], "torch_tensorrt": [0, 1, 2, 14, 16, 17, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 54, 56, 62, 63, 64, 65, 68, 69, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 103, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 118], "h": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 46, 47, 48, 49, 50, 51, 52, 55, 68, 76, 89, 91], "support": [0, 1, 2, 27, 31, 46, 48, 49, 52, 54, 56, 61, 63, 65, 67, 68, 69, 72, 75, 76, 77, 80, 81, 88, 89, 90, 93, 95, 100, 102, 104, 106, 107, 108, 109, 112, 114, 117, 118], "data": [0, 2, 3, 4, 29, 30, 44, 46, 48, 49, 52, 53, 56, 57, 59, 60, 64, 65, 70, 71, 72, 74, 76, 77, 82, 86, 90, 91, 95, 97, 108, 109, 111], "type": [0, 1, 2, 30, 49, 50, 52, 53, 56, 58, 60, 62, 63, 64, 65, 71, 72, 74, 75, 76, 77, 82, 89, 90, 91, 93, 94, 95, 97, 108, 109, 111, 114, 116], "can": [0, 1, 4, 29, 30, 34, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 71, 74, 75, 76, 77, 80, 82, 88, 89, 90, 91, 92, 93, 94, 95, 97, 99, 100, 101, 104, 105, 108, 109, 111, 112, 113, 114, 115, 116, 117], "us": [0, 1, 2, 3, 4, 29, 30, 32, 34, 36, 43, 44, 45, 46, 48, 49, 52, 53, 54, 56, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 76, 77, 78, 80, 81, 82, 83, 88, 91, 94, 97, 98, 99, 100, 109, 110, 112, 114, 115, 116, 117, 118], "tensorrt": [0, 1, 3, 4, 29, 30, 31, 32, 33, 34, 37, 44, 45, 46, 48, 49, 52, 53, 54, 55, 56, 57, 59, 60, 62, 67, 68, 71, 72, 74, 75, 76, 77, 88, 91, 94, 97, 98, 101, 102, 103, 104, 105, 109], "engin": [0, 1, 17, 32, 33, 34, 45, 46, 48, 49, 52, 53, 56, 57, 59, 62, 63, 64, 69, 71, 72, 75, 76, 77, 80, 89, 90, 91, 92, 93, 98, 100, 102, 104, 109, 110, 113, 115, 117, 118], "thi": [0, 1, 2, 29, 30, 42, 43, 44, 45, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67, 68, 71, 72, 75, 76, 77, 80, 81, 82, 84, 85, 88, 89, 91, 92, 93, 95, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 111, 112, 113, 114, 115, 116, 117], "compat": [0, 1, 46, 55, 58, 64, 65, 71, 75, 76, 77, 117], "c10": [0, 1, 45, 46, 48, 49, 89, 91], "check": [0, 1, 31, 46, 52, 55, 60, 65, 67, 71, 75, 77, 89, 95, 99, 100, 112, 115], "trt": [0, 1, 3, 4, 46, 48, 53, 55, 58, 60, 62, 64, 65, 67, 68, 70, 71, 75, 76, 89, 93, 95, 104, 106, 107, 109, 113, 115, 116], "so": [0, 44, 52, 53, 54, 55, 58, 59, 60, 62, 64, 65, 66, 67, 72, 75, 76, 81, 82, 83, 89, 91, 93, 95, 97, 101, 102, 104, 106, 107, 113], "should": [0, 3, 4, 29, 45, 49, 52, 53, 54, 55, 56, 57, 59, 60, 63, 64, 65, 67, 71, 75, 76, 77, 80, 82, 85, 91, 93, 95, 96, 97, 100, 105, 112], "reason": [0, 65, 88, 93, 95, 97, 117], "you": [0, 1, 2, 29, 30, 46, 48, 49, 52, 53, 54, 55, 56, 58, 59, 60, 63, 65, 66, 67, 68, 71, 75, 76, 77, 80, 82, 83, 84, 88, 89, 90, 91, 92, 93, 95, 97, 98, 99, 100, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117], "need": [0, 1, 2, 25, 29, 43, 46, 53, 54, 55, 60, 65, 66, 67, 71, 72, 75, 76, 82, 89, 90, 91, 93, 95, 96, 97, 99, 100, 111, 112, 113, 115], "explictli": 0, "public": [0, 1, 2, 3, 4, 44, 45, 46, 47, 48, 49, 83, 91], "enum": [0, 1, 2, 42, 45, 46, 71, 77, 91, 93], "valu": [0, 1, 2, 16, 17, 45, 46, 48, 53, 56, 58, 60, 63, 70, 71, 74, 76, 80, 89, 99, 101, 102, 104, 109, 111], "underli": [0, 1, 2, 46, 60], "In": [0, 1, 2, 46, 53, 54, 56, 57, 58, 59, 60, 64, 65, 66, 75, 76, 82, 83, 85, 90, 91, 93, 95, 99, 111, 112, 113, 114, 115, 116], "case": [0, 1, 2, 46, 49, 53, 54, 56, 58, 60, 62, 64, 65, 66, 67, 75, 76, 91, 93, 95, 99, 100, 113, 114, 115], "itself": [0, 1, 2, 46, 52, 55, 92, 93, 112], "interfac": [0, 1, 2, 46, 58, 59, 60, 64, 69, 91], "vs": [0, 1, 2, 46, 55, 66, 71, 76, 77, 92], "normal": [0, 1, 2, 46, 65, 82, 88, 89, 91, 93, 99, 100, 105, 108, 112, 118], "instatin": [0, 1, 2, 46], "ex": [0, 1, 2, 33, 46, 67, 77, 83, 85], "kfloat": [0, 45, 49], "enumer": [0, 1, 2, 16, 17, 46], "klong": [0, 45], "int64": [0, 76, 77, 109], "kdoubl": [0, 45], "fp64": [0, 76], "fp32": [0, 48, 49, 52, 64, 65, 71, 76, 77, 91, 106, 107, 111, 112, 114], "khalf": [0, 45, 89], "fp16": [0, 48, 49, 52, 64, 65, 71, 72, 76, 89, 90, 99, 103, 106, 107, 109, 114, 118], "kchar": [0, 45], "int8": [0, 44, 48, 49, 52, 64, 71, 76, 77, 91, 108, 118], "kint": [0, 45], "int": [0, 3, 4, 36, 44, 45, 49, 52, 54, 56, 63, 64, 70, 71, 72, 76, 77, 80, 89, 95, 108, 109], "kbool": [0, 45], "bool": [0, 1, 2, 3, 4, 24, 27, 30, 31, 42, 44, 45, 46, 49, 55, 60, 64, 70, 71, 72, 74, 75, 76, 77, 80, 89, 91, 94, 95], "kunknown": [0, 2, 45], "sentinel": [0, 2, 76], "function": [0, 1, 2, 3, 4, 46, 48, 49, 54, 55, 56, 58, 60, 62, 64, 65, 66, 88, 89, 91, 92, 93, 95, 100, 101, 104, 105, 106, 107, 111, 112, 113, 115, 117, 118], "default": [0, 1, 2, 3, 4, 16, 29, 30, 33, 43, 45, 46, 48, 49, 52, 54, 56, 62, 64, 65, 66, 71, 72, 75, 76, 77, 80, 81, 82, 89, 90, 91, 92, 93, 94, 95, 97, 108, 113, 115, 116, 117], "construct": [0, 1, 2, 3, 4, 46, 48, 49, 53, 54, 55, 57, 59, 60, 65, 74, 75, 76, 82, 83, 89, 91, 93, 95, 97, 113], "new": [0, 1, 2, 3, 4, 32, 33, 46, 48, 49, 56, 58, 59, 60, 62, 64, 65, 68, 69, 71, 77, 82, 89, 97, 98, 99, 102, 104, 105, 110, 112, 115], "object": [0, 1, 2, 3, 4, 46, 48, 49, 52, 58, 60, 62, 63, 64, 71, 75, 76, 77, 91, 92, 93, 113, 116], "inlin": [0, 1, 2, 3, 4, 29, 30, 44, 46, 48, 55, 83, 86, 89], "constexpr": [0, 1, 2, 45, 46, 95], "t": [0, 1, 2, 45, 46, 55, 60, 65, 66, 70, 76, 80, 82, 83, 88, 89, 91, 93, 95, 108, 112, 113], "constructor": [0, 2, 46, 48, 49, 58, 88], "from": [0, 1, 2, 3, 4, 29, 30, 44, 46, 48, 49, 52, 53, 55, 56, 57, 58, 59, 60, 63, 64, 65, 67, 69, 71, 72, 75, 76, 77, 78, 80, 81, 82, 83, 88, 89, 91, 93, 94, 95, 96, 97, 99, 100, 103, 104, 106, 107, 108, 109, 111, 112, 115, 116, 117], "torchtrt_api": [0, 2, 19, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33, 34, 35, 36, 37, 42, 43, 44, 45, 48, 49, 50], "scalartyp": [0, 45, 70], "torch": [0, 1, 2, 4, 20, 21, 29, 30, 31, 32, 33, 34, 37, 44, 45, 46, 47, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 67, 71, 72, 74, 75, 76, 77, 78, 88, 91, 94, 96, 97, 98, 109, 118], "paramet": [0, 1, 2, 3, 4, 25, 26, 27, 29, 30, 31, 32, 33, 34, 36, 46, 48, 49, 53, 54, 55, 60, 64, 65, 71, 72, 74, 75, 76, 77, 86, 88, 89, 100, 106, 107], "oper": [0, 1, 2, 3, 4, 31, 44, 45, 46, 49, 52, 53, 55, 56, 57, 58, 59, 60, 62, 63, 65, 69, 71, 76, 77, 90, 91, 93, 100, 102, 104, 117, 118], "const": [0, 1, 2, 3, 4, 29, 30, 31, 32, 33, 34, 36, 44, 45, 46, 55, 60, 70, 89, 91], "get": [0, 1, 2, 3, 4, 23, 35, 44, 46, 55, 56, 60, 62, 63, 65, 67, 75, 76, 89, 91, 93, 97, 106, 107, 109, 111, 112], "return": [0, 1, 2, 3, 4, 23, 24, 29, 30, 31, 32, 33, 34, 35, 42, 43, 44, 45, 46, 54, 55, 56, 57, 58, 59, 60, 62, 64, 65, 71, 72, 75, 76, 77, 88, 89, 90, 91, 93, 95, 97, 100, 101, 108, 109, 112, 113, 114], "explicit": [0, 1, 2, 3, 4, 45, 46, 55, 65, 72, 75, 82, 91, 117], "delet": [0, 1, 2, 45, 46, 55], "other": [0, 1, 2, 45, 46, 52, 53, 55, 58, 62, 64, 65, 66, 70, 71, 75, 76, 81, 82, 89, 90, 93, 115], "comparis": [0, 2], "true": [0, 1, 2, 4, 46, 49, 55, 56, 60, 62, 64, 65, 70, 71, 72, 75, 76, 77, 80, 83, 89, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 112, 114, 115, 118], "fals": [0, 1, 2, 3, 4, 44, 45, 46, 49, 54, 62, 64, 65, 70, 71, 72, 75, 76, 77, 80, 81, 82, 83, 89, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 115], "struct": [1, 21, 38, 41, 45, 54, 91], "onli": [1, 3, 4, 16, 29, 44, 46, 48, 52, 54, 55, 56, 59, 60, 64, 65, 67, 68, 71, 72, 75, 76, 82, 91, 93, 94, 95, 99, 100, 107, 109, 114, 115, 118], "applic": [1, 29, 46, 52, 55, 59, 64, 71, 75, 76, 89, 90, 92, 115, 118], "kcuda": [1, 46, 56, 89], "which": [1, 2, 29, 32, 34, 46, 49, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 71, 72, 74, 75, 76, 77, 80, 82, 83, 88, 89, 90, 91, 92, 93, 94, 95, 97, 101, 102, 105, 106, 107, 111, 112, 113, 114, 115, 116, 117], "map": [1, 46, 53, 54, 55, 57, 59, 60, 65, 76, 89, 91, 92, 97, 101, 111, 112], "kgpu": [1, 45, 46], "To": [1, 46, 52, 54, 56, 64, 66, 71, 80, 88, 89, 90, 92, 95, 100, 106, 107, 112], "datatyp": [1, 21, 38, 45, 46, 48, 49, 50, 71, 76, 77, 90, 95, 112, 114], "target": [1, 33, 45, 46, 48, 49, 52, 54, 56, 58, 59, 64, 65, 66, 69, 71, 75, 76, 77, 90, 91, 92, 93, 95, 100, 117, 118], "gpu": [1, 32, 34, 36, 45, 46, 52, 64, 65, 71, 75, 76, 77, 89, 91, 92, 95, 106, 107, 109, 112, 115, 117, 118], "run": [1, 34, 46, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 64, 65, 66, 67, 68, 71, 72, 75, 76, 77, 82, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 111, 112, 113, 114, 115, 116, 117, 118], "kdla": [1, 45, 46, 118], "dla": [1, 45, 46, 49, 52, 64, 69, 71, 76, 77], "intern": [1, 16, 46, 60, 63, 73, 75, 82, 89], "note": [1, 46, 48, 54, 60, 62, 65, 66, 67, 75, 76, 80, 82, 89, 95, 100, 113, 118], "The": [1, 46, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 71, 75, 76, 77, 80, 83, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 102, 105, 106, 109, 111, 112, 113, 116, 117], "valid": [1, 46, 56, 60, 62, 71, 75, 76, 93], "kcpu": [1, 46], "comparison": [1, 46], "an": [2, 3, 4, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 64, 65, 66, 68, 71, 72, 74, 75, 76, 77, 80, 82, 83, 88, 89, 90, 91, 93, 95, 97, 100, 101, 105, 106, 107, 109, 111, 112, 113, 115, 116, 117], "memeori": 2, "layout": [2, 48, 70, 71, 76, 77], "store": [2, 4, 49, 52, 53, 58, 60, 64, 65, 71, 75, 76, 77, 88, 89, 95, 97, 100], "tensor": [2, 33, 44, 45, 48, 49, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 70, 71, 72, 75, 76, 77, 88, 89, 90, 91, 93, 95, 101, 109, 111], "kcontigu": [2, 45, 48], "contigu": [2, 48, 49, 52, 71, 76, 77], "nchw": [2, 71, 76, 77], "linear": [2, 56, 70, 76, 88, 95, 108, 114], "kchannelslast": [2, 45], "channel": [2, 76, 81], "last": [2, 55, 65, 76, 108], "nhwc": [2, 52], "memoryformat": [2, 45], "ptq": [3, 4, 15, 18, 19, 38, 50, 51, 52, 69, 71, 76, 77], "privat": [3, 4, 44, 45, 91], "algorithm": [3, 4, 29, 30, 44, 65, 74, 91, 107], "typenam": [3, 4, 29, 30, 44], "gener": [3, 4, 29, 52, 55, 58, 59, 60, 62, 64, 65, 66, 71, 72, 80, 82, 83, 86, 88, 89, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 108, 109, 110, 115], "int8calibr": [3, 20, 30, 40, 44, 50], "implement": [3, 4, 55, 56, 58, 63, 65, 75, 81, 89, 91, 95, 97, 115], "specifi": [3, 4, 33, 52, 54, 60, 64, 65, 66, 71, 76, 77, 80, 82, 90, 92, 109, 112, 113, 114, 116, 117], "calibr": [3, 4, 29, 30, 44, 49, 52, 71, 74, 76, 77, 89, 91], "read": [3, 4, 29, 30, 44, 80, 82, 91], "nvinfer1": [3, 4, 29, 30, 44, 45, 49, 60, 91], "iint8calibr": [3, 4, 29, 30, 44, 45, 49, 71, 76, 77, 91], "iint8entropycalibrator2": [3, 4, 29, 30, 44, 91], "std": [3, 4, 22, 26, 28, 29, 30, 31, 33, 34, 35, 42, 44, 45, 47, 48, 49, 56, 89, 91, 112, 118], "string": [3, 4, 18, 20, 21, 22, 26, 28, 29, 30, 31, 33, 34, 35, 42, 44, 45, 49, 54, 56, 58, 60, 64, 71, 76, 80, 89, 91], "cache_file_path": [3, 4, 29, 30, 44], "8": [3, 52, 55, 63, 64, 66, 75, 76, 82, 83, 86, 89, 94, 95, 102, 105, 112, 113], "cach": [3, 4, 29, 30, 44, 52, 64, 65, 69, 71, 72, 74, 76, 89, 91, 98, 110, 115], "getbatchs": [3, 4, 44], "noexceptoverrid": [3, 4], "batch": [3, 4, 44, 64, 65, 72, 75, 91, 97, 102, 104, 108, 109, 112, 113, 118], "size": [3, 4, 44, 48, 49, 52, 55, 56, 64, 65, 70, 71, 72, 76, 77, 80, 89, 91, 95, 97, 102, 104, 108, 111, 113], "next": [3, 4, 53, 54, 58, 63, 72, 76, 80, 82, 83, 91, 93, 101, 105, 108, 112], "alwai": [3, 4, 27, 52, 76, 82, 100, 109], "1": [3, 4, 33, 44, 45, 48, 49, 52, 54, 55, 56, 58, 60, 62, 63, 64, 65, 66, 70, 71, 72, 74, 75, 76, 77, 79, 80, 82, 83, 86, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 102, 104, 105, 106, 107, 108, 109, 111, 113, 114, 116, 118], "due": [3, 4, 66, 81, 82, 108], "issu": [3, 4, 64, 71, 76, 89, 101, 104], "getbatch": [3, 4, 44], "void": [3, 4, 25, 26, 27, 28, 36, 37, 42, 44, 45], "bind": [3, 4, 33, 44, 75, 77, 82], "char": [3, 4, 44, 52, 89], "name": [3, 4, 31, 33, 34, 44, 54, 56, 58, 60, 65, 66, 67, 72, 74, 75, 76, 77, 82, 83, 88, 89, 92, 93, 95, 100, 105, 108, 112, 114], "nbbind": [3, 4, 44], "Not": 3, "arrai": [3, 4, 33, 53, 54, 76, 77, 93, 95, 109], "pointer": [3, 4, 91], "fed": [3, 4, 48], "buffer": [3, 4, 65, 95], "each": [3, 4, 49, 53, 55, 56, 58, 60, 64, 65, 66, 71, 72, 75, 80, 82, 89, 93, 100, 107, 115], "input": [3, 4, 21, 29, 33, 38, 44, 45, 47, 49, 50, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 68, 70, 71, 72, 73, 75, 76, 77, 83, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 105, 106, 107, 108, 109, 111, 112, 113, 114, 115, 116, 117, 118], "number": [3, 4, 49, 52, 54, 55, 56, 60, 63, 64, 65, 71, 72, 76, 77, 80, 89, 90, 95, 100, 102, 104, 109, 111, 117], "readcalibrationcach": [3, 4, 44], "size_t": [3, 4, 44, 91], "length": [3, 4, 44, 65, 70, 83, 109], "how": [3, 4, 66, 67, 82, 84, 86, 88, 92, 93, 95, 97, 99, 101, 108, 109, 111, 112, 113, 115], "enabl": [3, 4, 24, 49, 52, 54, 56, 57, 59, 64, 65, 66, 71, 72, 74, 75, 76, 77, 80, 97, 99, 100, 102, 104, 105, 106, 107, 109, 114, 115], "use_cach": [3, 4, 30, 44, 74, 91, 106, 107, 109], "set": [3, 4, 16, 21, 25, 27, 29, 32, 34, 36, 45, 46, 48, 49, 52, 53, 54, 55, 56, 57, 58, 59, 65, 66, 71, 72, 75, 76, 77, 80, 84, 87, 88, 89, 90, 91, 93, 95, 100, 106, 108, 109, 111, 113, 114, 115, 117, 118], "writecalibrationcach": [3, 4, 44], "write": [3, 4, 29, 30, 44, 65, 69, 82, 89, 91, 112], "provid": [3, 4, 49, 52, 54, 56, 58, 60, 62, 64, 65, 66, 68, 71, 72, 75, 76, 77, 82, 89, 90, 91, 92, 93, 97, 98, 100, 101, 105, 109, 110, 112, 113, 115, 116, 117], "cast": [3, 4, 55, 64, 71, 106, 107, 114], "convienc": [3, 4, 49], "convert": [3, 4, 31, 32, 34, 52, 55, 56, 57, 59, 63, 64, 69, 71, 76, 77, 90, 92, 95, 98, 102, 104, 109, 110, 111, 115], "easili": [3, 4, 99], "assign": [3, 4, 81], "ptq_calibr": [3, 4, 45, 49, 91], "field": [3, 4, 63, 72, 76, 91], "compilespec": [3, 4, 21, 32, 34, 41, 45, 50, 56, 77, 89, 91, 118], "dataloaderuniqueptr": [4, 44], "libtorch": [4, 37, 60, 66, 68, 89, 91, 117], "dataload": [4, 29, 30, 44, 49, 74, 91, 108], "unique_ptr": [4, 30], "unqiue_ptr": 4, "A": [4, 29, 30, 32, 33, 47, 48, 54, 55, 56, 60, 65, 66, 71, 72, 76, 77, 83, 91, 103, 112], "uniqu": [4, 90], "what": [4, 54, 55, 65, 68, 76, 82, 88, 89, 90, 106, 107, 117], "make_data_load": [4, 91], "factori": [4, 29, 30, 64, 71, 91], "path": [4, 13, 14, 15, 29, 30, 52, 64, 65, 66, 67, 71, 74, 76, 88, 89, 91, 94, 97, 105, 108, 112, 117], "find": [4, 65, 66, 67, 89, 95, 109], "whether": [4, 52, 54, 64, 65, 71, 72, 76, 81, 91, 102, 104, 115], "exist": [4, 31, 32, 34, 54, 63, 64, 65, 67, 71, 74, 76, 77, 91, 97, 111], "There": [4, 53, 54, 59, 60, 62, 63, 65, 66, 83, 88, 91, 100, 111, 112, 113, 115], "consum": [4, 53, 88], "macro": [5, 6, 7, 8, 9, 10, 11, 12, 15, 18, 20, 21, 42, 44, 45, 50, 51], "x": [5, 10, 33, 43, 55, 56, 66, 67, 68, 75, 77, 83, 88, 89, 93, 95, 97, 101, 105, 108, 109, 113, 114, 116], "includ": [13, 15, 16, 35, 37, 42, 43, 44, 45, 51, 52, 54, 56, 57, 58, 59, 62, 64, 65, 66, 67, 68, 71, 72, 75, 76, 80, 82, 88, 89, 91, 95, 115], "parent": [14, 15, 18, 19, 20, 21], "cpp": [14, 15, 42, 43, 44, 45, 51, 55, 59, 66, 89, 91], "log": [15, 16, 19, 20, 38, 44, 50, 51, 55, 60, 64, 65, 69, 70, 71, 72, 76, 93, 102, 104, 114], "emum": [16, 17], "messag": [16, 25, 26, 52, 73], "sever": [16, 26, 73], "kinternal_error": [16, 42], "print": [16, 31, 44, 62, 64, 67, 71, 77, 82, 89, 92, 93, 94, 95, 96, 97, 99, 100, 102, 104, 106, 107, 108, 109, 112], "error": [16, 49, 52, 53, 55, 59, 64, 65, 71, 73, 76, 77, 82, 89, 113], "kerror": [16, 42], "all": [16, 42, 43, 44, 45, 49, 52, 54, 55, 56, 58, 62, 64, 65, 66, 67, 71, 73, 75, 76, 78, 82, 83, 88, 89, 90, 91, 93, 95, 106, 107, 110, 111, 112, 114, 115, 117], "kwarn": [16, 42], "warn": [16, 44, 52, 60, 73, 75], "kinfo": [16, 42, 44], "info": [16, 32, 34, 45, 52, 60, 73, 75, 76, 114], "kdebug": [16, 42, 44], "debug": [16, 27, 45, 49, 52, 60, 62, 64, 71, 73, 75, 76, 77, 92, 94, 95, 96, 97, 99, 100, 101, 102, 104, 108, 114], "kgraph": [16, 42, 55], "everyth": [16, 64, 71, 76], "intermedi": [16, 49, 52, 54, 64, 71, 73, 76, 77, 88, 114, 117], "graph": [16, 31, 32, 34, 45, 49, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 65, 71, 72, 73, 76, 77, 88, 89, 93, 95, 97, 99, 100, 102, 104, 105, 111, 113, 115], "lower": [16, 54, 63, 65, 69, 71, 72, 73, 76, 83, 95, 97, 102, 104, 109, 111, 117], "phase": [16, 60, 63, 89, 93, 100, 113, 117], "select": [17, 29, 30, 34, 49, 52, 58, 64, 65, 66, 70, 71, 76, 77, 81, 84, 90, 91, 95, 117], "capabl": [17, 45, 49, 52, 58, 71, 76, 77, 92, 93, 94], "kstandard": [17, 45, 49], "ksafeti": [17, 45], "kdla_standalon": [17, 45], "directori": [18, 19, 20, 21, 42, 43, 44, 45, 50, 66, 67, 71, 91, 97], "program": [18, 19, 20, 21, 29, 51, 52, 57, 58, 59, 69, 71, 88, 97, 98, 106, 107, 110, 113], "list": [18, 19, 20, 21, 31, 49, 51, 53, 56, 58, 60, 62, 63, 65, 68, 70, 71, 72, 75, 76, 77, 86, 89, 90, 93, 95, 112], "torchscript": [19, 21, 38, 43, 45, 49, 50, 52, 56, 57, 58, 59, 63, 68, 71, 72, 74, 75, 76, 77, 90, 111, 113, 118], "str": [19, 43, 44, 50, 54, 64, 65, 70, 71, 74, 75, 76, 77, 93, 94, 95, 97, 108], "torch_tensorrt_major_vers": [19, 43, 50], "torch_tensorrt_minor_vers": [19, 43, 50], "torch_tensorrt_patch_vers": [19, 43, 50], "torch_tensorrt_vers": [19, 43, 50], "torchtrt_hidden": [19, 43, 50], "xstr": [19, 43, 50], "nvinfer": [20, 44], "fstream": [20, 44], "iostream": [20, 21, 44, 45, 89], "iter": [20, 44, 49, 52, 53, 64, 71, 74, 76, 77, 96, 97, 108, 109], "memori": [20, 21, 44, 45, 55, 60, 71, 76, 77, 89, 90, 95, 97, 106, 107, 109], "sstream": [20, 44], "vector": [20, 21, 33, 44, 45, 47, 48, 49, 56, 58, 76, 89, 91, 118], "templat": [20, 40, 44, 45, 50, 80, 89], "int8cachecalibr": [20, 29, 40, 44, 50], "cuda_runtim": [21, 45], "custom_class": [21, 45], "devic": [21, 33, 36, 38, 45, 49, 50, 52, 58, 64, 70, 71, 72, 74, 75, 76, 77, 90, 91, 92, 95, 99, 103, 106, 107, 109, 111, 118], "graphinput": [21, 38, 45, 49, 50], "devicetyp": [21, 38, 45, 46, 50, 75, 76, 77, 91, 92, 95, 118], "tensorformat": [21, 38, 45, 48, 50, 76, 95], "level": [23, 25, 26, 39, 42, 44, 50, 54, 55, 56, 59, 64, 65, 71, 76, 77, 86, 88, 93, 95, 112, 117], "current": [23, 54, 56, 58, 60, 62, 63, 64, 65, 66, 67, 71, 72, 75, 76, 77, 80, 93, 95, 99, 106, 107, 108, 109, 115], "report": [23, 44, 75], "Is": [24, 76], "color": [24, 27, 82], "output": [24, 27, 33, 49, 52, 53, 54, 55, 56, 58, 60, 62, 63, 64, 65, 66, 71, 73, 75, 76, 77, 80, 82, 83, 89, 93, 95, 97, 99, 100, 103, 109, 111, 112, 113, 114, 116], "lvl": [25, 26, 42], "inform": [25, 33, 35, 37, 48, 52, 53, 56, 58, 62, 64, 65, 66, 71, 72, 73, 76, 82, 88, 89, 91, 92, 95, 97, 109, 113], "ad": [25, 52, 53, 54, 56, 62, 65, 66, 95, 99], "abov": [25, 54, 56, 62, 65, 66, 73, 81, 82, 89, 95, 102, 104, 114, 116], "msg": [26, 42], "add": [26, 53, 54, 55, 56, 60, 63, 66, 70, 80, 82, 87, 89, 90, 93, 95], "global": [26, 52, 64, 71, 76, 89], "colored_output_on": [27, 42], "prefix": [27, 28, 42, 82], "help": [27, 52, 53, 60, 64, 65, 89, 94, 97, 108, 109, 111, 115], "when": [27, 44, 45, 46, 52, 53, 55, 56, 57, 58, 59, 60, 64, 65, 66, 71, 75, 76, 77, 80, 82, 84, 88, 89, 91, 93, 95, 97, 99, 100, 109, 111, 113, 115], "termin": [27, 52, 89], "If": [27, 33, 53, 54, 55, 56, 62, 63, 64, 65, 66, 68, 71, 72, 76, 80, 82, 89, 90, 91, 93, 95, 97, 100, 101, 105, 109, 112, 113, 114, 115, 117, 118], "build": [29, 30, 35, 49, 52, 53, 57, 59, 60, 63, 64, 65, 71, 75, 76, 81, 86, 89, 91, 93, 95, 102, 104, 109, 113], "post": [29, 30, 49, 52, 63, 69, 89, 97], "train": [29, 30, 49, 52, 69, 70, 89, 90, 97, 109], "quantiz": [29, 30, 52, 64, 69, 74, 76, 89, 98, 110], "creat": [29, 30, 33, 52, 53, 54, 56, 58, 60, 65, 69, 76, 77, 82, 89, 93, 95, 100, 109, 112], "previous": [29, 33, 89, 97, 100], "therefor": [29, 58, 65, 66, 75, 82, 89, 111, 115], "have": [29, 33, 44, 52, 53, 54, 55, 56, 60, 62, 63, 64, 65, 66, 67, 71, 72, 74, 75, 76, 77, 82, 88, 89, 90, 91, 95, 98, 102, 104, 108, 110, 111, 112, 113], "requir": [29, 49, 52, 53, 54, 55, 63, 64, 65, 66, 67, 71, 76, 77, 80, 89, 91, 93, 94, 95, 98, 108, 109, 110, 112, 113, 115], "dataset": [29, 74, 91, 111], "save": [29, 44, 52, 58, 64, 65, 68, 69, 71, 75, 76, 77, 89, 90, 94, 96, 97, 100, 103, 109, 111, 112, 115, 117], "later": [29, 71, 89, 100, 116, 117], "differ": [29, 55, 56, 59, 64, 65, 66, 76, 80, 88, 93, 95, 97, 99, 106, 109, 111, 115, 117], "scratch": [29, 97, 100], "depend": [29, 35, 53, 59, 64, 65, 67, 68, 71, 89, 90, 109, 112, 115], "howev": [29, 66, 80, 81, 89, 93, 95, 97, 112, 113, 117], "network": [29, 30, 54, 60, 65, 76, 89, 91, 93, 95, 109, 111, 112, 118], "also": [29, 53, 54, 60, 62, 64, 66, 68, 80, 82, 83, 89, 90, 91, 97, 105, 108, 111], "recalibr": 29, "its": [29, 53, 56, 58, 60, 66, 75, 76, 82, 95, 108, 112, 115, 117], "structur": [29, 46, 49, 56, 59, 60, 64, 71, 76, 80, 82, 86, 88, 95, 112], "chang": [29, 55, 56, 59, 62, 64, 65, 75, 76, 77, 80, 91, 93, 97, 99, 100, 112, 115, 117], "respons": [29, 54, 58, 82, 115], "ensur": [29, 54, 55, 56, 62, 64, 66, 67, 71, 75, 106, 107], "By": [29, 30, 51, 56, 64, 66, 71, 80, 88, 97, 113], "entropi": [29, 30, 91], "v2": [29, 30, 82], "perform": [29, 30, 54, 62, 63, 71, 75, 76, 91, 95, 105, 109, 111, 112, 114, 115, 116, 117], "recommend": [29, 30, 65, 66, 76, 82, 89, 95, 112, 113], "feed": [29, 30, 89], "forward": [29, 30, 32, 33, 56, 58, 60, 64, 68, 71, 75, 76, 77, 88, 89, 90, 91, 92, 93, 95, 101, 108, 113, 114], "overrid": [29, 30, 44, 54, 65, 91], "minmax": [29, 30, 91], "recomend": [29, 30], "nlp": [29, 30, 91], "task": [29, 30, 65, 91, 111], "call": [29, 30, 32, 49, 54, 55, 58, 60, 65, 71, 72, 75, 76, 77, 82, 88, 89, 92, 93, 95, 97, 99, 101, 104, 111, 113, 115, 117], "make_int8_calibr": [29, 40, 44, 50, 91], "class": [29, 30, 44, 45, 46, 51, 58, 60, 64, 65, 73, 77, 82, 83, 88, 89, 90, 91, 93, 95, 97, 101, 108, 111, 113, 114], "e": [29, 30, 52, 55, 60, 65, 66, 67, 68, 72, 76, 88, 89, 91, 95, 97, 100], "g": [29, 30, 52, 55, 65, 66, 67, 72, 76, 82, 91, 95, 100], "iint8minmaxcalibr": [29, 30, 91], "calibration_cache_fil": [29, 30, 91], "move": [30, 44, 55, 58, 77, 89, 91, 93, 106, 107], "calibration_dataload": [30, 91], "contain": [30, 31, 52, 53, 54, 55, 56, 60, 65, 66, 72, 75, 76, 82, 83, 88, 89, 91, 95, 97, 112, 115], "jit": [31, 32, 33, 34, 45, 47, 49, 52, 53, 55, 56, 57, 58, 59, 60, 61, 64, 68, 69, 71, 75, 76, 77, 88, 89, 90, 92, 95, 100, 112, 116, 117], "modul": [31, 32, 33, 34, 45, 49, 52, 56, 57, 58, 59, 60, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 76, 77, 81, 82, 83, 90, 91, 92, 93, 94, 95, 98, 100, 101, 108, 110, 111, 113, 114, 116, 118], "method_nam": [31, 34, 45, 52, 76, 77, 89], "see": [31, 55, 56, 58, 62, 64, 65, 66, 76, 77, 82, 88, 89, 90, 93, 95, 97, 100, 101], "fulli": [31, 52, 55, 64, 71, 75, 76, 77, 89, 91, 95, 118], "compil": [31, 34, 41, 45, 49, 50, 52, 54, 55, 56, 58, 60, 62, 65, 71, 72, 73, 75, 76, 77, 78, 80, 88, 90, 91, 92, 93, 95, 96, 98, 99, 108, 110, 112, 115, 118], "take": [31, 32, 33, 34, 53, 54, 57, 58, 59, 60, 62, 65, 71, 72, 75, 76, 77, 80, 82, 89, 91, 92, 93, 95, 101, 111, 113], "method": [31, 32, 33, 34, 48, 52, 55, 60, 66, 71, 76, 77, 82, 88, 89, 92, 97, 111], "pure": [31, 71, 76], "Will": 31, "out": [31, 44, 53, 55, 56, 57, 59, 60, 64, 66, 71, 76, 77, 82, 89, 95, 99, 108, 109, 112, 113], "unsupport": [31, 49, 54, 64, 76, 95, 117], "script": [31, 55, 56, 68, 76, 77, 88, 89, 90, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 115, 117], "nvidia": [32, 34, 42, 43, 44, 45, 52, 61, 64, 65, 66, 67, 71, 76, 77, 89, 101, 104, 112, 117, 118], "configur": [32, 34, 48, 62, 64, 66, 71, 75, 76, 77, 86, 89, 91, 95, 109, 112, 113], "equival": [32, 57, 59, 60, 71, 76, 77, 88, 89, 91, 93, 95, 102, 104], "specif": [32, 49, 54, 55, 57, 59, 62, 64, 71, 76, 77, 82, 93, 109, 117], "traget": 32, "input_binding_nam": [33, 45, 75, 77], "output_binding_nam": [33, 45, 75, 77], "emb": [33, 52, 63, 77, 83], "pre": [33, 55, 74, 77, 91, 97, 109, 115], "built": [33, 52, 58, 59, 64, 66, 71, 75, 76, 77, 97, 100], "serial": [33, 34, 52, 57, 59, 66, 71, 75, 76, 77, 89, 95, 97, 117], "regist": [33, 54, 58, 60, 65, 75, 77, 93, 95], "execut": [33, 49, 52, 55, 57, 58, 59, 63, 64, 65, 66, 69, 71, 72, 75, 76, 77, 78, 88, 89, 91, 93, 95, 112], "must": [33, 48, 49, 52, 54, 55, 56, 60, 62, 65, 66, 71, 72, 76, 77, 82, 83, 89, 97, 113, 115, 117], "follow": [33, 52, 54, 56, 58, 62, 63, 64, 65, 66, 77, 80, 82, 83, 87, 88, 89, 91, 93, 95, 97, 98, 102, 106, 107, 110, 111, 112, 113, 114, 115], "format": [33, 45, 48, 49, 52, 70, 71, 76, 77, 82, 83, 90, 95, 97, 108, 111, 112, 114, 116], "symbol": [33, 65, 66, 77, 82, 115], "index": [33, 61, 62, 66, 67, 69, 70, 77, 80, 86, 91, 95], "0": [33, 43, 44, 45, 49, 52, 54, 56, 59, 60, 62, 64, 65, 66, 67, 69, 70, 71, 72, 74, 75, 76, 77, 78, 79, 81, 82, 89, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 112, 113, 114, 118], "2": [33, 43, 54, 56, 60, 63, 64, 65, 66, 67, 69, 70, 71, 74, 75, 76, 77, 80, 82, 83, 86, 88, 89, 91, 93, 95, 96, 97, 99, 100, 101, 102, 104, 106, 107, 108, 109, 113, 116], "y": [33, 56, 77, 83, 93, 95, 101], "compilesepc": 33, "order": [33, 49, 54, 56, 60, 62, 65, 66, 71, 72, 75, 76, 77, 89, 90, 93, 97, 114], "pass": [33, 53, 54, 56, 57, 58, 59, 60, 63, 64, 65, 66, 69, 73, 74, 75, 76, 77, 88, 89, 91, 93, 95, 97, 100], "origin": [33, 65, 72, 76, 95, 97, 99, 117], "pytorch": [33, 48, 49, 52, 54, 55, 56, 57, 58, 59, 60, 63, 64, 66, 67, 68, 71, 74, 75, 76, 77, 88, 89, 90, 91, 93, 97, 99, 100, 108, 112, 113, 114, 115, 116, 117], "assum": [33, 75, 92, 95, 98, 110], "convent": 33, "below": [33, 56, 60, 62, 63, 64, 65, 66, 67, 82, 89, 90, 97, 103, 112], "equivil": 34, "librari": [35, 42, 43, 44, 45, 52, 54, 57, 58, 59, 60, 76, 89, 95, 98, 110], "version": [35, 37, 59, 62, 64, 65, 67, 71, 75, 76, 80, 83, 95, 111, 112, 116], "gpu_id": [36, 45, 46, 52, 75, 76, 77, 91, 92, 95, 118], "id": [36, 45, 52, 76, 80, 81, 85, 118], "cudasetdevic": 36, "dump": [37, 52, 95], "base": [37, 50, 58, 63, 64, 66, 71, 72, 76, 82, 88, 90, 91, 96, 100, 104, 111, 117], "stdout": [37, 75], "enginecap": [38, 45, 49, 50, 64, 71, 75, 76, 77, 92, 95], "dump_build_info": [38, 45, 50], "get_build_info": [38, 45, 50], "set_devic": [38, 45, 50, 115], "get_is_colored_output_on": [39, 42, 50], "get_logging_prefix": [39, 42, 50], "get_reportable_log_level": [39, 42, 50], "set_is_colored_output_on": [39, 42, 50], "set_logging_prefix": [39, 42, 50], "set_reportable_log_level": [39, 42, 50], "make_int8_cache_calibr": [40, 44, 50, 91], "check_method_operator_support": [41, 45, 50], "convert_method_to_trt_engin": [41, 45, 50, 76, 77, 89, 92], "embed_engine_in_new_modul": [41, 45, 50, 77], "document": [42, 43, 44, 45, 50, 59, 80, 82, 83, 87, 88, 89, 91, 92, 112, 113, 115], "copyright": [42, 43, 44, 45, 83, 89], "c": [42, 43, 44, 45, 52, 59, 64, 67, 70, 71, 72, 75, 76, 83, 90, 95, 99, 112, 115, 118], "corpor": [42, 43, 44, 45], "right": [42, 43, 44, 45, 55, 59, 60, 82], "reserv": [42, 43, 44, 45, 106, 107], "licens": [42, 43, 44, 45, 89], "under": [42, 43, 44, 45, 59, 65, 82, 93, 102, 117], "bsd": [42, 43, 44, 45], "style": [42, 43, 44, 45, 64, 68, 80, 82, 83], "found": [42, 43, 44, 45, 63, 66, 75, 82, 89, 91, 93, 95, 97, 115], "root": [42, 43, 44, 45, 66, 80, 91, 108], "sourc": [42, 43, 44, 45, 54, 59, 64, 65, 67, 71, 72, 73, 74, 75, 76, 77, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110], "tree": [42, 43, 44, 45, 80, 91, 108, 115], "pragma": [42, 43, 44, 45, 91], "onc": [42, 43, 44, 45, 53, 55, 56, 58, 64, 65, 66, 67, 76, 91, 95, 107, 109, 112, 115], "namespac": [42, 43, 44, 45, 51, 55, 69, 76, 91, 95], "ar": [42, 46, 49, 52, 53, 54, 55, 56, 58, 59, 60, 62, 63, 64, 65, 66, 71, 74, 75, 76, 77, 80, 82, 83, 84, 88, 89, 91, 92, 93, 95, 96, 97, 99, 100, 102, 106, 107, 109, 111, 112, 113, 114, 115, 116, 117], "ones": [42, 56, 57, 59, 66, 82, 89, 93, 95, 117], "necessari": [42, 62, 64, 66, 75, 93, 100, 115], "user": [42, 48, 54, 56, 57, 58, 59, 62, 63, 64, 66, 67, 71, 82, 83, 89, 90, 91, 93, 97, 100, 109, 112, 113, 114, 115, 117], "dont": 42, "know": [42, 60, 80, 82, 93, 95], "we": [42, 44, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 72, 75, 80, 82, 88, 89, 91, 93, 95, 97, 98, 99, 100, 101, 102, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 116, 117], "want": [42, 56, 65, 66, 67, 68, 72, 88, 89, 91, 92, 93, 95, 100, 101, 112], "use_cmake_generated_export_head": 43, "torch_tensorrt_export": 43, "els": [43, 44, 48, 77, 82, 83, 94, 96, 97, 108], "__gnuc__": 43, "__attribute__": 43, "__visibility__": 43, "hidden": [43, 80], "endif": [43, 44, 45], "doe": [43, 44, 55, 56, 60, 62, 65, 66, 76, 82, 91, 95, 102, 104], "gaurd": 43, "someth": [43, 55, 82, 112], "6": [43, 55, 56, 58, 66, 70, 82, 86, 88, 89, 94, 95], "setup": [43, 67, 91, 112], "alias": 43, "eas": 43, "ts": [43, 52, 56, 68, 69, 76, 88, 89, 90, 92, 113, 116], "torchtrt": [43, 56, 94, 95, 108], "ifndef": [44, 45], "doxygen_should_skip_thi": [44, 45], "get_batch_impl": 44, "element_typ": 44, "super": [44, 88, 93, 95, 101, 108, 113, 114], "batchtyp": 44, "dataloader_": 44, "cache_file_path_": 44, "use_cache_": 44, "auto": [44, 56, 60, 64, 68, 71, 82, 83, 89, 91, 106, 107, 109, 118], "batched_data_": 44, "push_back": [44, 56], "it_": 44, "begin": [44, 65, 66, 82, 101, 105], "noexcept": [44, 91], "hack": 44, "explict": 44, "work": [44, 55, 59, 60, 64, 65, 68, 71, 74, 75, 76, 82, 83, 91, 95, 100, 101, 105, 109, 113], "here": [44, 53, 54, 56, 58, 63, 64, 65, 66, 68, 80, 82, 83, 88, 89, 91, 93, 95, 98, 105, 106, 107, 108, 110, 112, 113, 115, 116], "explic": 44, "just": [44, 45, 55, 56, 64, 65, 69, 73, 75, 82, 84, 88, 89, 90, 92, 95, 97, 99, 111, 115], "still": [44, 56, 65, 66, 91, 93, 101, 117], "static_cast": 44, "option": [44, 48, 52, 56, 57, 59, 62, 63, 64, 65, 71, 75, 76, 77, 82, 86, 91, 93, 95, 96, 97, 101, 103, 114, 115, 116, 118], "batch_siz": [44, 91, 108], "end": [44, 52, 60, 62, 70, 71, 76, 77, 82, 89, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "statu": [44, 83], "reset": [44, 96, 97, 101, 104, 115], "incas": 44, "go": [44, 55, 56, 65, 68, 88, 89, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 111, 112, 117], "again": [44, 58, 60, 82, 95, 99], "stringstream": 44, "ss": 44, "cache_": 44, "clear": 44, "ifstream": 44, "io": [44, 67, 112], "binari": [44, 91], "noskipw": 44, "good": [44, 60, 65, 82, 97], "copi": [44, 60, 65, 67, 70, 74, 83, 109, 112], "istream_iter": 44, "back_insert": 44, "nullptr": [44, 45, 49], "ofstream": [44, 89], "cache_fil": [44, 74, 91], "reinterpret_cast": 44, "cache_size_": 44, "int8_t": 45, "arrayref": [45, 48, 49], "friend": 45, "ostream": 45, "os": [45, 67, 97], "dtype": [45, 48, 49, 52, 63, 64, 65, 70, 71, 72, 75, 76, 77, 90, 95, 96, 102, 104, 105, 109, 111, 113, 114], "device_typ": [45, 46, 76, 91, 92, 118], "int64_t": [45, 46, 48, 49, 91, 118], "core": [45, 52, 55, 56, 59, 64, 71, 76, 89, 93, 117, 118], "agx": 45, "platform": [45, 52, 59, 64, 66, 67, 71, 94, 112, 118], "xavier": [45, 118], "dla_cor": [45, 46, 52, 76, 91, 92, 118], "allow_gpu_fallback": [45, 46, 71, 76, 77, 91, 92, 118], "customclasshold": [45, 48], "min_shap": [45, 48, 63, 65, 71, 76, 77, 90, 102, 105, 111, 113], "opt_shap": [45, 48, 63, 71, 76, 77, 90, 102, 105, 111, 113], "max_shap": [45, 48, 63, 65, 71, 76, 77, 90, 102, 105, 111, 113], "shape": [45, 47, 48, 49, 52, 56, 60, 63, 65, 69, 70, 71, 72, 75, 76, 77, 78, 90, 93, 95, 98, 105, 108, 109, 110, 112, 115, 118], "doubl": [45, 48, 49, 52, 63, 71, 76, 77, 82, 115], "tensor_domain": [45, 48, 76], "input_is_dynam": 45, "ivalu": [45, 47, 49, 53, 58, 60, 89], "input_signatur": [45, 47, 49, 77, 90], "nest": [45, 49, 50, 82, 83], "full": [45, 49, 52, 60, 64, 71, 73, 76, 89, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 112, 115, 118], "spec": [45, 48, 49, 52, 73, 76, 77, 92, 97], "flatten": [45, 47, 70, 88, 89, 108], "fixed_s": [45, 49], "reflect": [45, 76], "builderconfig": 45, "graph_input": [45, 49], "enabled_precis": [45, 49, 63, 64, 71, 75, 76, 77, 89, 90, 91, 92, 95, 96, 97, 99, 100, 101, 102, 103, 104, 106, 107, 108, 109, 112, 114, 118], "disable_tf32": [45, 49, 64, 71, 75, 76, 77, 91, 95, 106, 107], "sparse_weight": [45, 49, 64, 65, 71, 75, 76, 77, 95], "refit": [45, 49, 64, 69, 71, 76, 77, 92, 95, 97, 98, 99, 110], "truncate_long_and_doubl": [45, 49, 63, 64, 77, 103], "allow_shape_tensor": [45, 49, 77], "uint64_t": [45, 49], "num_avg_timing_it": [45, 49, 64, 71, 75, 76, 77, 92, 95], "workspace_s": [45, 49, 52, 64, 71, 75, 76, 77, 95, 100, 102, 104], "dla_sram_s": [45, 49, 52, 64, 71, 75, 76, 77, 95], "1048576": [45, 49, 64, 71, 75, 76, 77, 95], "dla_local_dram_s": [45, 49, 52, 64, 71, 75, 76, 77, 95], "1073741824": [45, 49, 64, 71, 75, 76, 77, 95], "dla_global_dram_s": [45, 49, 52, 64, 71, 75, 76, 77, 95], "536870912": [45, 49, 64, 71, 75, 76, 77, 95], "require_full_compil": [45, 49, 64, 71, 75, 76, 77, 95], "min_block_s": [45, 49, 56, 63, 64, 71, 75, 76, 77, 93, 94, 95, 96, 97, 100, 101, 102, 104, 108], "3": [45, 49, 52, 55, 56, 58, 63, 64, 65, 67, 68, 70, 71, 74, 76, 77, 82, 83, 86, 88, 89, 91, 92, 94, 95, 96, 97, 99, 100, 102, 105, 106, 107, 108, 109, 111, 113, 116, 118], "torch_executed_op": [45, 49, 56, 63, 64, 71, 75, 76, 77, 95, 100, 101, 102, 104], "torch_executed_modul": [45, 49, 56, 71, 76, 77], "member": [46, 47, 48, 49], "hold": [46, 47, 48, 53, 60, 76, 91], "relat": [46, 82, 101, 104], "let": [46, 52, 55, 60, 65, 71, 76, 77, 80, 82, 111, 112, 117], "layer": [46, 49, 52, 53, 55, 60, 62, 64, 65, 71, 75, 76, 77, 89, 91, 93, 95, 106, 107, 108, 111, 112, 113, 114, 117, 118], "thei": [46, 52, 53, 54, 55, 58, 60, 64, 65, 71, 74, 75, 76, 80, 82, 90, 93, 97], "complex": [47, 49, 64, 66, 88, 90, 99, 107], "either": [47, 48, 52, 60, 62, 71, 76, 77, 80, 82, 88, 89, 90, 93, 94, 95, 97, 116], "one": [47, 54, 55, 60, 64, 65, 67, 71, 75, 76, 82, 88, 89, 90, 93, 95, 101, 104, 106, 107, 112], "rang": [48, 49, 52, 65, 76, 95, 96, 97, 102, 109, 111, 113], "optim": [48, 52, 63, 64, 65, 69, 71, 72, 74, 76, 88, 89, 90, 100, 102, 103, 104, 109, 111, 113, 117], "profil": [48, 72, 75, 114], "singl": [48, 52, 55, 56, 65, 76, 82, 88, 89, 91, 109, 115], "repres": [48, 49, 54, 60, 65, 68, 82], "signifi": [48, 55], "static": [48, 49, 53, 60, 63, 64, 71, 76, 77, 80, 89, 108, 113], "three": [48, 57, 59, 65, 72, 76, 82, 83, 111, 112], "min": [48, 52, 60, 70, 76, 97, 102, 113], "optimin": 48, "max": [48, 52, 60, 70, 76, 80, 97, 102, 108, 113], "allow": [48, 49, 52, 53, 54, 55, 56, 62, 64, 65, 66, 71, 76, 77, 80, 93, 95, 97, 100, 102, 104, 109, 115], "argument": [48, 52, 54, 55, 58, 60, 62, 64, 65, 71, 75, 76, 77, 82, 83, 89, 90, 93, 94, 95, 113], "expect": [48, 54, 55, 60, 76, 89, 90, 111], "tradit": [48, 71, 76, 77, 91], "convect": 48, "produc": [48, 53, 54, 58, 60, 63, 76, 82, 89, 111], "low": [48, 65, 93, 99], "high": [48, 55, 56, 80, 93, 95, 117], "weight": [48, 49, 52, 53, 64, 65, 69, 70, 71, 76, 77, 82, 89, 97, 98, 99, 103, 110, 111], "first": [48, 53, 54, 55, 65, 68, 82, 83, 89, 90, 91, 93, 95, 97, 99, 101, 112, 113, 116, 117], "calcul": [48, 53, 56, 89, 95, 109], "detect": [48, 58, 76], "float32": [48, 49, 52, 63, 64, 65, 71, 76, 77, 95, 99, 103, 106, 107, 109, 113, 114], "dynam": [48, 49, 63, 65, 69, 71, 72, 76, 77, 78, 93, 97, 98, 101, 103, 104, 107, 109, 110, 115], "opt": [48, 66, 75, 76, 105], "minimum": [48, 49, 52, 56, 63, 64, 71, 76, 77, 95, 109], "maximum": [48, 49, 52, 64, 65, 71, 72, 76, 77, 102, 104, 109, 112], "accept": [48, 52, 54, 58, 60, 66, 76, 89, 90, 101, 116], "exampl": [48, 56, 58, 59, 60, 65, 66, 71, 73, 75, 76, 77, 78, 80, 81, 83, 86, 88, 89, 90, 91, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116], "s": [48, 49, 53, 56, 58, 60, 63, 65, 66, 67, 69, 71, 72, 75, 76, 80, 82, 83, 88, 89, 91, 93, 95, 97, 109, 111, 112, 113, 115, 116], "cannot": [48, 55, 56, 65, 66, 71, 75, 76, 77, 81, 88, 94, 95], "through": [48, 53, 54, 55, 56, 58, 64, 65, 71, 73, 74, 82, 89, 90, 95, 99, 100, 111, 117], "altern": [48, 56, 62, 63, 76, 90, 93, 105, 111, 116], "refer": [48, 54, 57, 59, 65, 81, 86, 89, 91, 95, 108, 112, 113, 116], "given": [48, 49, 52, 54, 55, 65, 71, 72, 74, 76, 77, 88, 89, 90, 92, 93, 113], "kernel": [48, 49, 52, 60, 64, 65, 69, 71, 76, 77, 93, 98, 110, 114, 115], "ani": [48, 52, 53, 54, 60, 62, 64, 65, 70, 71, 74, 75, 76, 77, 80, 82, 89, 90, 91, 93, 95, 102, 113], "event": [48, 64, 96, 97], "place": [48, 55, 62, 65, 82, 83, 84, 91, 95, 108], "variabl": [48, 65, 75, 76], "dimens": [48, 55, 65, 72, 76, 102, 111, 113, 114], "domain": [48, 76, 83, 91], "convien": 49, "fix": [49, 65, 82, 95, 115, 118], "describ": [49, 56, 60, 76, 88, 92, 112], "entri": [49, 60, 97], "okai": 49, "ha": [49, 53, 54, 55, 56, 57, 59, 60, 62, 64, 65, 66, 67, 71, 72, 76, 82, 83, 88, 89, 91, 93, 94, 97, 100, 108, 111, 113, 117], "flaten": 49, "precis": [49, 52, 63, 64, 65, 69, 71, 76, 89, 90, 91, 102, 104, 106, 107, 109, 118], "dure": [49, 52, 54, 56, 60, 63, 64, 71, 74, 76, 91, 93, 106, 107, 109, 111, 113, 115], "prevent": [49, 52, 54, 56], "tf32": [49, 52, 64, 71], "comput": [49, 64, 65, 66, 67, 71, 75, 82, 91, 94, 98, 110, 111], "inner": [49, 83, 111], "product": [49, 67, 76], "round": [49, 71, 76, 77, 95], "10": [49, 66, 67, 71, 72, 76, 77, 86, 88, 89, 91, 108, 109, 111, 112, 113, 114], "bit": [49, 60, 65, 66, 71, 76, 77, 89], "mantissa": [49, 71, 76, 77], "befor": [49, 54, 55, 56, 59, 60, 65, 71, 76, 77, 89, 112, 113], "multipli": [49, 71, 76, 77], "accumul": [49, 64, 71, 76, 77, 106, 107], "sum": [49, 65, 70, 71, 76, 77, 95, 108], "23": [49, 55, 71, 76, 77, 83], "behavior": [49, 56, 65, 71, 76, 77, 93, 106, 107, 113, 115, 116], "sparsiti": [49, 52, 65, 71, 76, 77], "conv": [49, 52, 89, 95], "fc": [49, 52, 55], "truncat": [49, 52, 63, 64, 71, 76, 77], "long": [49, 52, 53, 63, 76, 82, 83], "float": [49, 52, 63, 64, 70, 76, 88, 89, 90, 91, 92, 95, 96, 97, 100, 101, 104, 105, 114], "ishap": 49, "restrict": [49, 64, 71, 76, 77, 113], "cuda": [49, 58, 63, 65, 67, 68, 71, 72, 75, 76, 89, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 102, 103, 105, 106, 107, 108, 109, 112, 113, 114, 115, 116], "safeti": [49, 52, 76], "averag": [49, 52, 64, 71, 76, 77, 95], "time": [49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 64, 65, 66, 68, 69, 71, 72, 75, 76, 77, 80, 82, 89, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "workspac": [49, 52, 64, 65, 66, 71, 72, 76, 77, 95, 101, 102, 104], "fast": [49, 52, 64, 68, 71, 76, 77], "softwar": [49, 52, 64, 71, 76, 77, 82], "manag": [49, 52, 53, 55, 57, 59, 60, 64, 66, 67, 71, 73, 75, 76, 77, 89, 105, 115], "ram": [49, 52, 64, 71, 76, 77], "commun": [49, 52, 64, 71, 76, 77, 89], "within": [49, 52, 57, 59, 64, 69, 71, 75, 76, 77, 80, 82, 98, 106, 107, 110, 111], "host": [49, 52, 64, 66, 71, 76, 77, 95, 109, 112], "share": [49, 52, 64, 66, 71, 75, 76, 77, 97], "across": [49, 52, 55, 56, 64, 71, 76, 77, 80], "metadata": [49, 52, 54, 58, 60, 64, 71, 76, 77, 80, 100, 113, 114], "quantizatiom": 49, "instead": [49, 52, 53, 54, 55, 66, 71, 75, 76, 89, 93, 100, 108, 115], "potenti": [49, 71, 76, 85], "subgraph": [49, 52, 53, 54, 55, 60, 62, 89, 95, 97, 117], "aten": [49, 54, 55, 56, 60, 61, 64, 69, 70, 71, 76, 77, 89, 93, 101, 117], "thrown": [49, 71, 76, 77], "empti": [49, 71, 72, 76, 77, 83, 88, 95], "torch_tensorrtnamespac": 50, "loggingenum": 50, "levelnamespac": 50, "ptqtemplat": 50, "int8cachecalibratortempl": 50, "int8calibratornamespac": 50, "torchscriptstruct": 50, "compilespecstruct": 50, "deviceclass": 50, "devicetypestruct": 50, "graphinputsstruct": 50, "inputclass": 50, "datatypeclass": 50, "tensorformatenum": 50, "cppdirectori": 50, "includedirectori": 50, "torch_tensorrtfil": 50, "hfile": 50, "relationship": 50, "inherit": [50, 65, 71, 91], "subdirectori": 51, "definit": [51, 54, 60, 82], "cli": [52, 90], "It": [52, 54, 55, 56, 57, 59, 60, 65, 66, 69, 76, 80, 82, 94, 95, 109, 111, 115, 117], "serv": [52, 58, 65, 69, 71, 76], "easi": [52, 53, 55, 89, 91], "wai": [52, 64, 65, 66, 88, 89, 91, 93, 95, 97, 100, 111, 115, 116], "command": [52, 64, 66, 82, 83, 88, 89, 112], "line": [52, 66, 83, 89, 99], "quickli": [52, 89, 91], "part": [52, 56, 59, 65, 75, 80, 81, 82, 95, 97], "deploy": [52, 75, 89, 90, 91, 111, 112, 115, 118], "pipelin": [52, 89, 99, 103, 118], "basic": [52, 56, 65, 83, 112], "featur": [52, 56, 65, 66, 89, 91, 92, 103, 108, 109, 111, 117], "though": [52, 59, 60, 88, 89, 117], "alreadi": [52, 53, 54, 55, 89, 91, 93, 95, 98, 110, 113], "two": [52, 55, 60, 62, 64, 65, 66, 76, 82, 83, 87, 88, 90, 91, 93, 97, 112, 113], "embed": [52, 54, 58, 70, 77, 82, 118], "plan": [52, 59, 63, 64, 71], "after": [52, 53, 55, 56, 62, 65, 71, 75, 76, 88, 89, 90, 101, 104, 112, 115], "link": [52, 53, 62, 69, 80, 81, 86, 89, 95, 115], "against": [52, 89, 93], "libtorchtrt": [52, 66, 89], "python": [52, 56, 59, 62, 64, 65, 67, 71, 72, 75, 76, 77, 82, 83, 89, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 115, 118], "import": [52, 55, 56, 63, 64, 65, 66, 67, 68, 75, 80, 82, 88, 89, 90, 92, 93, 95, 96, 97, 99, 112, 113, 115, 116], "packag": [52, 55, 64, 67, 89], "aspect": 52, "ident": [52, 62, 71, 76, 100], "standard": [52, 58, 66, 69, 71, 75, 76, 77, 82, 92, 93, 95, 99, 111, 115], "load": [52, 56, 58, 64, 65, 68, 71, 74, 75, 76, 77, 89, 90, 91, 92, 94, 95, 96, 97, 99, 100, 109, 111, 112, 115, 117], "like": [52, 53, 55, 58, 60, 65, 66, 68, 76, 81, 82, 88, 89, 90, 91, 93, 95, 97, 99, 100, 109, 112, 115], "would": [52, 54, 60, 64, 65, 66, 67, 75, 89, 90, 92, 93, 95, 112, 115], "input_file_path": [52, 118], "output_file_path": [52, 118], "input_spec": [52, 65, 72], "displai": [52, 62, 64, 73, 80, 115], "menu": [52, 80, 82], "verbios": 52, "v": [52, 67, 83, 108, 112], "verbos": [52, 64, 65, 71, 72, 83, 102, 104], "about": [52, 53, 58, 60, 66, 75, 80, 89, 112, 113], "process": [52, 56, 64, 76, 81, 82, 88, 91, 92, 100, 101, 105, 111, 112, 115], "onto": [52, 58], "consol": 52, "w": [52, 66, 76], "disabl": [52, 64, 66, 71, 75, 80, 81, 93, 97, 109, 115], "i": [52, 55, 60, 66, 68, 70, 82, 83, 88, 89, 91, 95, 96, 97, 106, 108], "debugg": [52, 71, 76, 77], "fallback": [52, 57, 59, 60, 100, 118], "model": [52, 56, 58, 63, 68, 71, 72, 73, 74, 76, 78, 88, 89, 90, 91, 92, 96, 97, 99, 113, 115, 117], "throw": [52, 55, 76, 89], "spars": [52, 54, 64, 70, 71], "p": [52, 70, 89, 112, 118], "repeat": [52, 70], "f32": [52, 71, 75, 76, 95], "half": [52, 64, 76, 82, 89, 90, 91, 92, 95, 101, 102, 106, 107, 109, 112, 114, 118], "float16": [52, 76, 95, 99, 103, 114], "f16": [52, 76, 89, 118], "i8": [52, 76], "d": [52, 67, 76, 82, 83, 89, 118], "multi": [52, 75], "dlacor": 52, "avail": [52, 54, 60, 62, 64, 65, 66, 67, 71, 75, 76, 80, 95, 109, 111, 117, 118], "dla_standalon": [52, 76], "file_path": [52, 76, 94, 116], "teo": 52, "op_nam": 52, "op": [52, 53, 54, 55, 56, 57, 59, 60, 62, 63, 64, 75, 76, 89, 93, 101, 115, 117], "partial": [52, 82], "tem": 52, "module_nam": 52, "mod": [52, 56, 65, 71, 86, 89, 91, 114], "mb": [52, 78], "num_op": 52, "block": [52, 53, 55, 56, 64, 71, 86, 117], "treat": 52, "num": 52, "avg": 52, "num_it": 52, "sram": 52, "local": [52, 55, 66, 67, 80, 89], "dram": 52, "atol": 52, "absolut": [52, 66], "toler": 52, "threshold": 52, "numer": [52, 65, 83], "deviat": 52, "1e": [52, 99, 100], "rtol": 52, "rel": [52, 56], "5": [52, 56, 58, 59, 64, 65, 66, 67, 71, 75, 76, 82, 83, 86, 88, 89, 93, 95, 99, 101, 109, 112], "skip": 52, "complianc": 52, "64bit": [52, 94], "32bit": 52, "custom": [52, 62, 63, 65, 66, 69, 98, 106, 107, 110], "dll": 52, "n": [52, 60, 62, 76, 89, 91, 93, 95, 96], "min_n": 52, "min_c": 52, "min_h": 52, "min_w": 52, "opt_n": 52, "opt_c": 52, "opt_h": 52, "opt_w": 52, "max_n": 52, "max_c": 52, "max_h": 52, "max_w": 52, "32": [52, 76, 88, 89, 90, 91, 106, 107, 108, 118], "flag": [52, 56, 57, 59, 64, 66, 71, 74, 76, 90, 105, 106, 107, 115, 116], "forc": [52, 63, 65, 71, 76, 77, 80], "posit": [52, 54, 65, 76, 80], "test": [52, 56, 59, 65, 66, 67, 71, 76, 82, 83, 91, 108, 111, 112], "ssd_trace": 52, "pt": [52, 65, 89, 106, 107, 112], "ssd_trt": 52, "300": [52, 92], "512": [52, 71, 76, 77, 108, 111], "1024": [52, 71, 76, 77, 106, 111], "simplifi": [53, 95], "form": [53, 75, 76, 82, 90, 112], "up": [53, 55, 56, 57, 58, 59, 62, 65, 66, 71, 76, 82, 88, 93, 95, 97, 100, 101, 104, 109, 111], "context": [53, 57, 58, 59, 64, 73, 75, 93, 105, 115], "inetworkdefinit": [53, 54], "record": [53, 88, 96, 97, 105, 115], "togeth": [53, 60, 89], "start": [53, 56, 65, 70, 74, 76, 83, 89, 92, 95, 96, 97, 111], "look": [53, 54, 55, 68, 71, 76, 88, 91, 92, 93, 97, 112, 113], "assembl": [53, 62, 89], "resourc": [53, 91, 95], "coupl": [53, 59, 65, 115], "state": [53, 54, 60, 62, 75, 89, 93, 99], "been": [53, 60, 64, 66, 67, 74, 83, 89, 94, 97, 100, 117], "evaluated_value_map": [53, 60], "stage": [53, 65], "arg": [53, 54, 62, 65, 71, 74, 75, 76, 86, 89, 93, 94, 95, 97, 108, 111], "itensor": [53, 54, 60, 65, 89, 93, 95], "value_tensor_map": [53, 60], "typic": [53, 60, 76, 112], "abl": [53, 55, 60, 62, 65, 91, 92, 95, 100], "system": [53, 60, 62, 64, 69, 71, 75, 76, 77, 93, 94, 95, 97, 100, 117], "registri": [53, 54, 89, 95], "enter": [53, 76], "recurs": 53, "resolv": [53, 55, 57, 59, 101, 104], "until": [53, 56, 59, 60, 66, 71, 76, 117], "final": [53, 56, 57, 59, 66, 93, 95, 101, 104, 111], "some": [53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 65, 66, 76, 81, 82, 89, 91, 93, 95, 97, 113, 117], "These": [53, 54, 56, 58, 62, 64, 66, 71, 74, 75, 76, 80, 82, 91, 93, 112, 117], "those": [53, 54, 62, 64, 82], "do": [53, 54, 55, 56, 60, 63, 65, 81, 83, 88, 89, 90, 91, 93, 95, 106, 107, 118], "theori": [53, 82], "kind": [53, 65], "common": [53, 55, 65, 72, 82, 93, 97], "prim": [53, 55, 56, 58, 70, 88, 89], "constant": [53, 54, 55, 56, 89, 95], "emit": 53, "listconstruct": [53, 56, 58, 89], "make": [53, 54, 65, 66, 67, 71, 76, 82, 84, 89, 90, 91, 95, 97, 111, 112, 118], "associ": [53, 60, 89, 97, 115], "where": [53, 54, 55, 60, 62, 64, 65, 71, 75, 76, 77, 83, 89, 91, 93, 100], "result": [53, 55, 56, 66, 68, 71, 73, 75, 76, 77, 80, 88, 90, 94, 95, 99, 100, 109, 112, 114, 117], "done": [53, 56, 59, 95, 100, 112, 116], "mai": [53, 54, 56, 58, 59, 65, 66, 71, 75, 76, 77, 82, 83, 88, 89, 90, 91, 93, 95, 100, 101, 104, 109, 112, 115], "For": [53, 56, 62, 63, 64, 65, 66, 68, 72, 76, 80, 82, 83, 88, 89, 91, 92, 93, 95, 99, 101, 108, 111, 112, 115, 116], "more": [53, 64, 65, 66, 67, 69, 71, 76, 80, 83, 88, 89, 90, 91, 92, 95, 97, 99, 102, 104, 112, 115], "writing_convert": [53, 89], "locat": [54, 62, 66, 91, 93, 95], "py": [54, 55, 59, 62, 65, 66, 67, 78, 80, 82, 87, 88, 89, 91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 113], "convers": [54, 55, 56, 58, 63, 64, 65, 71, 76, 77, 89, 93, 95, 111, 113], "decror": 54, "dynamo_tensorrt_convert": [54, 93, 95], "signatur": [54, 77], "leaky_relu": [54, 70], "def": [54, 62, 65, 82, 88, 90, 93, 95, 96, 97, 101, 108, 109, 112, 113, 114], "leaky_relu_convert": 54, "ctx": [54, 60, 89, 93, 95, 109], "conversionctx": [54, 60, 89, 93], "tupl": [54, 58, 63, 65, 71, 72, 75, 76, 77, 90, 93, 95, 97, 100, 113, 114], "kwarg": [54, 65, 71, 74, 75, 76, 93, 95, 111], "dict": [54, 71, 75, 76, 77, 93, 95, 97], "union": [54, 60, 64, 71, 75, 76, 77, 89, 93], "sequenc": [54, 62, 65, 71, 72, 75, 76, 77, 82, 93, 95, 109, 111], "decor": [54, 62, 65, 93], "kei": [54, 82, 88, 97, 112, 113], "node": [54, 55, 56, 57, 59, 60, 62, 64, 65, 71, 72, 89, 93, 95, 108, 111, 113], "capability_valid": [54, 93], "lambda": [54, 60, 82, 89, 93, 112], "fx": [54, 62, 63, 71, 75, 76, 89, 90, 93, 95, 100, 116], "determin": [54, 55, 64, 65, 76, 93, 109, 113, 115], "properli": [54, 66], "handl": [54, 55, 56, 58, 64, 65, 75, 76, 95], "partition": [54, 71, 76, 95], "sure": [54, 66, 67, 89, 90, 112, 118], "prioriti": [54, 93], "develop": [54, 65, 66, 67, 69, 82, 83, 89, 93, 95], "bodi": [54, 82, 83], "nativ": [54, 59, 61, 89, 93, 95, 100], "numpi": [54, 76, 95, 96, 97, 99, 100, 109, 112], "frozen": 54, "attribut": [54, 55, 56, 58, 65, 76, 82, 89], "previou": [54, 80, 101], "correspond": [54, 60, 65, 66, 75, 76, 93, 97, 99, 108, 115], "edg": [54, 82], "well": [54, 63, 66, 69, 73, 75, 82, 89, 91, 93, 97, 105, 116], "being": [54, 65, 66, 71, 89, 93, 95, 100], "truth": 54, "http": [54, 61, 64, 66, 67, 80, 82, 88, 89, 91, 93, 95, 99, 101, 104, 108, 111, 112, 113, 115], "github": [54, 61, 64, 66, 67, 80, 89, 91, 101, 104, 108, 112, 115], "com": [54, 61, 64, 66, 67, 89, 91, 99, 101, 104, 108, 112, 115], "blob": [54, 61, 66, 80, 91, 97], "main": [54, 55, 56, 57, 58, 59, 60, 63, 65, 66, 80, 82, 84, 89, 93, 95, 106, 108], "src": [54, 58, 61, 70], "native_funct": [54, 61], "yaml": [54, 61], "sinc": [54, 55, 64, 65, 67, 75, 82, 88, 89, 91, 93, 96, 97, 100], "mani": [54, 56, 64, 65, 80, 82, 83, 93, 97, 100, 117], "composit": [54, 89], "raw": [54, 80, 93], "impl": [54, 93], "subpackag": 54, "chain": [54, 60], "primarili": [54, 59, 66, 89, 93], "manipul": [54, 62, 76], "net": [54, 60, 82, 83, 89, 95], "addit": [54, 55, 64, 65, 75, 76, 89, 93, 95, 97, 100, 111, 113], "call_modul": 54, "call_funct": [54, 62, 65], "eg": [54, 112, 114], "aten_": 54, "_leaky_relu": 54, "opoverloadpacket": 54, "while": [54, 56, 66, 75, 91, 93, 99, 109, 111, 112, 115, 117], "opoverload": 54, "particular": [54, 64, 97], "collect": [54, 56, 64, 71, 76, 77, 89, 90, 108], "trtinterpret": [54, 65, 72], "along": [54, 76], "match": [54, 55, 93, 100], "special": [54, 56], "account": [54, 112], "illustr": [54, 65, 102, 106, 107, 111], "scale_grad_by_freq": [54, 70], "embedding_param_valid": 54, "establish": 54, "subset": [54, 64, 71, 76, 91, 111], "converter_util": [54, 95], "enforce_tensor_typ": 54, "dictionari": [54, 76, 77, 92, 101], "between": [54, 55, 56, 60, 66, 76, 82, 83, 91, 97, 99, 106, 109], "possibl": [54, 66, 82, 93, 95, 97, 111, 112], "prefer": [54, 64, 66, 89], "keyword": [54, 62, 71, 75, 76, 77, 93, 101, 104], "both": [54, 56, 64, 66, 69, 71, 72, 75, 76, 80, 82, 88, 91, 93, 95, 97], "enforc": [54, 89], "situat": 54, "partit": [54, 55, 63, 64, 71, 76, 93, 117], "greater": [54, 71, 73, 76], "than": [54, 55, 64, 66, 71, 76, 81, 82, 93, 96, 97, 99, 109, 111, 115], "3d": [54, 65], "autocast": 54, "therebi": [54, 58, 95, 111], "limit": [54, 55, 73, 81, 91, 94, 97, 98, 109, 110, 117], "author": [54, 83], "conv_nod": 54, "7": [54, 56, 58, 59, 75, 76, 86, 89, 95, 101, 102, 104, 108, 113], "ignor": [54, 71, 75, 76, 95], "misc": [54, 95], "trttensor": 54, "np": [54, 93, 95, 96, 97, 99, 100, 109, 112], "ndarrai": [54, 95], "aten_ops_convolut": 54, "conversioncontext": [54, 93, 95], "side": [54, 55, 80, 89, 93], "effect": [54, 55, 64, 65, 71, 80, 89, 91, 93, 95, 111], "term": [54, 76, 82, 83, 91, 93, 95, 111], "getitem": 54, "categor": 54, "modif": [54, 62, 76], "op_evalu": 54, "capbility_valid": 54, "opcod": 54, "decompos": 54, "suboper": 54, "separ": [54, 56, 57, 59, 66], "Such": 54, "via": [54, 64, 65, 67, 69, 71, 75, 76, 77, 80, 86, 90, 91, 101, 102, 104, 106, 107, 111, 113, 115, 116, 117], "register_torch_trt_decomposit": 54, "addmm_replac": 54, "replac": [54, 56, 62, 66, 67, 74, 95, 108, 117], "input_": 54, "mat1": 54, "mat2": [54, 70], "beta": [54, 65, 70, 77], "alpha": [54, 65, 70, 83], "mul": [54, 56, 70, 93], "matmul": [54, 55, 64, 70, 71, 89, 106, 107, 113], "modifi": [54, 56, 62, 65, 83, 99, 113], "edit": [54, 66, 80], "torch_enabled_decomposit": 54, "torch_disabled_decomposit": 54, "disjoint": 54, "preced": [54, 82], "over": [54, 57, 59, 65, 82, 108, 109, 112, 117], "much": [54, 60, 80, 82, 91], "significantli": [54, 55, 80, 97], "easier": [54, 57, 59, 60, 65, 71, 75, 76, 89, 91, 95, 99], "tri": 54, "made": [55, 57, 59, 76, 82], "represent": [55, 60, 65, 88, 111, 117], "instanc": [55, 62, 64, 66, 71, 74, 75, 88, 89, 93, 111, 115], "idea": [55, 82, 93], "reduc": [55, 56, 57, 59, 65, 71, 76, 91, 95, 97, 111, 115], "actual": [55, 58, 60, 65, 88, 89, 95], "aim": [55, 117], "closer": 55, "scope": [55, 95, 101, 104], "csrc": [55, 61], "common_subexpression_elimin": 55, "subexpress": 55, "dead_code_elimin": 55, "exception_elimin": 55, "wa": [55, 58, 62, 64, 65, 71, 75, 76, 82, 89, 93, 94, 117], "1013": 55, "ne": [55, 70], "1012": 55, "24": [55, 67], "lib": [55, 66, 67, 89], "python3": [55, 66, 89], "site": [55, 66, 82, 89], "nn": [55, 61, 65, 71, 72, 75, 76, 77, 88, 89, 90, 93, 95, 101, 108, 113, 114, 117], "batchnorm": 55, "248": 55, "11": [55, 66, 82, 86, 89, 112], "block0": 55, "raiseexcept": 55, "249": 55, "12": [55, 56, 67, 82, 86, 88, 89, 102, 112, 113], "block1": 55, "guard_elimin": 55, "whose": [55, 65, 102], "freeze_modul": 55, "propag": 55, "fuse_addmm_branch": 55, "variant": [55, 115], "caught": 55, "ret": 55, "622": 55, "self": [55, 58, 60, 70, 75, 76, 88, 89, 90, 93, 95, 97, 101, 108, 111, 113, 114, 118], "bia": [55, 70, 89, 108], "x9": 55, "3677": 55, "output0": [55, 114], "add_": [55, 70, 89, 93], "fuse_linear": 55, "back": [55, 56, 58, 59, 75, 76, 82, 88, 89, 95, 117], "fuse_flatten_linear": 55, "implicitli": [55, 76], "connect": [55, 71, 76, 77, 82, 99, 112, 118], "higher": [55, 64, 71, 76, 80, 82, 88, 109], "1d": 55, "lower_graph": 55, "access": [55, 60, 65, 80, 89, 92, 117], "rather": 55, "getattr": [55, 58, 88, 89], "trainabl": 55, "remain": [55, 76, 91, 117], "lower_tupl": 55, "lowersimpletupl": 55, "tupleconstruct": [55, 58], "tupleunpack": 55, "leav": [55, 62, 64, 71], "statement": [55, 82, 93], "loweralltupl": 55, "_all_": 55, "rais": [55, 65, 76, 94], "onnx": 55, "module_fallback": 55, "consist": [55, 65, 82, 95, 115, 117], "pair": [55, 60, 66, 82, 91, 111], "delimit": 55, "around": [55, 58, 60, 64, 66, 71, 75, 82, 85, 88, 95], "second": [55, 65, 82, 90, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "mark": [55, 56, 80, 97], "notatemoduleforfallback": 55, "marknodesforfallback": 55, "tell": [55, 56, 57, 58, 59, 60, 82, 117], "them": [55, 56, 58, 63, 64, 65, 66, 71, 75, 80, 89, 95, 97, 107, 111, 113, 117], "peephole_optimz": 55, "intent": [55, 82], "catch": [55, 76, 89], "small": [55, 95, 96, 112], "might": [55, 66, 80, 100, 113], "interest": [55, 82], "now": [55, 56, 59, 60, 65, 66, 76, 82, 89, 92, 93, 95, 97, 100, 109, 114, 115], "expand": [55, 70], "simpli": [55, 101, 111], "remove_contigu": 55, "remove_dropout": 55, "infer": [55, 64, 65, 71, 76, 77, 89, 91, 94, 100, 101, 109, 111, 113, 115, 116, 117], "remove_to": 55, "unpack_addmm": 55, "reus": [55, 65, 91, 97], "dedic": [55, 83], "unpack_log_softmax": 55, "softmax": [55, 65, 70, 108], "loop_unrol": 55, "suffici": [55, 66, 76], "short": [55, 64, 71, 82, 83, 100], "tile_to_repeat": 55, "instruct": [56, 57, 59, 65, 66, 89, 112], "criteria": [56, 57, 59, 64], "lack": [56, 57, 59, 65, 95, 109], "explicitli": [56, 57, 59, 66, 77, 90, 91, 92, 106, 107, 114], "On": 56, "segment": [56, 63, 95, 102, 104, 111], "verifi": [56, 71, 93, 95, 100], "Then": [56, 91, 92, 100], "roughli": 56, "analysi": 56, "everi": [56, 72, 75, 76, 89, 115], "complet": [56, 63, 71, 76, 88, 89], "mean": [56, 60, 64, 65, 70, 71, 72, 101, 109, 112, 117], "trace": [56, 65, 71, 75, 77, 88, 89, 113, 116, 117], "tensorlist": [56, 60], "figur": [56, 83, 85], "our": [56, 59, 63, 88, 89, 112], "stitch": [56, 89], "altogeth": [56, 80], "brief": 56, "descript": [56, 83, 94, 108], "partitioninfo": 56, "api": [56, 59, 60, 62, 63, 64, 65, 75, 76, 77, 81, 89, 90, 91, 92, 95, 101, 102, 105, 109, 111, 112, 113, 115, 116], "maintain": [56, 58, 60, 76, 99, 117], "code": [56, 59, 62, 64, 65, 66, 81, 83, 88, 89, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 113], "mymodel": [56, 63, 68, 90, 95, 113, 116], "ts_model": [56, 89], "trt_model": [56, 92, 95, 102, 106, 107, 108, 109, 112, 116], "off": [56, 58, 105], "consecut": [56, 63], "satisfi": [56, 62, 65], "forced_fallback_op": 56, "randn": [56, 63, 68, 71, 76, 77, 89, 92, 93, 97, 102, 105, 113, 114, 116], "224": [56, 63, 68, 71, 72, 76, 77, 89, 94, 97, 99, 100, 102, 105, 111, 112, 113, 116], "trt_ts_modul": [56, 90], "input_s": 56, "inputrang": 56, "cfg": [56, 89], "relu": [56, 70, 88, 89, 101, 108], "trt_mod": [56, 68, 89, 91, 118], "consid": [56, 77, 89, 95, 114], "segmentmodelwithdependencyawar": 56, "test_segment": 56, "20": [56, 67, 86, 100, 102, 104], "x_lgamma": 56, "lgamma": 56, "y_lgamma": 56, "div": [56, 70], "div_lgamma": 56, "27": [56, 89], "cat": [56, 66, 67, 70, 108, 109], "greedi": [56, 106, 107, 109], "strategi": [56, 76], "travers": [56, 57, 59, 64], "gather": 56, "same": [56, 58, 62, 64, 65, 66, 71, 76, 80, 82, 88, 89, 92, 94, 95, 97, 100, 102, 104, 112, 113, 115, 116], "encount": [56, 64, 66, 93, 101, 104], "4": [56, 58, 63, 64, 65, 66, 70, 76, 78, 80, 82, 83, 86, 89, 95, 101, 103, 104, 105, 108, 113], "suboptim": 56, "arithmet": 56, "split": [56, 65, 70], "own": [56, 60, 64, 66, 71, 82, 89, 97, 108, 112], "could": [56, 64, 65, 95, 102, 104, 115], "rewrit": [56, 62], "portion": [56, 82, 95, 103], "without": [56, 60, 68, 71, 80, 82, 89, 91, 95, 96, 97, 100, 115], "reorder": 56, "seri": 56, "cleanli": 56, "approach": [56, 97], "achiev": [56, 111], "hit": 56, "larger": [56, 71, 76, 80, 109, 111], "boundari": [56, 74, 76], "guarante": [56, 75], "trigger": [56, 64, 65, 76, 89, 97, 99, 100, 117], "appear": [56, 82], "adjac": [56, 71, 76, 82], "As": [56, 65, 66, 76, 89, 93, 95, 97, 100, 117], "clean": [56, 62, 82, 101, 104], "step": [56, 65, 67, 70, 76, 91, 95, 100, 111], "consolid": [56, 88], "further": [56, 64, 65, 115, 117], "merg": 56, "identifi": 56, "do_not_merg": 56, "combin": [56, 64, 65], "condit": [56, 82, 117], "loop": [56, 64, 65, 106, 107], "ir": [57, 59, 60, 63, 64, 68, 71, 76, 88, 89, 90, 98, 101, 102, 104, 105, 110, 113], "larg": [57, 59, 80, 82, 89, 91, 100, 109, 111], "opset": [57, 59, 93], "compon": [57, 59, 66, 67, 74, 88, 115, 117], "evalu": [57, 58, 59, 108], "deploi": [57, 59, 69, 89, 91, 98, 110, 112], "instanti": [57, 58, 59, 60, 89, 103], "wrap": [57, 58, 59, 65, 82, 85, 89, 92, 101, 104], "extend": [57, 59, 60, 70, 89, 97, 111], "providi": [57, 59], "stand": [58, 82], "interpret": [58, 65, 82], "execute_engin": [58, 75, 89], "stack": [58, 70, 91, 108, 117], "machin": [58, 66, 91, 94, 112], "pop": 58, "push": 58, "element": [58, 65, 82, 83, 86], "realiz": 58, "abstract": [58, 60, 83, 93], "__torch__": [58, 88, 89], "portabl": [58, 66, 77], "serializ": [58, 64, 88, 117], "instnanti": 58, "whatev": [58, 65, 95], "self_1": [58, 89], "torchvis": [58, 91, 92, 94, 97, 99, 100, 102, 105, 108, 112], "resnet": [58, 69, 78, 94, 98, 99, 110, 111, 112], "___torch_mangle_4847": 58, "resnet_trt": 58, "input_0": [58, 89], "__torch___torchvision_models_resnet____torch_mangle_4847_resnet_trt_engin": 58, "listunpack": [58, 89], "multipl": [58, 66, 71, 75, 76, 82, 83, 91, 109, 112, 115], "repack": 58, "ssd": 58, "ssd300_trt": 58, "__torch___pytorch_detection_ssd_src_model_ssd300_trt_engin": 58, "holder": [58, 84], "torchbind": 58, "pickler": 58, "seril": 58, "zip": [58, 66, 99, 100, 110], "depickl": 58, "encod": [58, 111], "sm": 58, "correct": [58, 66, 80, 98, 99, 100, 108, 110], "bazel": [59, 66, 67], "linux": [59, 64, 67, 71, 89, 94], "x86_64": [59, 66], "aarch64": 59, "gcc": [59, 89], "untest": 59, "try": [59, 76, 82, 83, 89, 92, 95, 97, 117], "older": 59, "repositori": [59, 66, 80, 87, 112], "notebook": [59, 69, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110], "doc": [59, 61, 66, 67, 80, 81, 82, 87, 93, 95, 113], "docsrc": 59, "third_parti": [59, 66], "toolchain": [59, 66, 67], "unstabl": 59, "subject": [59, 62, 117], "matur": 59, "most": [59, 65, 66, 72, 95, 100, 112, 115, 117], "hood": [59, 102, 117], "major": [59, 65, 76], "top": [59, 80, 84], "coordin": [59, 76], "ingest": 59, "flow": [60, 65, 82, 88, 111], "ilay": 60, "analogu": 60, "goal": [60, 64, 97], "registernodeconversionpattern": [60, 89], "helper": [60, 93], "pattern": [60, 76, 89, 109], "schema": [60, 89, 93, 95], "caus": [60, 64, 80, 101, 102, 104, 109, 115], "acthardtanh": 60, "torchtrt_unus": 60, "hardtanh": [60, 70], "scalar": [60, 70], "min_val": [60, 70], "max_val": [60, 70], "unwraptodoubl": 60, "new_lay": 60, "addactiv": 60, "activationtyp": [60, 65], "kclip": 60, "torchtrt_check": 60, "unabl": [60, 89, 95], "setalpha": 60, "setbeta": 60, "setnam": [60, 89], "util": [60, 62, 74, 77, 89, 91, 101, 104, 106, 107, 108, 109, 111, 112, 117], "node_info": [60, 89], "c_str": [60, 89], "out_tensor": [60, 89], "associatevalueandtensor": [60, 89], "getoutput": [60, 89], "log_debug": 60, "getdimens": [60, 89], "accord": [60, 64, 77, 94], "unwrap": 60, "tool": [60, 64, 65, 66, 89, 93, 97, 111], "don": [60, 65, 80, 82, 83, 91, 93, 108, 112, 113], "annot": [60, 89], "your": [60, 63, 64, 66, 67, 68, 75, 80, 82, 83, 87, 88, 89, 90, 92, 97, 113, 115], "Its": [60, 82], "track": [60, 91], "sort": [60, 70, 92], "live": [60, 82], "directli": [60, 62, 63, 66, 69, 74, 76, 91, 93, 95, 101, 116], "associatevalueandivalu": 60, "inspect": [60, 88, 89], "dataflow": [60, 89], "mechan": [60, 64, 65, 95, 100, 111], "safe": [60, 64, 71, 75, 76, 77], "unsur": 60, "deep": [60, 64, 69, 80, 91, 95, 118], "straight": 60, "chanc": 60, "none": [60, 64, 65, 70, 71, 72, 74, 75, 76, 77, 80, 82, 93, 95, 97, 101, 108, 109], "wrapper": [60, 65, 116], "similar": [60, 63, 64, 65, 66, 89, 92, 95, 106, 107], "tocustomclass": 60, "tensorcontain": 60, "istensor": 60, "iscustomclass": 60, "lot": [60, 63], "singular": 60, "becaus": [60, 65, 66, 72, 88, 89, 93, 95, 96, 97, 109, 114], "alloc": 60, "freed": 60, "destructor": 60, "destroi": [60, 83], "realli": 60, "think": [60, 82], "becom": [60, 66, 99], "benefit": [60, 89, 97, 109], "deal": [60, 97], "quit": [60, 66, 89, 111], "effici": 60, "batch_norm": [60, 70], "fusion": [60, 62, 65], "deeplearn": [61, 65, 67], "sdk": [61, 67, 117], "matrix": 61, "html": [61, 66, 67, 82, 88, 91, 93, 95, 113], "c_api": 61, "python_api": 61, "org": [61, 66, 80, 82, 88, 89, 91, 93, 95, 113, 115], "stabl": [61, 67, 69, 77, 78, 80, 98, 110, 113], "master": [61, 66, 91, 115], "overview": [61, 69, 101, 105], "md": 61, "appli": [62, 63, 91, 100], "desir": [62, 71, 83, 91, 97], "coalesc": 62, "insert": [62, 64, 71, 89, 91, 93, 97, 100], "graphmodul": [62, 63, 71, 72, 76, 89, 90, 95, 100, 116, 117], "caller": 62, "invok": [62, 64, 65, 88, 89, 115], "lint": 62, "recompil": [62, 71, 76, 93, 97, 100, 104, 113, 117], "repair": 62, "disallow": 62, "repair_input_as_output": 62, "gm": [62, 71], "sample_input": [62, 65, 101], "scenario": [62, 64, 99, 109], "clone": [62, 66, 70, 95], "modified_graph": 62, "extract": [62, 89, 111], "placehold": [62, 93], "isinst": [62, 65, 95, 108], "issubclass": 62, "direct": [62, 86, 100, 115], "len": [62, 70, 95], "direct_output": 62, "inserting_aft": 62, "cloned_placehold": 62, "replace_input_with": 62, "date": [62, 83, 117], "eliminate_dead_cod": 62, "logger": [62, 73], "f": [62, 64, 65, 67, 76, 82, 88, 93, 94, 95, 108, 109], "__init__": [62, 75, 76, 82, 88, 93, 95, 97, 101, 108, 113, 114], "pass_manag": 62, "passmanag": 62, "backend": [62, 68, 69, 77, 78, 81, 92, 96, 97, 98, 101, 108, 110, 113], "offer": [62, 64], "registr": [62, 65], "conveni": [62, 91, 104, 111, 115, 117], "control": [62, 65, 88, 100, 109, 115], "_aten_lowering_pass": 62, "my_custom_pass": 62, "front": [62, 71], "passlist": 62, "arbitrari": [62, 75], "remov": [62, 63, 71, 80, 96, 97, 108], "dump_lowering_pass": 62, "apply_lowering_pass": 62, "graph_modul": [62, 71], "_remove_lowering_pass": 62, "evolv": 62, "introduc": [63, 65, 111], "exportedprogram": [63, 68, 71, 76, 100, 106, 107, 109, 113, 117], "dynamo": [63, 64, 66, 68, 74, 75, 76, 78, 89, 93, 94, 95, 96, 97, 98, 100, 101, 102, 104, 105, 108, 109, 110, 113, 114], "frontend": [63, 71, 74, 90, 95, 98, 102, 104, 108, 110, 113], "simpl": [63, 64, 65, 82, 83, 88, 111, 112, 113], "usag": [63, 65, 69, 74, 78, 82, 89, 98, 109, 110, 113, 116], "eval": [63, 68, 89, 90, 93, 94, 96, 97, 99, 100, 101, 102, 104, 105, 106, 107, 108, 109, 112, 113, 114, 116], "exp_program": [63, 97, 100, 108, 113], "trt_gm": [63, 68, 97, 100, 113, 114, 116], "interact": [63, 82, 99, 101, 102, 103, 104, 105], "ideal": 63, "discuss": [63, 64, 112], "section": [63, 65, 80, 82, 83, 84, 86, 89, 91, 112, 116], "frequent": 63, "builder": [63, 64, 65, 71], "respect": [63, 64, 66, 71, 76, 106, 107, 114], "releas": [63, 64, 67, 82], "insid": [63, 82, 95, 112], "decomposit": [63, 64, 71, 76, 95], "downstream": [63, 111], "constraint": [63, 109], "guid": [64, 81], "present": [64, 100], "learn": [64, 66, 69, 89, 91, 95, 112, 118], "acceler": [64, 72, 76, 115, 117, 118], "workflow": [64, 65, 68, 69, 71, 72, 76, 89, 92, 97, 98, 99, 102, 103, 104, 106, 107, 110, 111], "wide": [64, 76, 86], "varieti": [64, 112], "primari": [64, 93, 97, 116], "simplic": 64, "optimized_model": [64, 68, 96, 101, 102, 104], "depth": [64, 80, 111], "challeng": [64, 99, 112], "addition": [64, 95], "fit": [64, 82], "compilationset": [64, 71, 75, 93, 95, 101], "_enum": [64, 71], "callabl": [64, 71, 76], "pass_through_build_failur": [64, 71, 75, 76, 95], "max_aux_stream": [64, 71, 75, 76, 95], "version_compat": [64, 71, 75, 76, 95], "optimization_level": [64, 71, 75, 76, 95, 101], "use_python_runtim": [64, 71, 75, 76, 95, 96, 97, 99, 100, 101], "truncate_doubl": [64, 71, 75, 76, 95, 96, 106, 107, 109], "use_fast_partition": [64, 71, 75, 76, 95], "enable_experimental_decomposit": [64, 71, 75, 76, 95], "_devic": [64, 71], "assume_dynamic_shape_support": [64, 71, 75, 76], "make_refitt": [64, 71, 75, 76, 96, 97, 99, 100], "engine_cap": [64, 71, 75, 76, 95], "dryrun": [64, 71, 75, 76, 95], "hardware_compat": [64, 71, 75, 76, 95], "timing_cache_path": [64, 71, 75, 76, 97], "tmp": [64, 71, 75, 76, 89, 96], "torch_tensorrt_engine_cach": [64, 71, 75, 76], "timing_cach": [64, 65, 71, 75, 76], "bin": [64, 66, 67, 71, 75, 76], "lazy_engine_init": [64, 71, 75, 76], "cache_built_engin": [64, 71, 75, 96, 97], "reuse_cached_engin": [64, 71, 75, 96, 97, 100], "use_explicit_typ": [64, 71, 75, 106, 107, 109, 114], "use_fp32_acc": [64, 71, 75, 106, 107], "enable_weight_stream": [64, 71, 75, 109], "enable_cross_compile_for_window": [64, 71, 75], "dpython": [64, 71, 76, 77], "per": [64, 71, 95, 115], "regardless": [64, 71, 83, 102, 104], "fail": [64, 71, 76, 89, 99, 100, 108, 118], "auxiliari": [64, 71], "stream": [64, 69, 71, 76, 95, 98, 110], "impli": [64, 71], "longer": [64, 66, 71, 76, 80, 94, 115], "search": [64, 69, 71, 76, 80], "strictli": [64, 71], "runtim": [64, 66, 68, 69, 71, 76, 89, 93, 98, 99, 101, 104, 105, 109, 110, 117], "presenc": [64, 71], "preferenti": [64, 71], "choos": [64, 65, 71, 88], "float64": [64, 71, 76, 77], "refitt": [64, 71, 97], "toggl": [64, 71, 76], "mode": [64, 65, 71, 75, 76, 90, 91, 93, 105, 108], "detail": [64, 65, 67, 71, 88, 89, 95, 97, 112, 115], "natur": [64, 71, 82], "architectur": [64, 66, 69, 71, 76, 94, 97, 111], "amper": [64, 71, 76], "newer": [64, 66, 71, 76], "storag": [64, 71, 91], "use_strong_typ": [64, 71], "strong": [64, 71, 82], "mix": [64, 69, 71], "happen": [64, 65, 71, 88, 99, 102, 113], "were": [64, 71, 95, 100, 115], "cross": [64, 71, 82, 98, 110], "window": [64, 71, 82], "sub": [64, 70, 82, 88, 101], "slate": 64, "futur": [64, 65, 71, 76, 77, 115], "occur": [64, 109], "first_output": 64, "subsequ": [64, 97], "second_output": 64, "session": [64, 68, 82, 97, 105], "point": [64, 66, 76, 80, 81, 82, 89, 108, 112], "cover": [64, 93], "benchmark": [64, 70], "automat": [64, 67, 76, 82, 89, 100, 113, 117], "vari": [64, 72, 109, 113], "distribut": [64, 67, 89, 91, 109, 115], "inf": 64, "dynamo_convers": 64, "contribut": 64, "demonstr": [64, 82, 83, 84, 91, 93, 95, 97, 99, 108, 111, 112], "break": [64, 65, 71, 75, 76, 82, 95, 107], "successfulli": [64, 94, 99, 100], "_dynamo": [64, 96, 97, 101, 102, 104, 113], "explain": [64, 65, 69], "veri": [64, 65, 83, 84, 91, 92, 106, 107, 112], "explan": [64, 65], "graph_break_count": 64, "furthermor": 64, "durat": [64, 82], "latter": [64, 75], "logic": [64, 65, 93], "guard": 64, "compos": [65, 88, 91, 93, 108, 112], "variou": [65, 118], "etc": [65, 80, 82, 95, 118], "environ": [65, 68, 112], "research": 65, "few": [65, 66, 76, 93], "nightli": 65, "lower_exampl": 65, "welcom": [65, 89], "finish": 65, "converison": 65, "pleas": [65, 67, 76, 82, 89, 98, 108, 110, 112, 113], "max_batch_s": [65, 72, 112], "2048": [65, 72], "max_workspace_s": [65, 72], "33554432": [65, 72], "explicit_batch_dimens": [65, 72], "lower_precis": [65, 72], "lowerprecis": [65, 72], "verbose_log": [65, 72], "timing_cache_prefix": [65, 72], "save_timing_cach": [65, 72], "cuda_graph_batch_s": [65, 72], "dynamic_batch": [65, 72], "turn": [65, 72, 105], "trtmodul": [65, 72], "otherwis": [65, 66, 72, 97, 115], "implicit": [65, 70, 72, 82], "config": [65, 66, 72, 112], "updat": [65, 66, 67, 71, 72, 76, 95, 100], "dim": [65, 70, 72, 95, 97, 108, 109, 112, 113], "fx2trt_exampl": 65, "acc_trac": 65, "come": [65, 66, 81, 95, 99, 112], "my_pytorch_model": 65, "build_model": 65, "prepar": [65, 112], "acc_mod": 65, "earli": [65, 100], "deprec": [65, 70], "continu": [65, 82, 115], "backward": [65, 75, 95, 117], "vision": [65, 98, 110, 112], "activ": [65, 75, 77, 82, 89, 91, 93, 111, 115, 118], "except": [65, 71, 76], "permut": [65, 70], "transpos": [65, 70, 113], "ll": [65, 97], "inputtensorspec": [65, 72, 76], "experiment": [65, 76, 77], "dataclass": [65, 101], "re": [65, 76, 82, 97, 99, 105, 115], "manual": [65, 76, 81, 82, 100, 109], "sampl": [65, 71, 82, 90, 91, 99, 100, 101, 102, 103, 104, 105, 106, 107, 112], "rand": [65, 89, 94, 97, 99, 100, 101], "from_tensor": [65, 76], "slightli": [65, 66, 95], "promis": 65, "optimize_target_shap": 65, "input_tensor_spec": 65, "shape_rang": [65, 72], "100": [65, 72, 95, 97, 108, 109], "accordingli": [65, 80, 113, 115], "trtinterpreterresult": [65, 72], "namedtupl": 65, "input_nam": [65, 72], "output_nam": [65, 72], "serialized_cach": [65, 72], "bytearrai": [65, 75, 77], "afford": 65, "temporari": [65, 97], "best": [65, 71, 76, 82, 99, 109, 114], "perforamnc": 65, "examin": 65, "suitabl": [65, 93], "force_fp32_output": 65, "strict_type_constraint": 65, "usual": [65, 66, 80], "unless": 65, "certain": [65, 66, 101, 106, 107, 109, 115], "algorithm_selector": 65, "profiling_verbos": 65, "trt_interpreter_result": 65, "64": [65, 76, 90, 107, 108, 113], "25": [65, 72, 89], "runtimeerror": [65, 108], "xxx": 65, "One": [65, 82, 83, 89, 111, 115], "reload_trt_mod": 65, "reload_model_output": 65, "far": [65, 82], "give": [65, 80, 82], "convtert": 65, "scheme": [65, 71, 76], "action": [65, 82], "tensort": [65, 117], "thing": [65, 66, 82], "compar": [65, 71, 76, 90, 100], "vanilla": 65, "mainli": 65, "builtin": 65, "purpos": [65, 111, 112], "acc_op": 65, "leverag": [65, 91], "power": [65, 82, 89, 109, 111], "goe": [65, 82], "whole": 65, "sigmoid": [65, 70], "tensorrt_convert": 65, "acc_ops_sigmoid": 65, "rest": [65, 82, 83], "input_v": [65, 93], "receiv": 65, "region": 65, "add_activ": 65, "get_output": [65, 95], "wherev": 65, "rememb": [65, 66], "mapper": 65, "todo": [65, 67, 80], "logist": 65, "down": [65, 66, 80, 107], "acc_norm": 65, "foo": [65, 82, 83], "register_acc_op": 65, "register_acc_op_map": 65, "this_arg_is_opt": 65, "op_and_target": 65, "arg_replacement_tupl": 65, "rule": [65, 66, 77], "third": [65, 83], "boolean": [65, 76, 93], "matter": [65, 95], "register_custom_acc_mapper_fn": 65, "design": [65, 74, 93, 99, 106, 109, 111, 118], "redund": 65, "throught": 65, "custom_mapp": 65, "_": [65, 82, 95, 108, 109, 114], "foo_kwarg": 65, "inserting_befor": 65, "foo_nod": 65, "meta": [65, 67, 86, 107, 109], "children": 65, "unit": [65, 76], "test_acc_trac": 65, "acc_op_convert": 65, "essenti": 65, "plugin": [65, 95], "yet": [65, 111], "folder": 65, "center": 66, "pypi": 66, "m": [66, 67, 83, 108], "pip": [66, 67, 98, 110, 112], "upload": [66, 112], "x86": [66, 115], "extra": [66, 75, 89, 95, 99], "url": [66, 80, 112], "download": [66, 67, 86, 91, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 112], "whl": [66, 67], "cu118": 66, "cu124": 66, "tarbal": [66, 89, 91], "easiest": [66, 95], "bazelisk": [66, 67], "bazelbuild": [66, 67], "export": [66, 67, 69, 71, 76, 97, 98, 100, 102, 106, 107, 108, 109, 110, 114, 116, 117], "bazel_vers": 66, "path_to_torchtrt_root": 66, "bazelvers": 66, "mkdir": 66, "cd": [66, 112], "curl": [66, 82], "fssl": 66, "o": [66, 82, 112], "dist": 66, "unzip": 66, "bash": 66, "sh": 66, "cp": [66, 67, 95], "usr": [66, 67], "driver": 66, "branch": [66, 67], "4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d": 66, "l46": 66, "pull": [66, 97, 112], "latest": [66, 67, 80], "l53c1": 66, "fact": 66, "reproduc": 66, "l71": 66, "http_archiv": 66, "build_fil": 66, "archiv": [66, 67], "sha256": 66, "strip_prefix": 66, "OR": 66, "TO": [66, 89], "gnu": 66, "tar": [66, 67, 82, 91], "gz": [66, 82, 83, 91], "ld_library_path": 66, "comment": [66, 82], "uncom": 66, "l114c1": 66, "l124c3": 66, "uv": 66, "astral": 66, "project": [66, 81, 86], "simpler": [66, 91], "wheel": [66, 67], "dep": 66, "lighter": 66, "executor": 66, "avoid": [66, 93, 95, 100, 113], "implic": 66, "python_onli": 66, "legaci": [66, 74], "mainten": 66, "torchdynamo": [66, 113, 117], "technolog": [66, 117], "exclud": [66, 95], "speed": [66, 97, 100], "no_torchscript": 66, "dbg": 66, "pre_cxx11_abi": 66, "complic": 66, "incompat": 66, "popular": [66, 81, 98, 106, 107, 110, 111], "ngc": [66, 67, 112], "tabl": [66, 86], "bdist_wheel": 66, "preinstal": 66, "forum": 66, "correctli": [66, 95], "declar": 66, "intend": [66, 101, 102, 103, 104, 105], "microsoft": 66, "2022": [66, 69], "open": [66, 111, 112], "app": 66, "x64": 66, "prompt": [66, 99, 103, 106, 107], "admin": 66, "privileg": 66, "launcher": 66, "chocolatei": 66, "navig": [66, 80], "ninja": 66, "setuptool": 66, "r": [66, 67, 82, 98, 110], "txt": [66, 67, 98, 110], "distutils_use_sdk": 66, "cuda_win": 66, "libtorch_win": 66, "tensorrt_win": 66, "non": [66, 76, 83, 85, 115], "similarli": [66, 97, 105, 115], "ci_workspac": 66, "win": 66, "tmpl": [66, 67], "torchtrtc": [66, 69, 118], "websit": 66, "finder": 66, "dcmake_module_path": 66, "doesn": [66, 82, 88, 89], "dtorch_dir": 66, "dtensorrt_root": 66, "choic": [66, 74], "b": [66, 70, 76, 83, 109, 112], "dcmake_build_typ": 66, "72048": 66, "jp_workspac": [66, 67], "new_local_repositori": 66, "sudo": [66, 67], "home": 66, "unlik": [66, 92], "libtorch_pre_cxx11_abi": 66, "shift": [66, 70, 82], "jetpack": 66, "jetpack_x": 66, "jetpack_5": 66, "drop": [66, 80, 108], "nvida": 67, "ofjetpack": 67, "With": [67, 80, 82, 89, 91, 97, 112], "incorpor": [67, 83], "cudnn": 67, "9": [67, 86, 89, 94, 95, 112], "dlfw": 67, "09": 67, "jetson": [67, 111], "framework": 67, "instal": [67, 69, 86, 89, 98, 110, 112, 115], "kit": 67, "flash": 67, "board": 67, "apt": 67, "show": [67, 80, 82, 97, 103, 109, 111], "dev": 67, "everth": 67, "nvcc": 67, "cmd": 67, "toolkit": [67, 74], "libcusparselt": 67, "lib64": 67, "wget": [67, 112], "cusparselt": 67, "redist": 67, "libcusparse_lt": 67, "sbsa": 67, "xz": 67, "xf": 67, "v1": [67, 99, 103], "arm64": 67, "mv": 67, "chmod": 67, "pypa": 67, "en": [67, 80], "bootstrap": 67, "jp": 67, "v61": 67, "0a0": 67, "872d972e41": 67, "nv24": 67, "08": 67, "17622132": 67, "cp310": 67, "linux_aarch64": 67, "test_requir": 67, "jetpack6": 67, "lanl": 67, "cuda_vers": 67, "grep": 67, "cut": [67, 82, 100], "sed": [67, 83, 85], "torch_install_path": 67, "dirnam": 67, "__file__": 67, "site_package_path": 67, "cuda_hom": 67, "envsubst": 67, "cxx11": [67, 115], "abi": [67, 115], "anywher": 68, "ahead": [68, 69, 89, 99], "ep": [68, 70, 94, 100, 114, 116], "output_format": [68, 76, 116], "input_tensor": [68, 95, 108, 109], "fill": 68, "aot": [69, 89, 98, 99, 100, 110, 117], "integr": [69, 99, 101], "seamlessli": [69, 76], "ecosystem": [69, 117], "hybrid": [69, 71, 76, 77, 117], "advanc": [69, 78, 83, 91, 98, 110], "bert": [69, 78, 98, 110], "triton": [69, 95], "cudagraph": [69, 98, 110], "overload": [69, 98, 110], "mutabl": [69, 98, 110], "diffus": [69, 78, 98, 110], "gpt2": [69, 98, 110], "llama2": [69, 98, 110], "page": [69, 84, 86, 112], "introductori": 69, "blog": [69, 115], "gtc": 69, "2020": [69, 89], "talk": 69, "fall": [69, 76, 95], "2021": 69, "dai": 69, "confer": 69, "_convolut": [70, 89], "stride": [70, 76, 95, 108], "pad": [70, 76, 95, 108], "dilat": 70, "output_pad": 70, "group": [70, 82, 83], "determinist": 70, "cudnn_en": 70, "allow_tf32": 70, "ab": 70, "aco": 70, "acosh": 70, "adaptive_avg_pool1d": 70, "output_s": 70, "adaptive_avg_pool2d": 70, "adaptive_avg_pool3d": 70, "adaptive_max_pool1d": 70, "adaptive_max_pool2d": 70, "adaptive_max_pool3d": 70, "argmax": [70, 109], "keepdim": 70, "argmin": 70, "asin": 70, "asinh": 70, "atan": 70, "atanh": 70, "avg_pool1d": 70, "kernel_s": [70, 95, 108], "ceil_mod": 70, "count_include_pad": 70, "avg_pool2d": 70, "divisor_overrid": 70, "avg_pool3d": 70, "gamma": 70, "var": 70, "momentum": 70, "bitwise_not": 70, "bmm": 70, "ceil": 70, "clamp": 70, "clamp_max": 70, "clamp_min": 70, "constant_pad_nd": 70, "co": [70, 83, 111], "cosh": 70, "cumsum": 70, "tensor_mod": 70, "rounding_mod": 70, "div_": 70, "elu": 70, "scale": [70, 91, 111], "input_scal": 70, "indic": [70, 71, 80, 82, 93, 100, 102, 113, 114], "padding_idx": 70, "eq": [70, 82], "erf": [70, 93], "exp": 70, "expand_a": 70, "fake_quantize_per_channel_affin": 70, "zero_point": 70, "axi": [70, 76], "quant_min": 70, "quant_max": 70, "fake_quantize_per_tensor_affin": 70, "using_int": [70, 89], "start_dim": [70, 89], "end_dim": [70, 89], "floor": 70, "floor_divid": 70, "ge": 70, "gru_cel": 70, "hx": 70, "w_ih": 70, "w_hh": 70, "b_ih": 70, "b_hh": 70, "gt": 70, "hardtanh_": 70, "instance_norm": 70, "running_mean": 70, "running_var": 70, "use_input_stat": 70, "layer_norm": 70, "normalized_shap": 70, "le": 70, "negative_slop": 70, "01": [70, 83, 89, 108], "leaky_relu_": 70, "lstm_cell": 70, "lt": 70, "masked_fil": 70, "mask": [70, 95], "max_pool1d": 70, "max_pool2d": [70, 88, 89], "max_pool3d": 70, "mul_": [70, 93], "narrow": 70, "neg": [70, 99], "norm": 70, "scalaropt_dim": 70, "pixel_shuffl": 70, "upscale_factor": 70, "pow": 70, "tensor_scalar": 70, "expon": 70, "tensor_tensor": 70, "prelu": 70, "prod": [70, 95], "dim_int": 70, "reciproc": 70, "reflection_pad1d": 70, "reflection_pad2d": 70, "relu_": 70, "repeat_interleav": 70, "self_int": 70, "replication_pad1d": 70, "replication_pad2d": 70, "replication_pad3d": 70, "reshap": [70, 95, 112], "roll": 70, "rsub": 70, "scatter": 70, "sigmoid_": 70, "sin": [70, 82], "sinh": 70, "slice": 70, "split_siz": 70, "split_with_s": 70, "sqrt": 70, "squar": 70, "squeez": [70, 111], "sub_": 70, "dim_intlist": 70, "tan": 70, "tanh": [70, 93], "tanh_": [70, 93], "non_block": [70, 108], "memory_format": [70, 76], "prim_devic": 70, "topk": 70, "k": [70, 91, 108], "largest": 70, "dim0": [70, 97], "dim1": 70, "unbind": 70, "unsqueez": 70, "upsample_bilinear2d": 70, "align_corn": 70, "scales_h": 70, "scales_w": 70, "vec": 70, "scale_factor": 70, "upsample_linear1d": 70, "upsample_nearest1d": 70, "upsample_nearest2d": 70, "upsample_nearest3d": 70, "scales_d": 70, "upsample_trilinear3d": 70, "view": [70, 80], "__and__": 70, "__derive_index": 70, "idx": 70, "__getitem__": 70, "__is__": 70, "t1": 70, "t2": 70, "obj": 70, "__isnot__": 70, "__not__": 70, "__or__": 70, "__range_length": 70, "lo": 70, "hi": [70, 82, 83], "__round_to_zero_floordiv": 70, "__xor__": 70, "append": [70, 93, 96, 97, 108, 109], "el": 70, "arang": [70, 95], "pin_memori": 70, "start_step": 70, "copy_": 70, "float_int": 70, "int_float": 70, "floordiv": 70, "is_floating_point": 70, "numel": 70, "l": [70, 108], "9223372036854775807": 70, "requires_grad": 70, "tupleindex": 70, "tup": 70, "exported_program": [71, 76, 116], "arg_input": [71, 76, 93, 100], "kwarg_input": [71, 76, 100], "engine_cache_dir": [71, 96, 97], "engine_cache_s": [71, 96, 97], "custom_engine_cach": [71, 97], "baseenginecach": [71, 97], "int32": [71, 76, 77, 95, 96, 104, 111], "channel_last": [71, 76, 77, 111], "244": [71, 76, 77], "alia": [71, 76], "better": [71, 76, 88, 111, 117], "understand": [71, 76, 113], "convolut": [71, 76, 77, 91, 95, 118], "_c": [71, 76, 77, 92], "oppos": [71, 76, 77], "lean": [71, 76], "spend": [71, 76], "integ": [71, 76, 85], "faster": [71, 76, 96, 97, 111], "parition": [71, 76], "increas": [71, 76, 97, 109], "amount": [71, 76, 109], "defer": [71, 76, 117], "lead": [71, 76, 82, 109, 115], "oversubscript": [71, 76], "hard": [71, 100], "disk": [71, 76, 97], "space": [71, 82, 83, 91], "byte": [71, 75, 76, 77, 95, 97, 109, 111], "1gb": [71, 96, 97], "exce": 71, "oldest": 71, "gear": [71, 91], "toward": [71, 91], "cross_compile_flag": 71, "cross_compil": 71, "refit_module_weight": [71, 100], "compiled_modul": [71, 100], "new_weight_modul": [71, 100], "verify_output": [71, 100], "use_weight_map_cach": [71, 100], "in_plac": [71, 100], "compmil": 71, "coverag": [71, 95], "min_acc_module_s": 72, "is_aten": 72, "use_experimental_fx_rt": 72, "correctness_atol": 72, "correctness_rtol": 72, "minim": [72, 91, 95], "submodul": [72, 88, 95], "fx2trt": 72, "cpu": [72, 106, 107, 109], "has_batch_dim": 72, "dtyep": 72, "prop": 72, "min_input_shap": 72, "optimized_input_shap": 72, "max_input_shap": 72, "popul": 72, "225": [72, 112], "explicit_precis": 72, "logger_level": 72, "model_trt": 73, "model_torchtrt": 73, "internal_error": 73, "dataloadercalibr": [74, 91], "preprocess": [74, 91, 112], "algo_typ": [74, 91], "calibrationalgo": [74, 91], "cachecalibr": [74, 91], "qualnam": [74, 76], "entropy_calibr": 74, "entropy_calibration_2": [74, 91], "legacy_calibr": 74, "minmax_calibr": 74, "set_multi_device_safe_mod": [75, 115], "_multidevicesafemodecontextmanag": 75, "impact": 75, "suppress": 75, "unsaf": 75, "trt_compiled_modul": 75, "torchtensorrtmodul": [75, 95], "encompass": [75, 77], "simpili": 75, "de": 75, "initi": [75, 76, 82, 100, 101, 102, 104, 105, 106, 107], "scriptmodul": [75, 76, 77, 89, 90, 116, 117], "overridden": [75, 76], "subclass": 75, "although": [75, 82], "recip": [75, 91], "afterward": 75, "former": 75, "care": 75, "hook": 75, "silent": 75, "get_extra_st": 75, "state_dict": [75, 76, 99], "set_extra_st": 75, "picklabl": 75, "pickl": [75, 95, 97], "load_state_dict": [75, 99, 108], "pythontorchtensorrtmodul": 75, "serialized_engin": [75, 77], "_set": [75, 101], "weight_name_map": 75, "trt_modul": 75, "engine_str": 75, "my_modul": 75, "current_devic": 75, "cudagraphs_validate_shap": 75, "versu": 75, "disable_profil": 75, "enable_profil": 75, "iprofil": 75, "spent": 75, "get_layer_info": 75, "request": [76, 89, 112], "decid": 76, "deseri": [76, 77, 89, 95], "retrac": 76, "strict": [76, 115], "valueerror": [76, 94], "mutabletorchtensorrtmodul": [76, 99], "pytorch_model": 76, "regular": 76, "whenev": 76, "refit_gm": 76, "shape_mod": 76, "_shapemod": 76, "interv": 76, "notat": 76, "bound": 76, "torch_tensor": 76, "tracer": 76, "example_tensor": 76, "optimization_profile_field": 76, "classmethod": 76, "disable_memory_format_check": 76, "core_id": 76, "schedul": [76, 112], "use_default": 76, "try_to": 76, "anoth": [76, 82, 83, 88, 90, 100], "typeerror": 76, "unknown": 76, "succe": 76, "float_dtyp": 76, "failur": 76, "bf16": 76, "try_from": [76, 95], "complex128": 76, "16": [76, 86, 88, 89, 90, 102, 105], "brain": 76, "bfloat16": 76, "f64": 76, "f8": 76, "fp8": 76, "float8": 76, "i32": 76, "sign": [76, 112], "i64": 76, "u8": 76, "unsign": 76, "uint8": 76, "trt_dla": 76, "torchtrt_dla": 76, "_from": 76, "torchtrt_dla_ec": 76, "torchtrt_safety_ec": 76, "saefti": 76, "trt_dla_ec": 76, "standalon": [76, 82], "certifi": 76, "tf": 76, "torchtrt_linear": 76, "cdhw32": 76, "thirti": 76, "row": [76, 83], "spatial": 76, "31": [76, 89], "subscript": [76, 82], "chw16": 76, "sixteen": 76, "15": [76, 82, 86], "chw2": 76, "chw32": 76, "chw4": 76, "four": [76, 82, 83], "dhwc": 76, "equivi": 76, "channels_last_3d": 76, "dhwc8": 76, "eight": 76, "dla_hwc4": 76, "imag": [76, 91, 95, 99, 103, 108, 112], "roundup": 76, "elements": 76, "dla_linear": 76, "planar": 76, "hwc": 76, "channels_last": 76, "hwc16": 76, "hwc8": 76, "least": [76, 82, 83], "ishapelay": 77, "check_method_op_support": 77, "seriali": 77, "put_binding_nam": 77, "tensorrtcompilespec": [77, 92], "scriptclass": 77, "0x7fdbd3710cf0": 77, "_jit_to_tensorrt": 77, "00": 78, "000": [78, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "total": [78, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "galleri": [78, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110], "mem": 78, "torch_compile_advanced_usag": [78, 101], "torch_compile_resnet_exampl": [78, 102], "torch_compile_stable_diffus": [78, 103], "torch_compile_transformers_exampl": [78, 104], "v0": [79, 112], "pytorch_sphinx_them": [80, 87], "conf": [80, 87], "html_theme_opt": 80, "canonical_url": 80, "analytics_id": 80, "logo_onli": 80, "display_vers": 80, "prev_next_buttons_loc": 80, "bottom": 80, "style_external_link": 80, "vcs_pageview_mod": 80, "collapse_navig": 80, "sticky_navig": [80, 84], "navigation_depth": 80, "includehidden": 80, "titles_onli": 80, "canon": 80, "rank": 80, "trail": 80, "slash": 80, "googl": 80, "analyt": 80, "isn": [80, 82, 95], "shown": [80, 82, 89, 114], "sidebar": [80, 86], "button": [80, 82], "icon": [80, 82], "extern": [80, 82, 98, 110], "display_github": 80, "display_gitlab": 80, "gitlab": 80, "bitbucket": 80, "bar": [80, 82], "www": [80, 82, 89, 91, 112], "sphinx": [80, 81, 82, 83, 87, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110], "toctre": 80, "lose": 80, "scroll": [80, 84], "unlimit": 80, "header": [80, 82, 83, 89, 112], "render": 80, "github_url": 80, "bitbucket_url": 80, "gitlab_url": 80, "left": [80, 82], "upon": [80, 101, 104], "rst": [80, 82], "visitor": 80, "revert": 80, "misbuild": 80, "properti": [80, 95], "stick": 80, "screen": 80, "vertic": [80, 82], "too": [80, 82, 83], "sticki": [80, 86], "nav": [80, 86], "At": [81, 93, 100], "django": 81, "payment": 81, "dotpai": 81, "dotpayprovid": 81, "seller_id": 81, "pin": 81, "lock": 81, "lang": 81, "pl": 81, "polish": 81, "gatewai": 81, "transfer": 81, "purchas": 81, "item": [81, 83, 108], "param": 81, "seller": 81, "consult": 81, "ui": 81, "languag": [81, 82, 83, 88, 95, 98, 106, 110, 112], "data_item_1": 81, "emphasi": 82, "hyperlink": 82, "uri": 82, "web": 82, "anonym": 82, "label": [82, 91, 108, 111, 112], "substitut": 82, "charact": 82, "exceedingli": 82, "ugli": 82, "problem": [82, 107], "problemat": 82, "ext": [82, 83], "autodoc": [82, 83], "demo": [82, 91], "test_py_modul": [82, 86], "my": [82, 106], "role": 82, "pep": 82, "287": 82, "rfc": 82, "2822": 82, "superscript": 82, "gui": 82, "taken": 82, "height": 82, "interfer": 82, "press": 82, "keyboard": 82, "mous": 82, "mmb": 82, "menuselect": 82, "seen": [82, 83], "whitespac": 82, "signific": [82, 95], "strang": 82, "hyphen": 82, "word": [82, 111], "adjust": 82, "width": [82, 111], "browser": 82, "sentenc": [82, 109, 111], "suppli": [82, 100], "258": 82, "equat": 82, "x_": 82, "x_0": 82, "x_1": 82, "x_2": 82, "x_3": 82, "x_4": 82, "nabla": 82, "frac": 82, "theta": 82, "phi": 82, "restructuredtext": [82, 83], "parser": [82, 94, 108], "colon": 82, "indent": 82, "literal_block": 82, "spaces_and_linebreak": 82, "preserv": [82, 88, 91], "markup_process": 82, "Or": 82, "great": [82, 89, 95, 97, 117], "why": [82, 115], "didn": 82, "blank": 82, "align": 82, "permit": 82, "awai": 82, "eric": 82, "orchestra": 82, "leader": 82, "bee": 82, "philosoph": 82, "ipso": 82, "facto": 82, "But": [82, 89, 100, 109], "got": [82, 89], "vi": 82, "entiti": 82, "said": 82, "entir": [82, 117], "ancient": 82, "injuri": 82, "sing": 82, "elk": 82, "bracket": 82, "miss": [82, 89], "brontosaurus": 82, "thin": 82, "thicker": 82, "middl": 82, "That": [82, 89], "mine": 82, "belong": 82, "me": [82, 83], "ann": 82, "begun": 82, "past": 82, "pars": [82, 89], "someurl": 82, "dev0": 82, "bc95015": 82, "caption": [82, 85], "pane": 82, "shell_command": 82, "echo": 82, "did": 82, "window_nam": 82, "session_nam": 82, "shorthand": 82, "some_funct": 82, "highlight": 82, "THE": 82, "heaven": 82, "hexagram": 82, "six": 82, "unbroken": 82, "primal": 82, "light": [82, 116], "spirit": 82, "weak": 82, "essenc": 82, "energi": 82, "unrestrict": 82, "conceiv": 82, "motion": 82, "regard": [82, 117], "basi": 82, "thu": 82, "persist": 82, "dual": 82, "sens": [82, 89], "univers": 82, "world": 82, "men": 82, "express": 82, "deiti": 82, "human": 82, "denot": [82, 95], "holi": 82, "man": [82, 83], "sage": 82, "ruler": 82, "who": 82, "awaken": 82, "utf": [82, 83], "sphinx_rtd_them": [82, 83], "docstr": [82, 83, 90], "dl": 82, "dt": 82, "tag": [82, 112], "tt": 82, "descnam": 82, "descclassnam": 82, "wrote": 82, "anyth": [82, 83, 115], "programm": 82, "myclass": 82, "dothismethod": 82, "flush": 82, "meth": 82, "capit": 82, "flox": 82, "unreferenc": 82, "nonexist": 82, "extrem": 82, "stuff": 82, "mayb": 82, "bold": 82, "ital": 82, "heck": 82, "put": [82, 111], "13": [82, 86], "backlink": 82, "knowledg": 82, "mind": 82, "ey": 82, "thought": 82, "medium": 82, "peopl": 82, "subsect": 82, "interpol": 82, "indirect": 82, "phrase": 82, "docutil": [82, 83], "sourceforg": [82, 83], "ref": 82, "clickabl": 82, "legend": 82, "revis": [82, 83, 99, 103], "revisit": 82, "enhanc": 82, "structuredtext": 82, "wooden": 82, "nickel": 82, "mad": 82, "scientist": 82, "bigger": 82, "bread": 82, "box": [82, 113, 117], "wash": 82, "behind": 82, "ear": 82, "room": 82, "closet": 82, "bathroom": 82, "trash": 82, "sink": 82, "mother": 82, "g_": 82, "mu": 82, "nu": 82, "pi": 82, "t_": 82, "rho_": 82, "servic": 82, "thing1": 82, "thing2": 82, "thing3": 82, "prose": 82, "provok": 82, "mental": 82, "exert": 82, "reader": 82, "discret": 82, "strongli": [82, 109], "advis": 82, "subtitl": 82, "outsid": 82, "often": 82, "besid": 82, "border": 82, "background": [82, 88], "ok": [82, 89], "transmit": 82, "disconnect": 82, "nonetheless": 82, "semant": 82, "blue": [82, 95], "white": 82, "arab": 83, "roman": 83, "upper": 83, "iii": 83, "iv": 83, "classifi": [83, 88, 89, 108, 111], "paragraph": [83, 86], "z": 83, "commonli": 83, "vm": 83, "david": 83, "goodger": 83, "address": [83, 95, 99], "123": 83, "street": 83, "canada": 83, "a1b": 83, "2c3": 83, "contact": 83, "myself": 83, "organ": 83, "humankind": 83, "2012": 83, "03": 83, "19": [83, 86], "53": 83, "0000": 83, "tue": 83, "jan": 83, "progress": 83, "7302": 83, "wish": 83, "redistribut": 83, "reattribut": 83, "sell": 83, "bui": 83, "rent": 83, "leas": 83, "improv": [83, 115], "quot": 83, "excerpt": 83, "collat": 83, "fold": 83, "stapl": 83, "mutil": 83, "anyon": 83, "heart": 83, "bibliograph": 83, "markup": [83, 86], "literal": 83, "yahoo": 83, "oh": 83, "liter": 83, "heh": 83, "child": 83, "beat": 83, "text": [83, 85, 106, 107, 111], "hehe": 83, "kept": 83, "sai": [83, 111], "cackl": 83, "night": 83, "lone": 83, "guangzhou": 83, "destini": 83, "hope": 83, "dream": 83, "forth": 83, "fifth": 83, "sixth": 83, "lorem": [83, 85], "ipsum": [83, 85], "dolor": [83, 85], "sit": [83, 85], "amet": [83, 85], "consectetur": [83, 85], "adipisc": [83, 85], "elit": [83, 85], "donec": [83, 85], "porttitor": [83, 85], "odio": [83, 85], "posuer": [83, 85], "vita": [83, 85], "ornar": [83, 85], "libero": [83, 85], "matti": 83, "loborti": [83, 85], "justo": [83, 85], "vestibulum": [83, 85], "nibh": [83, 85], "aliquet": [83, 85], "feugiat": [83, 85], "sagitti": [83, 85], "nequ": [83, 85], "qui": [83, 85], "eleifend": 83, "dui": [83, 85], "rutrum": [83, 85], "lectu": [83, 85], "suscipit": [83, 85], "letter": [83, 111], "column": 83, "cell": 83, "span": 83, "nam": [83, 85], "mauri": [83, 85], "arcu": [83, 85], "stub": 83, "behav": 84, "area": 84, "interdum": 85, "nec": 85, "finibu": 85, "dictum": 85, "velit": 85, "ut": 85, "eu": 85, "efficitur": 85, "aliquam": 85, "erat": 85, "diam": 85, "gravida": 85, "imperdiet": 85, "tellu": 85, "nisl": 85, "praesent": 85, "eget": 85, "elementum": 85, "rhoncu": 85, "tincidunt": 85, "suspendiss": 85, "volutpat": 85, "scelerisqu": 85, "tristiqu": 85, "aenean": 85, "condimentum": 85, "risu": 85, "accumsan": 85, "laoreet": 85, "maximu": 85, "sapien": 85, "ligula": 85, "fringilla": 85, "commodo": 85, "proin": 85, "et": 85, "pharetra": 85, "etiam": 85, "turpi": 85, "ant": 85, "luctu": 85, "vel": 85, "malesuada": 85, "dignissim": 85, "mi": 85, "nunc": 85, "augu": 85, "sem": 85, "cursu": 85, "nulla": 85, "pellentesqu": 85, "habit": 85, "morbi": 85, "senectu": 85, "netu": 85, "fame": 85, "ac": 85, "egesta": 85, "placerat": 85, "tortor": 85, "iaculi": 85, "venenati": 85, "cra": 85, "puru": 85, "ero": 85, "vehicula": 85, "fusc": 85, "auctor": 85, "phasellu": 85, "est": 85, "viverra": 85, "conval": 85, "faucibu": 85, "vulput": 85, "feli": 85, "sodal": 85, "maecena": 85, "congu": 85, "semper": 85, "enim": 85, "blandit": 85, "sollicitudin": 85, "urna": 85, "orci": 85, "lacu": 85, "quisqu": 85, "facilisi": 85, "hendrerit": 85, "curabitur": 85, "variu": 85, "bibendum": 85, "massa": 85, "magna": 85, "tempu": 85, "metu": 85, "nisi": 85, "pretium": 85, "leo": 85, "euismod": 85, "ultric": 85, "dapibu": 85, "lacinia": 85, "vivamu": 85, "molesti": 85, "hac": 85, "habitass": 85, "platea": 85, "dictumst": 85, "git": 86, "content": [86, 91, 112], "changelog": 86, "math": 86, "14": [86, 96, 104, 112], "17": 86, "18": [86, 89, 99], "submenu": 86, "symlink": 87, "subtre": 87, "_theme": 87, "html_theme": 87, "html_theme_path": 87, "optimiz": 88, "tutori": [88, 91, 93, 95, 97, 99, 100], "beginn": 88, "intro_to_torchscript_tutori": 88, "briefli": 88, "lenet": [88, 89], "lenetfeatextractor": 88, "conv1": [88, 89], "conv2d": [88, 95, 108], "conv2": [88, 89], "lenetclassifi": 88, "fc1": [88, 89], "120": [88, 89], "fc2": [88, 89], "84": [88, 89], "fc3": [88, 89], "feat": [88, 89], "obvious": 88, "pathwai": 88, "input_data": [88, 90], "traced_model": 88, "pick": [88, 114], "script_model": [88, 92], "perspect": 88, "___torch_mangle_10": 88, "129": 88, "___torch_mangle_9": 88, "119": 88, "___torch_mangle_5": 88, "137": 88, "callmethod": 88, "138": 88, "38": 88, "39": 88, "torch_script_modul": [88, 89], "in_tensor": 88, "fly": 88, "lenet_script": [88, 89], "haven": 89, "acquir": 89, "dyanmo": 89, "almost": [89, 117], "trt_lenet_script": 89, "apr": 89, "56": 89, "04": [89, 112], "credit": 89, "stop": 89, "argc": 89, "argv": 89, "cerr": 89, "cout": 89, "even": [89, 99], "cppdoc": 89, "pretti": 89, "fashion": [89, 111], "enable_precis": 89, "And": 89, "convertgraphtotrtengin": 89, "engine_converted_from_jit": 89, "close": [89, 93], "saw": 89, "576": 89, "346": 89, "539": 89, "0464": 89, "0383": 89, "0678": 89, "0932": 89, "1045": 89, "0805": 89, "0435": 89, "0818": 89, "0208": 89, "0358": 89, "cudafloattyp": 89, "0530": 89, "1691": 89, "2802": 89, "1502": 89, "1056": 89, "1549": 89, "input0": [89, 90], "1063": 89, "input1": [89, 90], "input2": 89, "28": 89, "29": 89, "33": 89, "35": 89, "36": 89, "37": 89, "compilegraph": [89, 91], "transform": [89, 91, 96, 98, 100, 104, 106, 107, 108, 109, 110, 112, 116], "laid": 89, "translat": [89, 100], "aren": 89, "techniqu": [89, 91, 107, 115], "checkmethodoperatorsupport": 89, "modular": 89, "ship": [89, 115], "exhaust": 89, "109": 89, "addlay": 89, "yourself": 89, "question": [89, 93], "outself": 89, "flatten_convert": 89, "unwraptoint": 89, "in_shap": 89, "tovec": 89, "out_shap": 89, "shuffl": [89, 91, 108], "addshuffl": 89, "setreshapedimens": 89, "todim": 89, "extens": [89, 117], "ctype": 89, "cdll": 89, "contributor": 89, "upstream": 89, "pr": 89, "usecas": 90, "sole": [90, 91, 117], "individu": 90, "accuraci": [91, 111], "loss": [91, 111], "infrastructur": [91, 112], "streamlin": 91, "expos": [91, 95], "cpp_frontend": 91, "loading_data_recip": 91, "cifar10": [91, 108], "cstddef": 91, "ktrain": 91, "ktest": 91, "un": 91, "cs": 91, "toronto": 91, "edu": 91, "kriz": 91, "cifar": 91, "is_train": 91, "trim": 91, "use_subset": 91, "new_siz": 91, "mode_": 91, "images_": 91, "targets_": 91, "calibration_dataset": 91, "data_dir": 91, "320": 91, "4914": [91, 108], "4822": [91, 108], "4465": [91, 108], "2023": [91, 108], "1994": [91, 108], "2010": [91, 108], "dataloaderopt": 91, "worker": 91, "virtual": 91, "input_shap": [91, 118], "compile_spec": [91, 94, 102, 118], "kf16": [91, 118], "ki8": 91, "vgg16": [91, 108], "testing_dataset": [91, 108], "totensor": [91, 108, 112], "testing_dataload": [91, 108], "num_work": [91, 108], "vgg": [91, 108], "test_ptq_dataloader_calibr": 91, "test_ptq_trt_calibr": 91, "krizhevski": 91, "hinton": 91, "2009": 91, "tini": 91, "simonyan": 91, "zisserman": 91, "2014": 91, "recognit": [91, 111], "arxiv": 91, "preprint": 91, "1409": 91, "1556": 91, "_jit_to_backend": 92, "mobilenet_v2": 92, "pretrain": [92, 97, 99, 102, 105, 111, 112], "gelu": 93, "sy": 93, "approxim": 93, "suppos": 93, "my_mod": 93, "ex_input": [93, 95], "baselin": 93, "my_standard_gelu": 93, "supports_dynamic_shap": 93, "supersed": 93, "converterprior": 93, "vers": 93, "prior": [93, 97, 113, 115], "distinct": 93, "prepend": 93, "candid": 93, "primit": 93, "compiler_ir": 93, "boilerpl": 93, "focu": [93, 99], "interoper": 93, "aten_ops_gelu": 93, "sourceir": 93, "cheap": 93, "unqiu": 93, "op_count": 93, "get_op_count": 93, "nonloc": 93, "elementwis": 93, "source_ir": 93, "lhs_val": 93, "rhs_val": 93, "x_7": 93, "x_8": 93, "79788456080000003": 93, "x_9": 93, "044714999999999998": 93, "x_10": 93, "x_11": 93, "x_12": 93, "x_13": 93, "x_14": 93, "x_15": 93, "my_custom_gelu": 93, "allclos": [93, 99, 100], "my_mod_erf": 93, "my_gelu_erf": 93, "notic": 93, "minut": [93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "converter_overload": 93, "jupyt": [93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110], "ipynb": [93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "geforcertx": 94, "4080": 94, "3080": 94, "cross_runtime_compilation_for_window": 94, "trt_resnet": 94, "argpars": [94, 108], "argumentpars": [94, 108], "comil": 94, "add_argu": [94, 108], "parse_arg": [94, 108], "manual_se": [94, 96, 97, 99, 100], "resnet18": [94, 97, 99, 100, 102, 105], "amd64": 94, "loaded_model": 94, "load_cross_compiled_exported_program": 94, "trt_output": 94, "cross_compile_for_window": 94, "cost": [95, 97, 100, 115], "perhap": 95, "overhead": [95, 109, 115], "sake": 95, "circular": 95, "red": 95, "green": 95, "twice": 95, "written": 95, "openai": 95, "formal": 95, "tl": 95, "custom_op": 95, "circ_pad_kernel": 95, "all_pads_0": 95, "all_pads_2": 95, "all_pads_4": 95, "all_pads_6": 95, "orig_dims_0": 95, "orig_dims_1": 95, "orig_dims_2": 95, "orig_dims_3": 95, "y_shape_1": 95, "y_shape_2": 95, "y_shape_3": 95, "x_len": 95, "y_len": 95, "block_siz": 95, "pid": 95, "program_id": 95, "mask_i": 95, "i3": 95, "i2": 95, "i1": 95, "i0": 95, "j0": 95, "j1": 95, "j2": 95, "j3": 95, "load_idx": 95, "mask_x": 95, "launch": [95, 112], "torchtrt_ex": 95, "triton_circular_pad": 95, "mutates_arg": 95, "out_dim": 95, "tolist": 95, "all_pad": 95, "zero": 95, "orig_dim": 95, "blocksiz": 95, "256": [95, 108, 109, 112], "numblock": 95, "tracabl": 95, "prerequisit": 95, "fake": 95, "real": 95, "faketensor": 95, "register_fak": 95, "autograd": 95, "beyond": 95, "register_autograd": 95, "padded_x": 95, "my_model": 95, "2604": 95, "4232": 95, "3041": 95, "0833": 95, "2461": 95, "1270": 95, "2450": 95, "4079": 95, "2887": 95, "2828": 95, "0373": 95, "0332": 95, "3143": 95, "6344": 95, "5638": 95, "1867": 95, "5068": 95, "4363": 95, "7937": 95, "3488": 95, "1350": 95, "7966": 95, "3517": 95, "1379": 95, "5537": 95, "1088": 95, "8950": 95, "0550": 95, "6163": 95, "0109": 95, "5245": 95, "9632": 95, "5686": 95, "3775": 95, "8162": 95, "4216": 95, "4311": 95, "1649": 95, "2091": 95, "3668": 95, "1006": 95, "1447": 95, "0352": 95, "7689": 95, "8131": 95, "_run_on_gpu_0": 95, "_run_on_acc_1": 95, "dry": 95, "50": [95, 111], "count": 95, "__": 95, "aggreg": 95, "stat": 95, "latenc": [95, 109, 115], "abstractli": 95, "pkl": [95, 99], "cupi": 95, "gap": 95, "prealloc": 95, "circularpaddingplugin": 95, "ipluginv2dynamicext": 95, "field_collect": 95, "pluginfieldcollect": 95, "x_shape": 95, "num_output": 95, "plugin_namespac": 95, "plugin_typ": 95, "plugin_vers": 95, "assert": [95, 99, 100], "get_output_datatyp": 95, "input_typ": 95, "get_output_dimens": 95, "output_index": 95, "dimsexpr": 95, "exprbuild": 95, "iexprbuild": 95, "output_dim": 95, "dimensionoper": 95, "configure_plugin": 95, "inp": 95, "dynamicplugintensordesc": 95, "x_dim": 95, "desc": 95, "supports_format_combin": 95, "po": 95, "in_out": 95, "plugintensordesc": 95, "num_input": 95, "enqueu": 95, "input_desc": 95, "output_desc": 95, "in_dtyp": 95, "a_mem": 95, "unownedmemori": 95, "items": 95, "c_mem": 95, "a_ptr": 95, "memorypoint": 95, "c_ptr": 95, "a_d": 95, "memptr": 95, "c_d": 95, "a_t": 95, "as_tensor": 95, "c_t": 95, "cloned_plugin": 95, "__dict__": 95, "circularpaddingplugincr": 95, "iplugincr": 95, "field_nam": 95, "pluginfield": 95, "pluginfieldtyp": 95, "create_plugin": 95, "pluginfieldcollection_": 95, "deserialize_plugin": 95, "pads_dict": 95, "creator": 95, "trt_plugin_registri": 95, "get_plugin_registri": 95, "register_cr": 95, "untyp": 95, "get_trt_tensor": 95, "set_layer_nam": 95, "recal": 95, "intlist": 95, "circular_padding_convert": 95, "retriev": 95, "elsewher": 95, "plugin_registri": 95, "plugin_cr": 95, "get_plugin_cr": 95, "field_config": 95, "eventu": 95, "freez": 95, "_input": 95, "add_plugin_v2": 95, "circular_padding_plugin": 95, "_run_on_acc_0": 95, "grad_fn": 95, "subbackward0": 95, "custom_kernel_plugin": 95, "engine_caching_exampl": [96, 97], "remove_timing_cach": [96, 97], "bertmodel": [96, 104], "random": [96, 97, 99, 100, 109], "seed": [96, 97, 99, 100], "from_pretrain": [96, 99, 103, 104, 106, 107, 109], "uncas": [96, 104, 111], "return_dict": 96, "randint": [96, 104, 109], "compile_bert": 96, "enable_tim": [96, 97], "1st": [96, 97], "measur": [96, 97, 109], "2nd": [96, 97], "3rd": [96, 97], "slower": [96, 97], "messur": [96, 97], "compilation_kwarg": [96, 104], "torch_trt_bert_engine_cach": 96, "30": [96, 97, 99, 100, 102, 104, 114], "synchron": [96, 97, 109], "elapsed_tim": [96, 97], "millisecond": 96, "__name__": [96, 101, 104], "__main__": [96, 101, 104], "engine_caching_bert_exampl": 96, "paid": 97, "upfront": 97, "invalid": 97, "repeatedli": 97, "mitig": 97, "explor": 97, "torch_trt": [97, 99, 100], "_default": 97, "_engine_cach": 97, "flexibl": [97, 117], "histor": 97, "barrier": 97, "reconstruct": 97, "ti": 97, "hash": 97, "magnitud": 97, "torch_compil": [97, 101, 102, 104, 105, 113, 117], "compiled_model": 97, "ms": [97, 109], "dynamo_compil": 97, "example_input": 97, "200": 97, "dynamic_shap": [97, 113], "remot": 97, "systen": 97, "agnost": 97, "implent": 97, "ramenginecach": 97, "held": 97, "engine_cach": 97, "torch_compile_my_cach": 97, "_torch_export_gpt2": [98, 110], "_torch_export_llama2": [98, 110], "sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_window": [98, 110], "straightforward": 99, "especi": 99, "hug": [99, 106, 107], "face": [99, 106, 107], "difficult": 99, "ever": 99, "walk": [99, 100, 106], "lora": [99, 100], "use_python": 99, "mutable_modul": 99, "model2": [99, 100], "expected_output": [99, 100], "refitted_output": [99, 100], "reload": [99, 117], "checkpoint": [99, 108], "civitai": 99, "12597": 99, "moxin": 99, "diffusionpipelin": [99, 103], "no_grad": [99, 106, 107, 108, 109], "model_id": [99, 103], "runwayml": 99, "hous": 99, "forest": 99, "shuimobysim": 99, "wuchangshuo": 99, "qualiti": 99, "worst": 99, "lowr": 99, "cloudi": 99, "watermark": 99, "pipe": [99, 103], "torch_dtyp": [99, 103], "unet": [99, 103], "negative_prompt": 99, "num_inference_step": 99, "without_lora_mut": 99, "jpg": [99, 112], "procedur": 99, "load_lora_weight": 99, "stablediffusionapi": 99, "load_lora_embed": 99, "weight_nam": 99, "safetensor": 99, "adapter_nam": 99, "lora1": 99, "set_adapt": 99, "adapter_weight": 99, "fuse_lora": 99, "unload_lora_weight": 99, "with_lora_mut": 99, "mutable_torchtrt_module_exampl": 99, "expens": 100, "involv": 100, "occasion": [100, 101, 104], "adapt": 100, "infeas": 100, "focus": 100, "mostli": 100, "recogn": 100, "behalf": 100, "init": [100, 108], "sett": 100, "randomli": 100, "exp_program2": 100, "compiled_trt_ep": 100, "new_trt_gm": 100, "accomplish": 100, "gaurente": 100, "attempt": [100, 108, 113], "rebuild": 100, "heurist": 100, "refit_engine_exampl": 100, "x_out": 101, "y_out": 101, "x_y_out": 101, "invoc": 101, "sample_inputs_half": 101, "model_half": 101, "backend_kwarg": 101, "optimized_model_custom": 101, "exit": [101, 104, 112], "2052": [101, 104], "compile_engine_and_inf": [101, 104], "new_input": [102, 104], "new_output": [102, 104], "new_batch_size_input": 102, "new_batch_size_output": 102, "inputs_bs8": 102, "mark_dynam": [102, 113], "outputs_bs8": 102, "No": [102, 113], "inputs_bs12": 102, "outputs_bs12": 102, "compvi": 103, "majest": 103, "castl": 103, "cloud": 103, "majestic_castl": 103, "png": 103, "enable_cudagraph": [105, 115], "out_trt": 105, "set_cudagraphs_mod": [105, 115], "inputs_2": 105, "inputs_3": 105, "out_trt_2": 105, "out_trt_3": 105, "torch_export_cudagraph": 105, "automodelforcausallm": [106, 107, 109], "autotoken": [106, 107], "export_llm": [106, 107, 109], "max_token": [106, 107, 109], "kv_cach": [106, 107], "token": [106, 107, 111], "pad_token_id": 106, "eos_token_id": [106, 107], "attn_implement": [106, 107, 109], "eager": [106, 107, 109], "enjoi": 106, "cute": 106, "dog": 106, "model_input": [106, 107], "return_tensor": [106, 107], "input_id": [106, 107], "regress": [106, 107], "huggingfac": [106, 107, 111], "pyt_gen_token": [106, 107], "gpt2_ep": 106, "max_seq_len": [106, 107, 109], "trt_gen_token": [106, 107], "skip_special_token": [106, 107], "parallel": 106, "paradigm": 106, "torch_export_gpt2": 106, "llama_path": [107, 109], "llama": [107, 109], "7b": [107, 109], "chat": [107, 109], "hf": [107, 109], "llama2_ep": [107, 109], "batch_decod": 107, "clean_up_tokenization_spac": 107, "solv": [107, 112], "smaller": [107, 111], "subproblem": 107, "torch_export_llama2": 107, "modelopt": 108, "mtq": 108, "export_torch_mod": 108, "layer_spec": 108, "num_class": 108, "1000": [108, 109, 112], "init_weight": 108, "in_channel": 108, "pool": [108, 118], "maxpool2d": 108, "batchnorm2d": 108, "sequenti": 108, "avgpool": 108, "adaptiveavgpool2d": 108, "4096": 108, "dropout": 108, "_initialize_weight": 108, "kaiming_normal_": 108, "fan_out": 108, "nonlinear": 108, "constant_": 108, "elif": 108, "normal_": 108, "vgg16_cfg": 108, "128": [108, 109], "ckpt": 108, "model_state_dict": 108, "device_count": 108, "ordereddict": 108, "new_state_dict": 108, "forget": 108, "training_dataset": 108, "randomcrop": 108, "randomhorizontalflip": 108, "training_dataload": 108, "drop_last": 108, "crit": 108, "crossentropyloss": 108, "calibrate_loop": 108, "pred": 108, "5f": 108, "acc": 108, "2f": 108, "quantize_typ": 108, "quant_cfg": 108, "int8_default_cfg": 108, "fp8_default_cfg": 108, "forward_loop": 108, "qdq": 108, "incomplet": 108, "functionaltensor": 108, "functionaltensormod": 108, "_trace": 108, "_export": 108, "float8_e4m3fn": 108, "class_prob": 108, "class_pr": 108, "test_prob": 108, "test_pr": 108, "test_loss": 108, "test_acc": 108, "vgg16_ptq": 108, "overcom": 109, "throughput": 109, "sometim": [109, 113], "outweigh": 109, "slowdown": 109, "hardwar": [109, 118], "experi": 109, "balanc": 109, "timeit": 109, "time_gener": 109, "output_seq_length": 109, "seq_len": [109, 113], "llm": 109, "input_seq": 109, "start_tim": 109, "default_tim": 109, "inputs_copi": 109, "decod": 109, "logit": 109, "next_token_logit": 109, "next_token": 109, "end_tim": 109, "time_mean_m": 109, "isl": 109, "osl": 109, "warm": 109, "solut": 109, "insight": 109, "weight_streaming_ctx": 109, "weight_stream": 109, "mean_lat": 109, "percentag": 109, "weight_budget_pct": 109, "device_budget": 109, "total_device_budget": 109, "permiss": 109, "equal": 109, "proportion": 109, "streamabl": 109, "streamable_budget": 109, "requested_budget": 109, "get_automatic_weight_streaming_budget": 109, "weight_streaming_exampl": 109, "_rendered_examples_python": 110, "_rendered_examples_jupyt": 110, "acoust": 111, "speech": 111, "quartznet": 111, "contextnet": 111, "subword": 111, "piec": 111, "excit": 111, "se": 111, "audio": 111, "transcrib": 111, "speedup": 111, "obtain": [111, 116], "feedforward": 111, "cnn": 111, "uniformli": 111, "resolut": 111, "highli": [111, 112], "compound": 111, "coeffici": 111, "b0": 111, "corpu": 111, "english": 111, "supervis": 111, "walkthrough": 111, "overal": 111, "adopt": 111, "mobilenetv2": 111, "classif": 111, "imagenet": 111, "imagenett": 111, "qat": 111, "simul": 111, "hand": 112, "consider": 112, "concurr": 112, "grpc": 112, "aforement": 112, "familiar": 112, "resnet50": 112, "torchhub": 112, "docker": 112, "login": 112, "xx": 112, "yy": 112, "mm": 112, "publish": 112, "22": 112, "pwd": 112, "scratch_spac": 112, "nvcr": 112, "py3": 112, "proce": 112, "hub": 112, "_validate_not_a_forked_repo": 112, "suggest": 112, "simplest": 112, "model_repositori": 112, "pbtxt": 112, "pytorch_libtorch": 112, "input__0": 112, "data_typ": 112, "type_fp32": 112, "output__0": 112, "exact": 112, "encourag": 112, "rm": 112, "8000": 112, "8001": 112, "8002": 112, "the_model_repositori": 112, "tritonserv": 112, "spin": 112, "proceed": 112, "flesh": 112, "img1": 112, "hakaimagazin": 112, "wp": 112, "gulf": 112, "bird": 112, "attrdict": 112, "pyindex": 112, "tritoncli": 112, "jump": 112, "firstli": 112, "resiz": 112, "pil": 112, "httpclient": 112, "triton_to_np_dtyp": 112, "rn50_preprocess": 112, "img_path": 112, "img": 112, "centercrop": 112, "485": 112, "456": 112, "406": 112, "229": 112, "transformed_img": 112, "inferenceservercli": 112, "localhost": 112, "secondli": 112, "inferinput": 112, "set_data_from_numpi": 112, "binary_data": 112, "inferrequestedoutput": 112, "class_count": 112, "lastli": 112, "send": 112, "model_nam": 112, "inference_output": 112, "as_numpi": 112, "468750": 112, "90": 112, "523438": 112, "92": 112, "664062": 112, "429688": 112, "136": 112, "234375": 112, "confidence_scor": 112, "classification_index": 112, "eagerli": 113, "swap": 113, "exactli": 113, "_tracer": 113, "queri": 113, "attn_weight": 113, "compiler_dynamic_shap": 113, "inputs_bs2": 113, "mymodul": 114, "linear1": 114, "linear2": 114, "linear3": 114, "40": 114, "__myl_mulsum_myl0_0": 114, "layertyp": 114, "kgen": 114, "__mye116_dconst": 114, "__myln_k_arg__bb1_2": 114, "tacticnam": 114, "__myl_mulsum_0xfa6c1858aea1b13b03f90165d7149ec6": 114, "streamid": 114, "__myl_addresmulsum_myl0_1": 114, "__mye131_dconst": 114, "addmm_constant_0": 114, "addmm_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 114, "__myln_k_arg__bb1_3": 114, "__myl_addresmulsum_0xb3915d7ebfe48be45b6d49083479e12f": 114, "__myl_addresmulsumadd_myl0_2": 114, "__mye146_dconst": 114, "addmm_2_constant_0": 114, "addmm_2_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 114, "addmm_1_constant_0": 114, "addmm_1_add_broadcast_to_same_shape_lhs_broadcast_constantfloat": 114, "__myl_addresmulsumadd_0xcdd0085ad25f5f45ac5fafb72acbffd6": 114, "__myl_mulsumaddcas_myl0_0": 114, "__mye112_dconst": 114, "__myl_mulsumaddcas_0xacf8f5dd9be2f3e7bb09cdddeac6c936": 114, "__myl_resmulsumaddcas_myl0_1": 114, "__mye127_dconst": 114, "addmm_1_add_broadcast_to_same_shape_lhs_broadcast_constanthalf": 114, "__myl_resmulsumaddcas_0x5a3b318b5a1c97b7d5110c0291481337": 114, "__myl_resmulsumadd_myl0_2": 114, "__mye142_dconst": 114, "__myl_resmulsumadd_0x3fad91127c640fd6db771aa9cde67db0": 114, "libtorchtrt_runtim": 115, "dl_open": 115, "ld_preload": 115, "load_librari": 115, "wl": 115, "ltorchtrt": 115, "torchtrt_runtime_exampl": 115, "libtorchtrt_plugin": 115, "neglig": 115, "thread": 115, "alert": 115, "switch": 115, "mismatch": 115, "crash": 115, "sacrif": 115, "incur": 115, "intens": 115, "trt_ep": 116, "stai": 116, "trt_t": 116, "ergonom": 117, "deleg": 117, "believ": 117, "amen": 117, "artifact": 117, "pack": 117, "year": 117, "superset": 117, "codebas": 117, "immedi": 117, "traceabl": 117, "scriptabl": 117, "neural": 118, "deconvolut": 118, "scripted_model": 118}, "objects": {"": [[5, 0, 1, "c.STR", "STR"], [9, 0, 1, "c.TORCHTRT_API", "TORCHTRT_API"], [11, 0, 1, "c.TORCHTRT_HIDDEN", "TORCHTRT_HIDDEN"], [7, 0, 1, "c.TORCH_TENSORRT_MAJOR_VERSION", "TORCH_TENSORRT_MAJOR_VERSION"], [8, 0, 1, "c.TORCH_TENSORRT_MINOR_VERSION", "TORCH_TENSORRT_MINOR_VERSION"], [6, 0, 1, "c.TORCH_TENSORRT_PATCH_VERSION", "TORCH_TENSORRT_PATCH_VERSION"], [12, 0, 1, "c.TORCH_TENSORRT_VERSION", "TORCH_TENSORRT_VERSION"], [10, 0, 1, "c.XSTR", "XSTR"], [0, 1, 1, "_CPPv4N14torch_tensorrt8DataTypeE", "torch_tensorrt::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value", "torch_tensorrt::DataType::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE", "torch_tensorrt::DataType::DataType"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEv", "torch_tensorrt::DataType::DataType"], [0, 3, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeE5Value", "torch_tensorrt::DataType::DataType::t"], [0, 3, 1, "_CPPv4N14torch_tensorrt8DataType8DataTypeEN3c1010ScalarTypeE", "torch_tensorrt::DataType::DataType::t"], [0, 4, 1, "_CPPv4N14torch_tensorrt8DataType5ValueE", "torch_tensorrt::DataType::Value"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kBoolE", "torch_tensorrt::DataType::Value::kBool"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kCharE", "torch_tensorrt::DataType::Value::kChar"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value7kDoubleE", "torch_tensorrt::DataType::Value::kDouble"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value6kFloatE", "torch_tensorrt::DataType::Value::kFloat"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kHalfE", "torch_tensorrt::DataType::Value::kHalf"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value4kIntE", "torch_tensorrt::DataType::Value::kInt"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kLongE", "torch_tensorrt::DataType::Value::kLong"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE", "torch_tensorrt::DataType::Value::kUnknown"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kBoolE", "torch_tensorrt::DataType::kBool"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kCharE", "torch_tensorrt::DataType::kChar"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value7kDoubleE", "torch_tensorrt::DataType::kDouble"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value6kFloatE", "torch_tensorrt::DataType::kFloat"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kHalfE", "torch_tensorrt::DataType::kHalf"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value4kIntE", "torch_tensorrt::DataType::kInt"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value5kLongE", "torch_tensorrt::DataType::kLong"], [0, 5, 1, "_CPPv4N14torch_tensorrt8DataType5Value8kUnknownE", "torch_tensorrt::DataType::kUnknown"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypecv5ValueEv", "torch_tensorrt::DataType::operator Value"], [0, 2, 1, "_CPPv4N14torch_tensorrt8DataTypecvbEv", "torch_tensorrt::DataType::operator bool"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeneE8DataType", "torch_tensorrt::DataType::operator!="], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE", "torch_tensorrt::DataType::operator!="], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeneE8DataType", "torch_tensorrt::DataType::operator!=::other"], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeneEN8DataType5ValueE", "torch_tensorrt::DataType::operator!=::other"], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType", "torch_tensorrt::DataType::operator=="], [0, 2, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE", "torch_tensorrt::DataType::operator=="], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqE8DataType", "torch_tensorrt::DataType::operator==::other"], [0, 3, 1, "_CPPv4NK14torch_tensorrt8DataTypeeqEN8DataType5ValueE", "torch_tensorrt::DataType::operator==::other"], [46, 1, 1, "_CPPv4N14torch_tensorrt6DeviceE", "torch_tensorrt::Device"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device6DeviceEv", "torch_tensorrt::Device::Device"], [1, 1, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypeE", "torch_tensorrt::Device::DeviceType"], [46, 1, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypeE", "torch_tensorrt::Device::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEv", "torch_tensorrt::Device::DeviceType::DeviceType"], [1, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [1, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [46, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeE5Value", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [46, 3, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType10DeviceTypeEN3c1010DeviceTypeE", "torch_tensorrt::Device::DeviceType::DeviceType::t"], [1, 4, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE", "torch_tensorrt::Device::DeviceType::Value"], [46, 4, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5ValueE", "torch_tensorrt::Device::DeviceType::Value"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::Value::kDLA"], [46, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::Value::kDLA"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::Value::kGPU"], [46, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::Value::kGPU"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kDLAE", "torch_tensorrt::Device::DeviceType::kDLA"], [1, 5, 1, "_CPPv4N14torch_tensorrt6Device10DeviceType5Value4kGPUE", "torch_tensorrt::Device::DeviceType::kGPU"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv", "torch_tensorrt::Device::DeviceType::operator Value"], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypecv5ValueEv", "torch_tensorrt::Device::DeviceType::operator Value"], [1, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv", "torch_tensorrt::Device::DeviceType::operator bool"], [46, 2, 1, "_CPPv4N14torch_tensorrt6Device10DeviceTypecvbEv", "torch_tensorrt::Device::DeviceType::operator bool"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!="], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!="], [1, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!=::other"], [46, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeneE10DeviceType", "torch_tensorrt::Device::DeviceType::operator!=::other"], [1, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator=="], [46, 2, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator=="], [1, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator==::other"], [46, 3, 1, "_CPPv4NK14torch_tensorrt6Device10DeviceTypeeqE10DeviceType", "torch_tensorrt::Device::DeviceType::operator==::other"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device18allow_gpu_fallbackE", "torch_tensorrt::Device::allow_gpu_fallback"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device11device_typeE", "torch_tensorrt::Device::device_type"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device8dla_coreE", "torch_tensorrt::Device::dla_core"], [46, 6, 1, "_CPPv4N14torch_tensorrt6Device6gpu_idE", "torch_tensorrt::Device::gpu_id"], [17, 4, 1, "_CPPv4N14torch_tensorrt16EngineCapabilityE", "torch_tensorrt::EngineCapability"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE", "torch_tensorrt::EngineCapability::kDLA_STANDALONE"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE", "torch_tensorrt::EngineCapability::kSAFETY"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE", "torch_tensorrt::EngineCapability::kSTANDARD"], [47, 1, 1, "_CPPv4N14torch_tensorrt11GraphInputsE", "torch_tensorrt::GraphInputs"], [47, 6, 1, "_CPPv4N14torch_tensorrt11GraphInputs15input_signatureE", "torch_tensorrt::GraphInputs::input_signature"], [47, 6, 1, "_CPPv4N14torch_tensorrt11GraphInputs6inputsE", "torch_tensorrt::GraphInputs::inputs"], [48, 1, 1, "_CPPv4N14torch_tensorrt5InputE", "torch_tensorrt::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input"], [48, 2, 1, "_CPPv4N14torch_tensorrt5Input5InputEv", "torch_tensorrt::Input::Input"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::dtype"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::format"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::max_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::min_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::opt_shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataType12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::shape"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN2at6TensorE", "torch_tensorrt::Input::Input::tensor"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputEN3c108ArrayRefI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEE8DataTypeNSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 3, 1, "_CPPv4N14torch_tensorrt5Input5InputENSt6vectorI7int64_tEENSt6vectorIdEE12TensorFormat", "torch_tensorrt::Input::Input::tensor_domain"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input5dtypeE", "torch_tensorrt::Input::dtype"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input6formatE", "torch_tensorrt::Input::format"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9max_shapeE", "torch_tensorrt::Input::max_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9min_shapeE", "torch_tensorrt::Input::min_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input9opt_shapeE", "torch_tensorrt::Input::opt_shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input5shapeE", "torch_tensorrt::Input::shape"], [48, 6, 1, "_CPPv4N14torch_tensorrt5Input13tensor_domainE", "torch_tensorrt::Input::tensor_domain"], [2, 1, 1, "_CPPv4N14torch_tensorrt12TensorFormatE", "torch_tensorrt::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEv", "torch_tensorrt::TensorFormat::TensorFormat"], [2, 3, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatE5Value", "torch_tensorrt::TensorFormat::TensorFormat::t"], [2, 3, 1, "_CPPv4N14torch_tensorrt12TensorFormat12TensorFormatEN2at12MemoryFormatE", "torch_tensorrt::TensorFormat::TensorFormat::t"], [2, 4, 1, "_CPPv4N14torch_tensorrt12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::Value"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE", "torch_tensorrt::TensorFormat::Value::kChannelsLast"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE", "torch_tensorrt::TensorFormat::Value::kContiguous"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE", "torch_tensorrt::TensorFormat::Value::kUnknown"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value13kChannelsLastE", "torch_tensorrt::TensorFormat::kChannelsLast"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value11kContiguousE", "torch_tensorrt::TensorFormat::kContiguous"], [2, 5, 1, "_CPPv4N14torch_tensorrt12TensorFormat5Value8kUnknownE", "torch_tensorrt::TensorFormat::kUnknown"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatcv5ValueEv", "torch_tensorrt::TensorFormat::operator Value"], [2, 2, 1, "_CPPv4N14torch_tensorrt12TensorFormatcvbEv", "torch_tensorrt::TensorFormat::operator bool"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat", "torch_tensorrt::TensorFormat::operator!="], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator!="], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneE12TensorFormat", "torch_tensorrt::TensorFormat::operator!=::other"], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormatneEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator!=::other"], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat", "torch_tensorrt::TensorFormat::operator=="], [2, 2, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator=="], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqE12TensorFormat", "torch_tensorrt::TensorFormat::operator==::other"], [2, 3, 1, "_CPPv4NK14torch_tensorrt12TensorFormateqEN12TensorFormat5ValueE", "torch_tensorrt::TensorFormat::operator==::other"], [37, 2, 1, "_CPPv4N14torch_tensorrt15dump_build_infoEv", "torch_tensorrt::dump_build_info"], [35, 2, 1, "_CPPv4N14torch_tensorrt14get_build_infoEv", "torch_tensorrt::get_build_info"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability15kDLA_STANDALONEE", "torch_tensorrt::kDLA_STANDALONE"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability7kSAFETYE", "torch_tensorrt::kSAFETY"], [17, 5, 1, "_CPPv4N14torch_tensorrt16EngineCapability9kSTANDARDE", "torch_tensorrt::kSTANDARD"], [16, 4, 1, "_CPPv4N14torch_tensorrt7logging5LevelE", "torch_tensorrt::logging::Level"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE", "torch_tensorrt::logging::Level::kDEBUG"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kERRORE", "torch_tensorrt::logging::Level::kERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE", "torch_tensorrt::logging::Level::kGRAPH"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level5kINFOE", "torch_tensorrt::logging::Level::kINFO"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE", "torch_tensorrt::logging::Level::kINTERNAL_ERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE", "torch_tensorrt::logging::Level::kWARNING"], [24, 2, 1, "_CPPv4N14torch_tensorrt7logging24get_is_colored_output_onEv", "torch_tensorrt::logging::get_is_colored_output_on"], [22, 2, 1, "_CPPv4N14torch_tensorrt7logging18get_logging_prefixEv", "torch_tensorrt::logging::get_logging_prefix"], [23, 2, 1, "_CPPv4N14torch_tensorrt7logging24get_reportable_log_levelEv", "torch_tensorrt::logging::get_reportable_log_level"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kDEBUGE", "torch_tensorrt::logging::kDEBUG"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kERRORE", "torch_tensorrt::logging::kERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level6kGRAPHE", "torch_tensorrt::logging::kGRAPH"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level5kINFOE", "torch_tensorrt::logging::kINFO"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level15kINTERNAL_ERRORE", "torch_tensorrt::logging::kINTERNAL_ERROR"], [16, 5, 1, "_CPPv4N14torch_tensorrt7logging5Level8kWARNINGE", "torch_tensorrt::logging::kWARNING"], [26, 2, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log"], [26, 3, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log::lvl"], [26, 3, 1, "_CPPv4N14torch_tensorrt7logging3logE5LevelNSt6stringE", "torch_tensorrt::logging::log::msg"], [27, 2, 1, "_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb", "torch_tensorrt::logging::set_is_colored_output_on"], [27, 3, 1, "_CPPv4N14torch_tensorrt7logging24set_is_colored_output_onEb", "torch_tensorrt::logging::set_is_colored_output_on::colored_output_on"], [28, 2, 1, "_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE", "torch_tensorrt::logging::set_logging_prefix"], [28, 3, 1, "_CPPv4N14torch_tensorrt7logging18set_logging_prefixENSt6stringE", "torch_tensorrt::logging::set_logging_prefix::prefix"], [25, 2, 1, "_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level", "torch_tensorrt::logging::set_reportable_log_level"], [25, 3, 1, "_CPPv4N14torch_tensorrt7logging24set_reportable_log_levelE5Level", "torch_tensorrt::logging::set_reportable_log_level::lvl"], [3, 1, 1, "_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE", "torch_tensorrt::ptq::Int8CacheCalibrator"], [3, 7, 1, "_CPPv4I0EN14torch_tensorrt3ptq19Int8CacheCalibratorE", "torch_tensorrt::ptq::Int8CacheCalibrator::Algorithm"], [3, 2, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE", "torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator"], [3, 3, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibrator19Int8CacheCalibratorERKNSt6stringE", "torch_tensorrt::ptq::Int8CacheCalibrator::Int8CacheCalibrator::cache_file_path"], [3, 2, 1, "_CPPv4N14torch_tensorrt3ptq19Int8CacheCalibratorcvPN8nvinfer115IInt8CalibratorEEv", "torch_tensorrt::ptq::Int8CacheCalibrator::operator nvinfer1::IInt8Calibrator*"], [4, 1, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator"], [4, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator::Algorithm"], [4, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq14Int8CalibratorE", "torch_tensorrt::ptq::Int8Calibrator::DataLoaderUniquePtr"], [4, 2, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::cache_file_path"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::dataloader"], [4, 3, 1, "_CPPv4N14torch_tensorrt3ptq14Int8Calibrator14Int8CalibratorE19DataLoaderUniquePtrRKNSt6stringEb", "torch_tensorrt::ptq::Int8Calibrator::Int8Calibrator::use_cache"], [4, 2, 1, "_CPPv4N14torch_tensorrt3ptq14Int8CalibratorcvPN8nvinfer115IInt8CalibratorEEv", "torch_tensorrt::ptq::Int8Calibrator::operator nvinfer1::IInt8Calibrator*"], [29, 2, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator"], [29, 7, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator::Algorithm"], [29, 3, 1, "_CPPv4I0EN14torch_tensorrt3ptq26make_int8_cache_calibratorE19Int8CacheCalibratorI9AlgorithmERKNSt6stringE", "torch_tensorrt::ptq::make_int8_cache_calibrator::cache_file_path"], [30, 2, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator"], [30, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::Algorithm"], [30, 7, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::DataLoader"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::cache_file_path"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::dataloader"], [30, 3, 1, "_CPPv4I00EN14torch_tensorrt3ptq20make_int8_calibratorE14Int8CalibratorI9Algorithm10DataLoaderE10DataLoaderRKNSt6stringEb", "torch_tensorrt::ptq::make_int8_calibrator::use_cache"], [36, 2, 1, "_CPPv4N14torch_tensorrt10set_deviceEKi", "torch_tensorrt::set_device"], [36, 3, 1, "_CPPv4N14torch_tensorrt10set_deviceEKi", "torch_tensorrt::set_device::gpu_id"], [49, 1, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpecE", "torch_tensorrt::torchscript::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 2, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorIN3c108ArrayRefI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorINSt6vectorI7int64_tEEEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::fixed_sizes"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecEN5torch3jit6IValueE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::input_signature"], [49, 3, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec11CompileSpecENSt6vectorI5InputEE", "torch_tensorrt::torchscript::CompileSpec::CompileSpec::inputs"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec19allow_shape_tensorsE", "torch_tensorrt::torchscript::CompileSpec::allow_shape_tensors"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec10capabilityE", "torch_tensorrt::torchscript::CompileSpec::capability"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec5debugE", "torch_tensorrt::torchscript::CompileSpec::debug"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec6deviceE", "torch_tensorrt::torchscript::CompileSpec::device"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec12disable_tf32E", "torch_tensorrt::torchscript::CompileSpec::disable_tf32"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec20dla_global_dram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_global_dram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec19dla_local_dram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_local_dram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec13dla_sram_sizeE", "torch_tensorrt::torchscript::CompileSpec::dla_sram_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec18enabled_precisionsE", "torch_tensorrt::torchscript::CompileSpec::enabled_precisions"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec12graph_inputsE", "torch_tensorrt::torchscript::CompileSpec::graph_inputs"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14min_block_sizeE", "torch_tensorrt::torchscript::CompileSpec::min_block_size"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec20num_avg_timing_itersE", "torch_tensorrt::torchscript::CompileSpec::num_avg_timing_iters"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14ptq_calibratorE", "torch_tensorrt::torchscript::CompileSpec::ptq_calibrator"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec5refitE", "torch_tensorrt::torchscript::CompileSpec::refit"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec24require_full_compilationE", "torch_tensorrt::torchscript::CompileSpec::require_full_compilation"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14sparse_weightsE", "torch_tensorrt::torchscript::CompileSpec::sparse_weights"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec22torch_executed_modulesE", "torch_tensorrt::torchscript::CompileSpec::torch_executed_modules"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec18torch_executed_opsE", "torch_tensorrt::torchscript::CompileSpec::torch_executed_ops"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec24truncate_long_and_doubleE", "torch_tensorrt::torchscript::CompileSpec::truncate_long_and_double"], [49, 6, 1, "_CPPv4N14torch_tensorrt11torchscript11CompileSpec14workspace_sizeE", "torch_tensorrt::torchscript::CompileSpec::workspace_size"], [31, 2, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support"], [31, 3, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support::method_name"], [31, 3, 1, "_CPPv4N14torch_tensorrt11torchscript29check_method_operator_supportERKN5torch3jit6ModuleENSt6stringE", "torch_tensorrt::torchscript::check_method_operator_support::module"], [32, 2, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile"], [32, 3, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile::info"], [32, 3, 1, "_CPPv4N14torch_tensorrt11torchscript7compileERKN5torch3jit6ModuleE11CompileSpec", "torch_tensorrt::torchscript::compile::module"], [34, 2, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::info"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::method_name"], [34, 3, 1, "_CPPv4N14torch_tensorrt11torchscript28convert_method_to_trt_engineERKN5torch3jit6ModuleENSt6stringE11CompileSpec", "torch_tensorrt::torchscript::convert_method_to_trt_engine::module"], [33, 2, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::device"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::engine"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::input_binding_names"], [33, 3, 1, "_CPPv4N14torch_tensorrt11torchscript26embed_engine_in_new_moduleERKNSt6stringE6DeviceRKNSt6vectorINSt6stringEEERKNSt6vectorINSt6stringEEE", "torch_tensorrt::torchscript::embed_engine_in_new_module::output_binding_names"], [76, 8, 0, "-", "torch_tensorrt"]], "torch_tensorrt": [[76, 9, 1, "", "Device"], [76, 9, 1, "", "DeviceType"], [76, 9, 1, "", "EngineCapability"], [76, 9, 1, "", "Input"], [76, 9, 1, "", "MutableTorchTensorRTModule"], [76, 12, 1, "", "compile"], [76, 12, 1, "", "convert_method_to_trt_engine"], [76, 9, 1, "", "dtype"], [116, 8, 0, "-", "dynamo"], [72, 8, 0, "-", "fx"], [76, 12, 1, "", "load"], [73, 8, 0, "-", "logging"], [76, 9, 1, "", "memory_format"], [75, 8, 0, "-", "runtime"], [76, 12, 1, "", "save"], [77, 8, 0, "-", "ts"]], "torch_tensorrt.Device": [[76, 10, 1, "", "__init__"], [76, 11, 1, "", "device_type"], [76, 11, 1, "", "dla_core"], [76, 11, 1, "", "gpu_id"]], "torch_tensorrt.DeviceType": [[76, 11, 1, "", "DLA"], [76, 11, 1, "", "GPU"], [76, 11, 1, "", "UNKNOWN"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.EngineCapability": [[76, 11, 1, "", "DLA_STANDALONE"], [76, 11, 1, "", "SAFETY"], [76, 11, 1, "", "STANDARD"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.Input": [[76, 10, 1, "", "__init__"], [76, 11, 1, "", "dtype"], [76, 10, 1, "", "example_tensor"], [76, 11, 1, "", "format"], [76, 10, 1, "", "from_tensor"], [76, 10, 1, "", "from_tensors"]], "torch_tensorrt.MutableTorchTensorRTModule": [[76, 10, 1, "", "__init__"], [76, 10, 1, "", "compile"], [76, 10, 1, "", "refit_gm"]], "torch_tensorrt.dtype": [[76, 11, 1, "", "b"], [76, 11, 1, "", "bf16"], [76, 11, 1, "", "f16"], [76, 11, 1, "", "f32"], [76, 11, 1, "", "f64"], [76, 11, 1, "", "f8"], [76, 11, 1, "", "i32"], [76, 11, 1, "", "i64"], [76, 11, 1, "", "i8"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"], [76, 11, 1, "", "u8"], [76, 11, 1, "", "unknown"]], "torch_tensorrt.dynamo": [[71, 9, 1, "", "CompilationSettings"], [71, 12, 1, "", "compile"], [71, 12, 1, "", "export"], [71, 12, 1, "", "refit_module_weights"], [71, 12, 1, "", "trace"]], "torch_tensorrt.fx": [[72, 9, 1, "", "InputTensorSpec"], [72, 9, 1, "", "TRTInterpreter"], [72, 9, 1, "", "TRTInterpreterResult"], [72, 9, 1, "", "TRTModule"], [72, 12, 1, "", "compile"]], "torch_tensorrt.logging": [[73, 9, 1, "", "debug"], [73, 9, 1, "", "errors"], [73, 9, 1, "", "graphs"], [73, 9, 1, "", "info"], [73, 9, 1, "", "internal_errors"], [73, 9, 1, "", "warnings"]], "torch_tensorrt.memory_format": [[76, 11, 1, "", "cdhw32"], [76, 11, 1, "", "chw16"], [76, 11, 1, "", "chw2"], [76, 11, 1, "", "chw32"], [76, 11, 1, "", "chw4"], [76, 11, 1, "", "dhwc"], [76, 11, 1, "", "dhwc8"], [76, 11, 1, "", "dla_hwc4"], [76, 11, 1, "", "dla_linear"], [76, 11, 1, "", "hwc"], [76, 11, 1, "", "hwc16"], [76, 11, 1, "", "hwc8"], [76, 11, 1, "", "linear"], [76, 10, 1, "", "to"], [76, 10, 1, "", "try_from"], [76, 10, 1, "", "try_to"]], "torch_tensorrt.runtime": [[75, 9, 1, "", "PythonTorchTensorRTModule"], [75, 9, 1, "", "TorchTensorRTModule"], [75, 12, 1, "", "set_multi_device_safe_mode"]], "torch_tensorrt.runtime.PythonTorchTensorRTModule": [[75, 10, 1, "", "__init__"], [75, 10, 1, "", "cudagraphs_validate_shapes"], [75, 10, 1, "", "disable_profiling"], [75, 10, 1, "", "enable_profiling"], [75, 10, 1, "", "forward"], [75, 10, 1, "", "get_layer_info"]], "torch_tensorrt.runtime.TorchTensorRTModule": [[75, 10, 1, "", "__init__"], [75, 10, 1, "", "forward"], [75, 10, 1, "", "get_extra_state"], [75, 10, 1, "", "set_extra_state"]], "torch_tensorrt.ts": [[77, 12, 1, "", "TensorRTCompileSpec"], [77, 12, 1, "", "check_method_op_support"], [77, 12, 1, "", "compile"], [77, 12, 1, "", "convert_method_to_trt_engine"], [77, 12, 1, "", "embed_engine_in_new_module"], [74, 8, 0, "-", "ptq"]], "torch_tensorrt.ts.ptq": [[74, 9, 1, "", "CacheCalibrator"], [74, 9, 1, "", "CalibrationAlgo"], [74, 9, 1, "", "DataLoaderCalibrator"]], "torch_tensorrt.ts.ptq.CalibrationAlgo": [[74, 11, 1, "", "ENTROPY_CALIBRATION"], [74, 11, 1, "", "ENTROPY_CALIBRATION_2"], [74, 11, 1, "", "LEGACY_CALIBRATION"], [74, 11, 1, "", "MINMAX_CALIBRATION"]]}, "objtypes": {"0": "c:macro", "1": "cpp:class", "2": "cpp:function", "3": "cpp:functionParam", "4": "cpp:enum", "5": "cpp:enumerator", "6": "cpp:member", "7": "cpp:templateParam", "8": "py:module", "9": "py:class", "10": "py:method", "11": "py:attribute", "12": "py:function"}, "objnames": {"0": ["c", "macro", "C macro"], "1": ["cpp", "class", "C++ class"], "2": ["cpp", "function", "C++ function"], "3": ["cpp", "functionParam", "C++ function parameter"], "4": ["cpp", "enum", "C++ enum"], "5": ["cpp", "enumerator", "C++ enumerator"], "6": ["cpp", "member", "C++ member"], "7": ["cpp", "templateParam", "C++ template parameter"], "8": ["py", "module", "Python module"], "9": ["py", "class", "Python class"], "10": ["py", "method", "Python method"], "11": ["py", "attribute", "Python attribute"], "12": ["py", "function", "Python function"]}, "titleterms": {"class": [0, 1, 2, 3, 4, 20, 21, 38, 40, 41, 50, 71, 72, 74, 75, 76], "datatyp": 0, "document": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 46, 47, 48, 49, 61, 69, 85, 86], "devic": [1, 46, 115], "devicetyp": 1, "nest": [1, 46], "relationship": [1, 3, 4, 46, 48], "tensorformat": 2, "templat": [3, 4, 29, 30], "int8cachecalibr": 3, "inherit": [3, 4, 48], "base": [3, 4, 48, 80], "type": [3, 4, 46, 48, 54], "int8calibr": 4, "defin": [5, 6, 7, 8, 9, 10, 11, 12, 19, 50, 108], "str": 5, "torch_tensorrt_patch_vers": 6, "torch_tensorrt_major_vers": 7, "torch_tensorrt_minor_vers": 8, "torchtrt_api": 9, "xstr": 10, "torchtrt_hidden": 11, "torch_tensorrt_vers": 12, "directori": [13, 14, 15, 51], "cpp": [13, 18, 19, 20, 21, 56], "subdirectori": [13, 14], "includ": [14, 18, 19, 20, 21], "torch_tensorrt": [15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 45, 67, 71, 72, 73, 74, 75, 76, 77, 102, 104, 105, 117], "file": [15, 18, 19, 20, 21, 42, 43, 44, 45, 50, 51], "enum": [16, 17, 38, 39, 50, 74, 76], "level": [16, 80, 82, 83], "enginecap": 17, "log": [18, 22, 23, 24, 25, 26, 27, 28, 39, 42, 73], "h": [18, 19, 20, 21, 42, 43, 44, 45, 56], "content": [18, 19, 20, 21, 38, 39, 40, 41, 80, 81, 82, 83, 84, 85], "definit": [18, 19, 20, 21, 83, 94, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "By": [18, 19], "namespac": [18, 19, 20, 21, 38, 39, 40, 41, 50], "macro": [19, 43], "ptq": [20, 29, 30, 40, 44, 74, 91, 108], "function": [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 50, 61, 71, 72, 75, 76, 77, 108], "get_logging_prefix": 22, "get_reportable_log_level": 23, "get_is_colored_output_on": 24, "set_reportable_log_level": 25, "set_is_colored_output_on": 27, "set_logging_prefix": 28, "make_int8_cache_calibr": 29, "make_int8_calibr": 30, "torchscript": [31, 32, 33, 34, 41, 60, 66, 69, 88, 89, 92, 116, 117], "check_method_operator_support": 31, "compil": [32, 57, 59, 63, 64, 66, 68, 69, 89, 94, 97, 100, 101, 102, 103, 104, 105, 106, 107, 109, 111, 113, 114, 116, 117], "embed_engine_in_new_modul": 33, "convert_method_to_trt_engin": 34, "get_build_info": 35, "set_devic": 36, "dump_build_info": 37, "program": [42, 43, 44, 45, 63, 100, 115], "list": [42, 43, 44, 45, 83], "struct": [46, 47, 48, 49, 50], "graphinput": 47, "input": [48, 102, 104], "compilespec": 49, "torch": [50, 61, 63, 64, 65, 66, 68, 69, 89, 90, 92, 93, 95, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117], "tensorrt": [50, 58, 61, 63, 64, 65, 66, 69, 89, 90, 92, 93, 95, 99, 100, 106, 107, 108, 110, 111, 112, 113, 114, 115, 116, 117], "c": [50, 61, 66, 68, 69, 89, 91, 111], "api": [50, 51, 61, 66, 69], "hierarchi": 50, "full": [50, 51], "torchtrtc": [52, 89], "convers": [53, 57, 59, 60], "phase": [53, 55, 56, 57, 58, 59], "node": 53, "evalu": [53, 54, 70], "convert": [53, 54, 60, 65, 70, 89, 93], "write": [54, 60, 62, 93, 95], "dynamo": [54, 62, 69, 71, 106, 107, 116, 117], "implement": [54, 93], "registr": 54, "capabl": 54, "valid": 54, "contract": [54, 60], "exampl": [54, 62, 82, 84, 94], "convolut": 54, "oper": [54, 64, 70, 89, 95], "decomposit": 54, "addmm": [54, 55], "lower": [55, 57, 59, 62], "pass": [55, 62], "us": [55, 61, 89, 90, 92, 93, 95, 101, 102, 103, 104, 105, 106, 107, 108, 111, 113], "eliminatecommonsubexpress": 55, "elimin": 55, "dead": 55, "code": [55, 69, 82], "except": 55, "Or": 55, "pattern": 55, "redund": 55, "guard": 55, "freez": 55, "modul": [55, 88, 89, 99, 117], "fuse": 55, "branch": 55, "linear": 55, "flatten": 55, "graph": [55, 58, 117], "tupl": 55, "fallback": [55, 56], "peephol": 55, "optim": [55, 68, 112], "remov": 55, "contigu": 55, "dropout": 55, "To": 55, "unpack": 55, "logsoftmax": 55, "unrol": 55, "loop": [55, 108], "replac": [55, 82], "tile": 55, "repeat": 55, "partit": [56, 57, 59], "partitoninfo": 56, "segmentedblock": 56, "shape_analysi": 56, "automat": [56, 109], "depend": [56, 66, 98, 110], "awar": [56, 111], "runtim": [57, 58, 59, 75, 94, 115], "background": [58, 60], "engin": [58, 65, 95, 96, 97], "executor": 58, "op": [58, 65, 95], "construct": 58, "result": 58, "serial": [58, 64, 68], "deseri": 58, "abi": [58, 66], "version": [58, 66], "format": [58, 117], "system": [59, 66], "overview": [59, 67], "what": 60, "guarante": 60, "respons": 60, "context": [60, 80, 109], "arg": [60, 81], "weight": [60, 100, 108, 109], "other": 60, "advic": 60, "link": [61, 82], "develop": 61, "avail": 61, "layer": 61, "expect": 61, "dimens": 61, "python": [61, 66, 68, 69, 88, 90, 91], "sometim": 61, "easier": 61, "read": 61, "pytorch": [61, 65, 69, 92, 95, 106, 107, 111], "native_op": 61, "ir": [61, 116, 117], "aten": 62, "basic": 62, "requir": 62, "regist": [62, 89], "export": [63, 68, 105, 113], "customiz": [63, 64], "set": [63, 64, 99, 101, 105, 112], "under": [63, 89, 113], "hood": [63, 89, 113], "trace": 63, "backend": [64, 102, 103, 104, 106, 107], "kei": 64, "featur": 64, "custom": [64, 89, 93, 95, 97, 101, 113], "usag": [64, 100, 101], "after": 64, "model": [64, 65, 69, 94, 95, 98, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 114, 116], "perform": 64, "coverag": 64, "feasibl": 64, "dynam": [64, 102, 111, 113], "shape": [64, 102, 111, 113], "support": [64, 70], "recompil": [64, 102], "condit": 64, "fx": [65, 69, 72, 111, 117], "frontend": [65, 66, 69, 92, 111, 117], "user": [65, 69], "guid": [65, 69], "acc": 65, "tracer": 65, "fx2trt": 65, "how": [65, 80, 91], "add": 65, "miss": 65, "instal": [66, 87], "precompil": 66, "binari": 66, "specif": 66, "cuda": [66, 101, 104], "nightli": 66, "build": [66, 67, 80, 112], "onli": 66, "from": [66, 92], "sourc": 66, "linux": 66, "packag": [66, 115], "addit": 66, "option": [66, 68, 80, 81, 83, 102, 104, 109, 117], "distribut": 66, "No": 66, "librari": [66, 115], "standalon": 66, "releas": 66, "debug": 66, "pre": [66, 108], "cxx11": 66, "choos": 66, "right": 66, "window": [66, 94], "step": [66, 68, 112], "advanc": [66, 100, 101], "setup": 66, "troubleshoot": 66, "altern": 66, "cmake": 66, "nativ": 66, "aarch64": 66, "jetson": 66, "prerequisit": [66, 67], "environ": 66, "cli": [66, 69], "jetpack": 67, "6": [67, 84], "1": [67, 68, 84, 112], "quick": 68, "start": [68, 69], "2": [68, 84, 85, 112], "deploi": [68, 108, 111, 115], "deploy": 68, "In": [69, 100], "framework": 69, "infer": [69, 102, 103, 104, 105, 108, 112], "nvidia": 69, "gpu": 69, "get": 69, "tutori": [69, 110], "zoo": [69, 98, 110], "contributor": 69, "indic": 69, "legaci": [69, 111, 117], "further": 69, "inform": 69, "current": 70, "through": 70, "ts": [74, 77, 117], "submodul": 76, "comput": 78, "time": [78, 117], "changelog": 79, "configur": 80, "project": 80, "wide": 80, "html": 80, "theme": [80, 86], "toc": 80, "page": 80, "tabl": [80, 81, 82, 83, 84, 85], "mod": 81, "test_py_modul": 81, "gener": [81, 106, 107], "index": 81, "paramet": 81, "data": 81, "paragraph": [82, 85], "markup": 82, "inlin": 82, "math": 82, "meta": 82, "block": 82, "liter": 82, "line": 82, "quot": 82, "doctest": 82, "emphas": 82, "number": [82, 83], "sidebar": 82, "ch": 82, "ien": 82, "The": [82, 89], "creativ": 82, "A": 82, "refer": 82, "footnot": 82, "citat": [82, 91], "glossari": 82, "target": 82, "direct": 82, "center": 82, "text": 82, "imag": [82, 83], "figur": 82, "admonit": 82, "And": 82, "wai": 82, "topic": 82, "rubric": 82, "titl": 82, "compound": 82, "download": [82, 87], "enumer": 83, "field": 83, "bullet": 83, "second": 83, "But": 83, "deeper": 83, "down": 83, "rabbit": 83, "hole": 83, "hlist": 83, "grid": 83, "giant": 83, "can": 83, "have": 83, "caption": [83, 86], "like": 83, "thi": [83, 86], "one": 83, "long": [84, 86], "sticki": 84, "nav": 84, "menu": [84, 86], "3": [84, 112], "4": 84, "5": 84, "7": 84, "8": 84, "9": 84, "10": 84, "11": 84, "12": 84, "13": 84, "14": 84, "15": 84, "16": 84, "17": 84, "18": 84, "19": 84, "20": 84, "submenu": 84, "subsubmenu": 84, "structur": 85, "element": 85, "section": 85, "subsect": 85, "subsubsect": 85, "demo": 86, "an": 86, "incred": 86, "via": 87, "git": 87, "creat": [88, 91], "work": [88, 89], "save": [88, 99, 116], "disk": 88, "quickstart": 89, "unsupport": 89, "post": 91, "train": [91, 108, 111], "quantiz": [91, 108, 111], "your": [91, 112], "own": 91, "applic": 91, "directli": 92, "overload": 93, "metadata": 93, "our": [93, 95], "cross": 94, "import": [94, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109], "kernel": 95, "within": 95, "test": 95, "wrap": 95, "insert": 95, "cach": [96, 97, 100], "bert": [96, 104, 111], "jit": [97, 113], "aot": [97, 113], "mutabl": 99, "initi": 99, "make": [99, 100], "modif": 99, "stabl": [99, 103], "diffus": [99, 103], "huggingfac": 99, "refit": 100, "new": 100, "standard": 100, "workflow": 100, "refitt": 100, "pretrain": 100, "map": 100, "place": 100, "default": [101, 105], "cleanup": [101, 104], "driver": [101, 104], "error": [101, 104], "note": [101, 104], "resnet": 102, "argument": [102, 104], "avoid": 102, "specifi": 102, "befor": 102, "trt": 102, "cudagraph": [105, 115], "integr": 105, "gpt2": 106, "output": [106, 107], "decod": [106, 107], "sentenc": [106, 107], "llama2": 107, "load": [108, 116], "dataset": 108, "loss": 108, "calibr": 108, "tune": 108, "fp8": 108, "stream": 109, "run": 109, "budget": 109, "size": 109, "manag": 109, "notebook": 111, "citrinet": 111, "efficientnet": 111, "mask": 111, "languag": 111, "mlm": 111, "hug": 111, "face": 111, "transform": 111, "acceler": 111, "serv": [111, 112], "resnet50": 111, "lenet": 111, "deep": 111, "learn": 111, "object": 111, "detect": 111, "ssd": 111, "int8": 111, "triton": 112, "up": 112, "server": 112, "client": 112, "queri": 112, "constraint": 113, "mix": 114, "precis": 114, "libtorchtrt": 115, "so": 115, "plugin": 115, "multi": 115, "safe": 115, "mode": 115, "exportedprogram": 116, "b": 116, "explain": 117, "just": 117, "accept": 117, "return": 117, "ahead": 117, "dla": 118}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx.ext.intersphinx": 1, "sphinx.ext.todo": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}})
\ No newline at end of file
diff --git a/docs/sg_execution_times.html b/docs/sg_execution_times.html
index e86b2f97aa..3d8c138084 100644
--- a/docs/sg_execution_times.html
+++ b/docs/sg_execution_times.html
@@ -10,7 +10,7 @@
- Computation times — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Computation times — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/changelog.html b/docs/src/pytorch-sphinx-theme/docs/changelog.html
index 83687253b7..722f4859cc 100644
--- a/docs/src/pytorch-sphinx-theme/docs/changelog.html
+++ b/docs/src/pytorch-sphinx-theme/docs/changelog.html
@@ -10,7 +10,7 @@
- Changelog — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Changelog — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/configuring.html b/docs/src/pytorch-sphinx-theme/docs/configuring.html
index 9f5413c795..d004da318c 100644
--- a/docs/src/pytorch-sphinx-theme/docs/configuring.html
+++ b/docs/src/pytorch-sphinx-theme/docs/configuring.html
@@ -10,7 +10,7 @@
- Configuration — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Configuration — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/api.html b/docs/src/pytorch-sphinx-theme/docs/demo/api.html
index 33e44dd2f8..92895ef7c0 100644
--- a/docs/src/pytorch-sphinx-theme/docs/demo/api.html
+++ b/docs/src/pytorch-sphinx-theme/docs/demo/api.html
@@ -10,7 +10,7 @@
- 5. :mod:`test_py_module` — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ 5. :mod:`test_py_module` — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/demo.html b/docs/src/pytorch-sphinx-theme/docs/demo/demo.html
index e5e3a2818b..2b9910e404 100644
--- a/docs/src/pytorch-sphinx-theme/docs/demo/demo.html
+++ b/docs/src/pytorch-sphinx-theme/docs/demo/demo.html
@@ -12,7 +12,7 @@
- 3. Paragraph Level Markup — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ 3. Paragraph Level Markup — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -664,7 +664,7 @@ 3.4.4.
3.4.5. Code Blocks¶
# parsed-literal test
-curl -O http://someurl/release-v2.6.0.dev0+e43833d.tar-gz
+curl -O http://someurl/release-v2.6.0.dev0+bc95015.tar-gz
Code Blocks can have captions.¶
{
diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html b/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html
index 0f495c8d92..0aee59635b 100644
--- a/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html
+++ b/docs/src/pytorch-sphinx-theme/docs/demo/lists_tables.html
@@ -10,7 +10,7 @@
- 4. Lists & Tables — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ 4. Lists & Tables — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/long.html b/docs/src/pytorch-sphinx-theme/docs/demo/long.html
index 5ff749ccab..ccc32a6079 100644
--- a/docs/src/pytorch-sphinx-theme/docs/demo/long.html
+++ b/docs/src/pytorch-sphinx-theme/docs/demo/long.html
@@ -10,7 +10,7 @@
- 1. Long Sticky Nav — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ 1. Long Sticky Nav — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/demo/structure.html b/docs/src/pytorch-sphinx-theme/docs/demo/structure.html
index 97bca07882..705fab8300 100644
--- a/docs/src/pytorch-sphinx-theme/docs/demo/structure.html
+++ b/docs/src/pytorch-sphinx-theme/docs/demo/structure.html
@@ -10,7 +10,7 @@
- 1. Structural Elements — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ 1. Structural Elements — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/index.html b/docs/src/pytorch-sphinx-theme/docs/index.html
index 34d6711acb..f3f727a041 100644
--- a/docs/src/pytorch-sphinx-theme/docs/index.html
+++ b/docs/src/pytorch-sphinx-theme/docs/index.html
@@ -10,7 +10,7 @@
- <no title> — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ <no title> — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/src/pytorch-sphinx-theme/docs/installing.html b/docs/src/pytorch-sphinx-theme/docs/installing.html
index f1b24af379..7aa2f8855f 100644
--- a/docs/src/pytorch-sphinx-theme/docs/installing.html
+++ b/docs/src/pytorch-sphinx-theme/docs/installing.html
@@ -10,7 +10,7 @@
- Installation — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Installation — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/ts/creating_torchscript_module_in_python.html b/docs/ts/creating_torchscript_module_in_python.html
index 57df80d557..f2cd314d4e 100644
--- a/docs/ts/creating_torchscript_module_in_python.html
+++ b/docs/ts/creating_torchscript_module_in_python.html
@@ -10,7 +10,7 @@
- Creating a TorchScript Module — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Creating a TorchScript Module — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/ts/getting_started_with_cpp_api.html b/docs/ts/getting_started_with_cpp_api.html
index 4b67101919..c76744ed07 100644
--- a/docs/ts/getting_started_with_cpp_api.html
+++ b/docs/ts/getting_started_with_cpp_api.html
@@ -10,7 +10,7 @@
- Using Torch-TensorRT in C++ — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Using Torch-TensorRT in C++ — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/ts/getting_started_with_python_api.html b/docs/ts/getting_started_with_python_api.html
index b6c102ae0d..eb1b1a44d7 100644
--- a/docs/ts/getting_started_with_python_api.html
+++ b/docs/ts/getting_started_with_python_api.html
@@ -10,7 +10,7 @@
- Using Torch-TensorRT in Python — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Using Torch-TensorRT in Python — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/ts/ptq.html b/docs/ts/ptq.html
index d315b04b55..180589f98f 100644
--- a/docs/ts/ptq.html
+++ b/docs/ts/ptq.html
@@ -10,7 +10,7 @@
- Post Training Quantization (PTQ) — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Post Training Quantization (PTQ) — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/ts/torchscript_frontend_from_pytorch.html b/docs/ts/torchscript_frontend_from_pytorch.html
index a5c8e40aab..4434857da4 100644
--- a/docs/ts/torchscript_frontend_from_pytorch.html
+++ b/docs/ts/torchscript_frontend_from_pytorch.html
@@ -10,7 +10,7 @@
- Using Torch-TensorRT TorchScript Frontend Directly From PyTorch — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Using Torch-TensorRT TorchScript Frontend Directly From PyTorch — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/converter_overloading.html b/docs/tutorials/_rendered_examples/dynamo/converter_overloading.html
index 300479caa0..44a66d6ec6 100644
--- a/docs/tutorials/_rendered_examples/dynamo/converter_overloading.html
+++ b/docs/tutorials/_rendered_examples/dynamo/converter_overloading.html
@@ -10,7 +10,7 @@
- Overloading Torch-TensorRT Converters with Custom Converters — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Overloading Torch-TensorRT Converters with Custom Converters — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.html b/docs/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.html
new file mode 100644
index 0000000000..ad5d896048
--- /dev/null
+++ b/docs/tutorials/_rendered_examples/dynamo/cross_runtime_compilation_for_windows.html
@@ -0,0 +1,883 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ Cross runtime compilation for windows example — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ -
+
+
+ Docs
+
+ >
+
+
+
+ - Cross runtime compilation for windows example
+
+
+ -
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Shortcuts
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Note
+Go to the end
+to download the full example code
+
+cross runtime compilation limitations:
+The cross compile and saved model can only be loaded in Windows, it can no longer be loaded in Linux
+The cross compile and saved model can only be loaded in the same Compute Capability as the Linux which it was cross compiled
+(for example, if the model was cross compiled in Linux with GeForceRTX 4080 which has Compute Capability of 8.9,
+It cannot be loaded in Windows with GeForceRTX 3080 which has Compute Capability of 8.6)
+
+Cross runtime compilation for windows example¶
+Compile and save the Resnet Model using Torch-TensorRT in Linux:
+python examples/dynamo/cross_runtime_compilation_for_windows.py –path trt_resnet.ep
+Load the Resnet Model saved in Windows:
+python examples/dynamo/cross_runtime_compilation_for_windows.py –path trt_resnet.ep –load True
+
+Imports and Model Definition¶
+import argparse
+import platform
+
+import torch
+import torch_tensorrt as torchtrt
+import torchvision.models as models
+
+PARSER = argparse.ArgumentParser(
+ description="Cross runtime comilation for windows example: Resnet Model"
+)
+PARSER.add_argument(
+ "--load", default=False, type=bool, required=False, help="Load the model in Windows"
+)
+PARSER.add_argument(
+ "--path",
+ type=str,
+ required=True,
+ help="Path to the saved model file",
+)
+
+args = PARSER.parse_args()
+torch.manual_seed(0)
+model = models.resnet18().eval().cuda()
+input = torch.rand((1, 3, 224, 224)).to("cuda")
+inputs = [input]
+
+
+According to the argument, it is either cross compile and save resnet model for windows in Linux
+or load the saved resnet model in Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+if args.load:
+ # load the saved model in Windows
+ if platform.system() != "Windows" or platform.machine() != "AMD64":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be loaded in Windows system"
+ )
+ loaded_model = torchtrt.load_cross_compiled_exported_program(args.path).module()
+ print(f"model has been successfully loaded from ${args.path}")
+ # inference
+ trt_output = loaded_model(input)
+ print(f"inference result: {trt_output}")
+else:
+ if platform.system() != "Linux" or platform.architecture()[0] != "64bit":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be compiled in Linux system"
+ )
+ compile_spec = {
+ "debug": True,
+ "min_block_size": 1,
+ }
+ torchtrt.cross_compile_for_windows(
+ model, file_path=args.path, inputs=inputs, **compile_spec
+ )
+ print(
+ f"model has been successfully cross compiled and saved in Linux to {args.path}"
+ )
+
+
+Total running time of the script: ( 0 minutes 0.000 seconds)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: Cookies Policy.
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html b/docs/tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html
index 6d1d3c5dda..f7fc6b84dc 100644
--- a/docs/tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html
+++ b/docs/tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html
@@ -10,7 +10,7 @@
- Using Custom Kernels within TensorRT Engines with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Using Custom Kernels within TensorRT Engines with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/engine_caching_bert_example.html b/docs/tutorials/_rendered_examples/dynamo/engine_caching_bert_example.html
index 11e5fcd607..2281d30f06 100644
--- a/docs/tutorials/_rendered_examples/dynamo/engine_caching_bert_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/engine_caching_bert_example.html
@@ -10,7 +10,7 @@
- Engine Caching (BERT) — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Engine Caching (BERT) — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/engine_caching_example.html b/docs/tutorials/_rendered_examples/dynamo/engine_caching_example.html
index caaf7f9350..2b3b2d7f93 100644
--- a/docs/tutorials/_rendered_examples/dynamo/engine_caching_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/engine_caching_example.html
@@ -10,7 +10,7 @@
- Engine Caching — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Engine Caching — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/index.html b/docs/tutorials/_rendered_examples/dynamo/index.html
index 47e90ab73d..398eb2dbd1 100644
--- a/docs/tutorials/_rendered_examples/dynamo/index.html
+++ b/docs/tutorials/_rendered_examples/dynamo/index.html
@@ -10,7 +10,7 @@
- Dependencies — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Dependencies — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -488,6 +488,9 @@ Model Zoo
Torch Export with Cudagraphs
+
+sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py
+ cross runtime compilation limitations:
Refitting Torch-TensorRT Programs with New Weights
Refitting Torch-TensorRT Programs with New Weights
diff --git a/docs/tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.html b/docs/tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.html
index c3b00c04ee..5f86cb8e27 100644
--- a/docs/tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.html
@@ -10,7 +10,7 @@
- Mutable Torch TensorRT Module — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Mutable Torch TensorRT Module — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/refit_engine_example.html b/docs/tutorials/_rendered_examples/dynamo/refit_engine_example.html
index 03ec4c07f7..6b9b593b4c 100644
--- a/docs/tutorials/_rendered_examples/dynamo/refit_engine_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/refit_engine_example.html
@@ -10,7 +10,7 @@
- Refitting Torch-TensorRT Programs with New Weights — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Refitting Torch-TensorRT Programs with New Weights — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.html b/docs/tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.html
index b8152240d5..040198395e 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.html
@@ -10,7 +10,7 @@
- Torch Compile Advanced Usage — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch Compile Advanced Usage — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.html b/docs/tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.html
index 3a2dc1a285..a8b5a14a11 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.html
@@ -10,7 +10,7 @@
- Compiling ResNet with dynamic shapes using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling ResNet with dynamic shapes using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.html b/docs/tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.html
index de2b866292..441f75d7d4 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.html
@@ -10,7 +10,7 @@
- Compiling Stable Diffusion model using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling Stable Diffusion model using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.html b/docs/tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.html
index 9cd1c16e65..157b723576 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.html
@@ -10,7 +10,7 @@
- Compiling BERT using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling BERT using the torch.compile backend — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.html b/docs/tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.html
index 609fc48c41..0a40531228 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.html
@@ -10,7 +10,7 @@
- Torch Export with Cudagraphs — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch Export with Cudagraphs — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_export_gpt2.html b/docs/tutorials/_rendered_examples/dynamo/torch_export_gpt2.html
index 6c766fbc21..e704950ef1 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_export_gpt2.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_export_gpt2.html
@@ -10,7 +10,7 @@
- Compiling GPT2 using the dynamo backend — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling GPT2 using the dynamo backend — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/torch_export_llama2.html b/docs/tutorials/_rendered_examples/dynamo/torch_export_llama2.html
index 46d3d02703..61259a4bf1 100644
--- a/docs/tutorials/_rendered_examples/dynamo/torch_export_llama2.html
+++ b/docs/tutorials/_rendered_examples/dynamo/torch_export_llama2.html
@@ -10,7 +10,7 @@
- Compiling Llama2 using the dynamo backend — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compiling Llama2 using the dynamo backend — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/vgg16_ptq.html b/docs/tutorials/_rendered_examples/dynamo/vgg16_ptq.html
index 649decfbf8..8060f3a221 100644
--- a/docs/tutorials/_rendered_examples/dynamo/vgg16_ptq.html
+++ b/docs/tutorials/_rendered_examples/dynamo/vgg16_ptq.html
@@ -10,7 +10,7 @@
- Deploy Quantized Models using Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Deploy Quantized Models using Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/dynamo/weight_streaming_example.html b/docs/tutorials/_rendered_examples/dynamo/weight_streaming_example.html
index dc3e38dca4..c4e3e6b770 100644
--- a/docs/tutorials/_rendered_examples/dynamo/weight_streaming_example.html
+++ b/docs/tutorials/_rendered_examples/dynamo/weight_streaming_example.html
@@ -10,7 +10,7 @@
- Weight Streaming — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Weight Streaming — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/_rendered_examples/index.html b/docs/tutorials/_rendered_examples/index.html
index 53c3dd6f0a..873e9770c0 100644
--- a/docs/tutorials/_rendered_examples/index.html
+++ b/docs/tutorials/_rendered_examples/index.html
@@ -10,7 +10,7 @@
- Torch-TensorRT Tutorials — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch-TensorRT Tutorials — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -273,7 +273,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
@@ -490,6 +490,9 @@ Model Zoo
Torch Export with Cudagraphs
+
+sphx_glr_tutorials__rendered_examples_dynamo_cross_runtime_compilation_for_windows.py
+ cross runtime compilation limitations:
Refitting Torch-TensorRT Programs with New Weights
Refitting Torch-TensorRT Programs with New Weights
diff --git a/docs/tutorials/notebooks.html b/docs/tutorials/notebooks.html
index ae0b76f2d9..a930a2584f 100644
--- a/docs/tutorials/notebooks.html
+++ b/docs/tutorials/notebooks.html
@@ -10,7 +10,7 @@
- Legacy notebooks — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Legacy notebooks — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/tutorials/serving_torch_tensorrt_with_triton.html b/docs/tutorials/serving_torch_tensorrt_with_triton.html
index fad19dd7c2..a974712825 100644
--- a/docs/tutorials/serving_torch_tensorrt_with_triton.html
+++ b/docs/tutorials/serving_torch_tensorrt_with_triton.html
@@ -10,7 +10,7 @@
- Serving a Torch-TensorRT model with Triton — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Serving a Torch-TensorRT model with Triton — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/dynamic_shapes.html b/docs/user_guide/dynamic_shapes.html
index bbbf462f79..4ba862426c 100644
--- a/docs/user_guide/dynamic_shapes.html
+++ b/docs/user_guide/dynamic_shapes.html
@@ -10,7 +10,7 @@
- Dynamic shapes with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Dynamic shapes with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/mixed_precision.html b/docs/user_guide/mixed_precision.html
index ad94ac87d4..96578d4101 100644
--- a/docs/user_guide/mixed_precision.html
+++ b/docs/user_guide/mixed_precision.html
@@ -10,7 +10,7 @@
- Compile Mixed Precision models with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Compile Mixed Precision models with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/runtime.html b/docs/user_guide/runtime.html
index eab011e947..a00365166c 100644
--- a/docs/user_guide/runtime.html
+++ b/docs/user_guide/runtime.html
@@ -10,7 +10,7 @@
- Deploying Torch-TensorRT Programs — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Deploying Torch-TensorRT Programs — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/saving_models.html b/docs/user_guide/saving_models.html
index d6199a9e15..a584c6aa16 100644
--- a/docs/user_guide/saving_models.html
+++ b/docs/user_guide/saving_models.html
@@ -10,7 +10,7 @@
- Saving models compiled with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Saving models compiled with Torch-TensorRT — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/torch_tensorrt_explained.html b/docs/user_guide/torch_tensorrt_explained.html
index c4cd64970d..4514965c06 100644
--- a/docs/user_guide/torch_tensorrt_explained.html
+++ b/docs/user_guide/torch_tensorrt_explained.html
@@ -10,7 +10,7 @@
- Torch-TensorRT Explained — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ Torch-TensorRT Explained — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/docs/user_guide/using_dla.html b/docs/user_guide/using_dla.html
index 03ce69f629..5f4aff74bb 100644
--- a/docs/user_guide/using_dla.html
+++ b/docs/user_guide/using_dla.html
@@ -10,7 +10,7 @@
- DLA — Torch-TensorRT v2.6.0.dev0+e43833d documentation
+ DLA — Torch-TensorRT v2.6.0.dev0+bc95015 documentation
@@ -275,7 +275,7 @@
- v2.6.0.dev0+e43833d
+ v2.6.0.dev0+bc95015
diff --git a/examples/dynamo/cross_runtime_compilation_for_windows.py b/examples/dynamo/cross_runtime_compilation_for_windows.py
new file mode 100644
index 0000000000..184470ffa0
--- /dev/null
+++ b/examples/dynamo/cross_runtime_compilation_for_windows.py
@@ -0,0 +1,82 @@
+"""
+.. _resnet_cross_runtime_compilation_for_windows_example:
+
+cross runtime compilation limitations:
+The cross compile and saved model can only be loaded in Windows, it can no longer be loaded in Linux
+The cross compile and saved model can only be loaded in the same Compute Capability as the Linux which it was cross compiled
+(for example, if the model was cross compiled in Linux with GeForceRTX 4080 which has Compute Capability of 8.9,
+It cannot be loaded in Windows with GeForceRTX 3080 which has Compute Capability of 8.6)
+
+Cross runtime compilation for windows example
+======================================================
+
+Compile and save the Resnet Model using Torch-TensorRT in Linux:
+
+python examples/dynamo/cross_runtime_compilation_for_windows.py --path trt_resnet.ep
+
+Load the Resnet Model saved in Windows:
+
+python examples/dynamo/cross_runtime_compilation_for_windows.py --path trt_resnet.ep --load True
+
+"""
+
+# %%
+# Imports and Model Definition
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+import argparse
+import platform
+
+import torch
+import torch_tensorrt as torchtrt
+import torchvision.models as models
+
+PARSER = argparse.ArgumentParser(
+ description="Cross runtime comilation for windows example: Resnet Model"
+)
+PARSER.add_argument(
+ "--load", default=False, type=bool, required=False, help="Load the model in Windows"
+)
+PARSER.add_argument(
+ "--path",
+ type=str,
+ required=True,
+ help="Path to the saved model file",
+)
+
+args = PARSER.parse_args()
+torch.manual_seed(0)
+model = models.resnet18().eval().cuda()
+input = torch.rand((1, 3, 224, 224)).to("cuda")
+inputs = [input]
+
+# %%
+# According to the argument, it is either cross compile and save resnet model for windows in Linux
+# or load the saved resnet model in Windows
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+if args.load:
+ # load the saved model in Windows
+ if platform.system() != "Windows" or platform.machine() != "AMD64":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be loaded in Windows system"
+ )
+ loaded_model = torchtrt.load_cross_compiled_exported_program(args.path).module()
+ print(f"model has been successfully loaded from ${args.path}")
+ # inference
+ trt_output = loaded_model(input)
+ print(f"inference result: {trt_output}")
+else:
+ if platform.system() != "Linux" or platform.architecture()[0] != "64bit":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be compiled in Linux system"
+ )
+ compile_spec = {
+ "debug": True,
+ "min_block_size": 1,
+ }
+ torchtrt.cross_compile_for_windows(
+ model, file_path=args.path, inputs=inputs, **compile_spec
+ )
+ print(
+ f"model has been successfully cross compiled and saved in Linux to {args.path}"
+ )
diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py
index 66cff1f9ea..eb2642755a 100644
--- a/py/torch_tensorrt/_compile.py
+++ b/py/torch_tensorrt/_compile.py
@@ -2,6 +2,7 @@
import collections.abc
import logging
+import platform
from enum import Enum
from typing import Any, Callable, List, Optional, Sequence, Set
@@ -29,11 +30,27 @@
from torch_tensorrt.dynamo._compiler import (
convert_exported_program_to_serialized_trt_engine as dynamo_convert_exported_program_to_serialized_trt_engine,
)
+ from torch_tensorrt.dynamo._compiler import (
+ cross_compile_for_windows as dynamo_cross_compile_for_windows,
+ )
+ from torch_tensorrt.dynamo._compiler import (
+ load_cross_compiled_exported_program as dynamo_load_cross_compiled_exported_program,
+ )
+ from torch_tensorrt.dynamo._compiler import (
+ save_cross_compiled_exported_program as dynamo_save_cross_compiled_exported_program,
+ )
from torch_tensorrt.dynamo._tracer import trace as dynamo_trace
logger = logging.getLogger(__name__)
-__all__ = ["compile", "convert_method_to_trt_engine", "save", "load"]
+__all__ = [
+ "compile",
+ "cross_compile_for_windows",
+ "load_cross_compiled_exported_program",
+ "convert_method_to_trt_engine",
+ "save",
+ "load",
+]
def _non_fx_input_interface(
@@ -281,6 +298,105 @@ def compile(
raise RuntimeError("Module is an unknown format or the ir requested is unknown")
+def cross_compile_for_windows(
+ module: torch.nn.Module,
+ file_path: str,
+ inputs: Optional[Sequence[Input | torch.Tensor]] = None,
+ arg_inputs: Optional[Sequence[Sequence[Any]]] = None,
+ kwarg_inputs: Optional[dict[Any, Any]] = None,
+ enabled_precisions: Optional[Set[torch.dtype | dtype]] = None,
+ **kwargs: Any,
+) -> None:
+ """Compile a PyTorch module using TensorRT in Linux for Inference in Windows
+
+ Takes an existing PyTorch module and a set of settings to configure the compiler
+ and it will convert methods to AOT graphs which call equivalent TensorRT serialized
+ engine info into the disk in the specified file_path user provided.
+ It will then allow user to load the deserialized model from the disk in Windows.
+ Note: the model cross compiled for windows in Linux environmen can only be loaded
+ in Windows.
+
+ Argument:
+ module (torch.nn.Module): Source module
+ file_path (str): the file path to store the serialized module into the disk
+
+ Keyword Arguments:
+ inputs (List[Union(torch_tensorrt.Input, torch.Tensor)]): **Required** List of specifications of input shape, dtype and memory layout for inputs to the module. This argument is required. Input Sizes can be specified as torch sizes, tuples or lists. dtypes can be specified using
+ torch datatypes or torch_tensorrt datatypes and you can use either torch devices or the torch_tensorrt device type enum
+ to select device type. ::
+
+ inputs=[
+ torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+ torch_tensorrt.Input(
+ min_shape=(1, 224, 224, 3),
+ opt_shape=(1, 512, 512, 3),
+ max_shape=(1, 1024, 1024, 3),
+ dtype=torch.int32
+ format=torch.channel_last
+ ), # Dynamic input shape for input #2
+ torch.randn((1, 3, 224, 244)) # Use an example tensor and let torch_tensorrt infer settings
+ ]
+ arg_inputs (Tuple[Any, ...]): Same as inputs. Alias for better understanding with kwarg_inputs.
+ kwarg_inputs (dict[Any, ...]): Optional, kwarg inputs to the module forward function.
+ enabled_precision (Set(Union(torch.dtype, torch_tensorrt.dtype))): The set of datatypes that TensorRT can use when selecting kernels
+ **kwargs: Additional settings for the specific requested strategy (See submodules for more info)
+
+ """
+
+ if platform.system() != "Linux" or platform.architecture()[0] != "64bit":
+ raise RuntimeError(
+ f"Cross compile for windows is only supported on x86-64 Linux architecture, current platform: {platform.system()=}, {platform.architecture()[0]=}"
+ )
+
+ if not file_path:
+ raise ValueError("File path cannot be empty. Please provide a valid file path")
+
+ enabled_precisions_set: Set[dtype | torch.dtype] = (
+ enabled_precisions
+ if enabled_precisions is not None
+ else _defaults.ENABLED_PRECISIONS
+ )
+
+ # Prepare torch and torchtrt inputs
+ if not arg_inputs and not inputs:
+ raise AssertionError("'arg_inputs' and 'inputs' should not both be None.")
+
+ elif arg_inputs and inputs:
+ raise AssertionError(
+ "'arg_inputs' and 'inputs' should not be used at the same time."
+ )
+
+ arg_inputs = inputs or arg_inputs
+
+ if kwarg_inputs is None:
+ kwarg_inputs = {}
+
+ from torch_tensorrt.dynamo.utils import prepare_inputs
+
+ if not isinstance(arg_inputs, collections.abc.Sequence):
+ arg_inputs = [arg_inputs] # type: ignore
+
+ # Export the module
+ torchtrt_arg_inputs = prepare_inputs(arg_inputs)
+ torchtrt_kwarg_inputs = prepare_inputs(kwarg_inputs)
+
+ exp_program = dynamo_trace(
+ module, torchtrt_arg_inputs, kwarg_inputs=torchtrt_kwarg_inputs, **kwargs
+ )
+ logger.debug("successfully exported the module")
+
+ # Compile and save the module
+ trt_gm = dynamo_cross_compile_for_windows(
+ exp_program,
+ arg_inputs=torchtrt_arg_inputs,
+ enabled_precisions=enabled_precisions_set,
+ **kwargs,
+ )
+
+ dynamo_save_cross_compiled_exported_program(trt_gm, file_path)
+ logger.debug("successfully compiled and saved the module for windows")
+
+
def torch_compile(module: torch.nn.Module, **kwargs: Any) -> Any:
"""
Returns a boxed model which is the output of torch.compile.
@@ -406,6 +522,19 @@ def convert_method_to_trt_engine(
raise RuntimeError("Module is an unknown format or the ir requested is unknown")
+def load_cross_compiled_exported_program(file_path: str = "") -> Any:
+ """
+ Load an ExportedProgram file in Windows which was previously cross compiled in Linux
+
+ Arguments:
+ file_path (str): Path to file on the disk
+
+ Raises:
+ ValueError: If the api is not called in windows or there is no file or the file is not a valid ExportedProgram file
+ """
+ return dynamo_load_cross_compiled_exported_program(file_path)
+
+
def load(file_path: str = "") -> Any:
"""
Load either a Torchscript model or ExportedProgram.
diff --git a/py/torch_tensorrt/dynamo/__init__.py b/py/torch_tensorrt/dynamo/__init__.py
index 79bd113ab8..6fabdad633 100644
--- a/py/torch_tensorrt/dynamo/__init__.py
+++ b/py/torch_tensorrt/dynamo/__init__.py
@@ -7,7 +7,13 @@
logger = logging.getLogger(__name__)
if version.parse(sanitized_torch_version()) >= version.parse("2.1.dev"):
- from ._compiler import compile, convert_exported_program_to_serialized_trt_engine
+ from ._compiler import (
+ compile,
+ convert_exported_program_to_serialized_trt_engine,
+ cross_compile_for_windows,
+ load_cross_compiled_exported_program,
+ save_cross_compiled_exported_program,
+ )
from ._exporter import export
from ._refit import refit_module_weights
from ._settings import CompilationSettings
diff --git a/py/torch_tensorrt/dynamo/_compiler.py b/py/torch_tensorrt/dynamo/_compiler.py
index 5e41abe473..730d47a254 100644
--- a/py/torch_tensorrt/dynamo/_compiler.py
+++ b/py/torch_tensorrt/dynamo/_compiler.py
@@ -2,6 +2,7 @@
import collections.abc
import logging
+import platform
import warnings
from typing import Any, Collection, List, Optional, Sequence, Set, Tuple, Union
@@ -19,6 +20,7 @@
parse_non_trt_nodes,
)
from torch_tensorrt.dynamo._engine_cache import BaseEngineCache, DiskEngineCache
+from torch_tensorrt.dynamo._exporter import replace_execute_engine_no_op_node
from torch_tensorrt.dynamo.conversion import (
CompilationSettings,
UnsupportedOperatorException,
@@ -47,6 +49,285 @@
logger = logging.getLogger(__name__)
+def cross_compile_for_windows(
+ exported_program: ExportedProgram,
+ inputs: Optional[Sequence[Sequence[Any]]] = None,
+ *,
+ arg_inputs: Optional[Sequence[Sequence[Any]]] = None,
+ kwarg_inputs: Optional[dict[Any, Any]] = None,
+ device: Optional[Union[Device, torch.device, str]] = _defaults.DEVICE,
+ disable_tf32: bool = _defaults.DISABLE_TF32,
+ assume_dynamic_shape_support: bool = _defaults.ASSUME_DYNAMIC_SHAPE_SUPPORT,
+ sparse_weights: bool = _defaults.SPARSE_WEIGHTS,
+ enabled_precisions: Union[
+ Set[Union[torch.dtype, dtype]], Tuple[Union[torch.dtype, dtype]]
+ ] = _defaults.ENABLED_PRECISIONS,
+ engine_capability: EngineCapability = _defaults.ENGINE_CAPABILITY,
+ make_refittable: bool = _defaults.MAKE_REFITTABLE,
+ debug: bool = _defaults.DEBUG,
+ num_avg_timing_iters: int = _defaults.NUM_AVG_TIMING_ITERS,
+ workspace_size: int = _defaults.WORKSPACE_SIZE,
+ dla_sram_size: int = _defaults.DLA_SRAM_SIZE,
+ dla_local_dram_size: int = _defaults.DLA_LOCAL_DRAM_SIZE,
+ dla_global_dram_size: int = _defaults.DLA_GLOBAL_DRAM_SIZE,
+ truncate_double: bool = _defaults.TRUNCATE_DOUBLE,
+ require_full_compilation: bool = _defaults.REQUIRE_FULL_COMPILATION,
+ min_block_size: int = _defaults.MIN_BLOCK_SIZE,
+ torch_executed_ops: Optional[Collection[Target]] = None,
+ torch_executed_modules: Optional[List[str]] = None,
+ pass_through_build_failures: bool = _defaults.PASS_THROUGH_BUILD_FAILURES,
+ max_aux_streams: Optional[int] = _defaults.MAX_AUX_STREAMS,
+ version_compatible: bool = _defaults.VERSION_COMPATIBLE,
+ optimization_level: Optional[int] = _defaults.OPTIMIZATION_LEVEL,
+ use_python_runtime: bool = _defaults.USE_PYTHON_RUNTIME,
+ use_fast_partitioner: bool = _defaults.USE_FAST_PARTITIONER,
+ enable_experimental_decompositions: bool = _defaults.ENABLE_EXPERIMENTAL_DECOMPOSITIONS,
+ dryrun: bool = _defaults.DRYRUN,
+ hardware_compatible: bool = _defaults.HARDWARE_COMPATIBLE,
+ timing_cache_path: str = _defaults.TIMING_CACHE_PATH,
+ lazy_engine_init: bool = _defaults.LAZY_ENGINE_INIT,
+ cache_built_engines: bool = _defaults.CACHE_BUILT_ENGINES,
+ reuse_cached_engines: bool = _defaults.REUSE_CACHED_ENGINES,
+ engine_cache_dir: str = _defaults.ENGINE_CACHE_DIR,
+ engine_cache_size: int = _defaults.ENGINE_CACHE_SIZE,
+ custom_engine_cache: Optional[BaseEngineCache] = _defaults.CUSTOM_ENGINE_CACHE,
+ use_explicit_typing: bool = _defaults.USE_EXPLICIT_TYPING,
+ use_fp32_acc: bool = _defaults.USE_FP32_ACC,
+ enable_weight_streaming: bool = _defaults.ENABLE_WEIGHT_STREAMING,
+ **kwargs: Any,
+) -> torch.fx.GraphModule:
+ """Compile an ExportedProgram module using TensorRT in Linux for Inference in Windows
+
+ Takes an exported program and a set of settings to configure the compiler
+ and it will convert methods to AOT graphs which call equivalent TensorRT engines
+
+ Arguments:
+ exported_program (torch.export.ExportedProgram): Source module, running torch.export on a ``torch.nn.Module``
+ inputs (Tuple[Any, ...]): List of specifications of input shape, dtype and memory layout for inputs to the module. This argument is required. Input Sizes can be specified as torch sizes, tuples or lists. dtypes can be specified using
+ torch datatypes or torch_tensorrt datatypes and you can use either torch devices or the torch_tensorrt device type enum
+ to select device type.
+
+ .. code-block:: py
+
+ inputs=[
+ torch_tensorrt.Input((1, 3, 224, 224)), # Static NCHW input shape for input #1
+ torch_tensorrt.Input(
+ min_shape=(1, 224, 224, 3),
+ opt_shape=(1, 512, 512, 3),
+ max_shape=(1, 1024, 1024, 3),
+ dtype=torch.int32
+ format=torch.channel_last
+ ), # Dynamic input shape for input #2
+ torch.randn((1, 3, 224, 244)) # Use an example tensor and let torch_tensorrt infer settings
+ ]
+
+ Keyword Arguments:
+ arg_inputs (Tuple[Any, ...]): Same as inputs. Alias for better understanding with kwarg_inputs.
+ kwarg_inputs (dict[Any, ...]): Optional, kwarg inputs to the module forward function.
+ device (Union(torch_tensorrt.Device, torch.device, dict)): Target device for TensorRT engines to run on ::
+
+ device=torch_tensorrt.Device("dla:1", allow_gpu_fallback=True)
+
+ disable_tf32 (bool): Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
+ assume_dynamic_shape_support (bool): Setting this to true enables the converters work for both dynamic and static shapes. Default: False
+ sparse_weights (bool): Enable sparsity for convolution and fully connected layers.
+ enabled_precision (Set(Union(torch.dtype, torch_tensorrt.dtype))): The set of datatypes that TensorRT can use when selecting kernels
+ refit (bool): Enable refitting
+ debug (bool): Enable debuggable engine
+ capability (torch_tensorrt.EngineCapability): Restrict kernel selection to safe gpu kernels or safe dla kernels
+ num_avg_timing_iters (int): Number of averaging timing iterations used to select kernels
+ workspace_size (int): Maximum size of workspace given to TensorRT
+ dla_sram_size (int): Fast software managed RAM used by DLA to communicate within a layer.
+ dla_local_dram_size (int): Host RAM used by DLA to share intermediate tensor data across operations
+ dla_global_dram_size (int): Host RAM used by DLA to store weights and metadata for execution
+ truncate_double (bool): Truncate weights provided in double (float64) to float32
+ calibrator (Union(torch_tensorrt._C.IInt8Calibrator, tensorrt.IInt8Calibrator)): Calibrator object which will provide data to the PTQ system for INT8 Calibration
+ require_full_compilation (bool): Require modules to be compiled end to end or return an error as opposed to returning a hybrid graph where operations that cannot be run in TensorRT are run in PyTorch
+ min_block_size (int): The minimum number of contiguous TensorRT convertible operations in order to run a set of operations in TensorRT
+ torch_executed_ops (Collection[Target]): Set of aten operators that must be run in PyTorch. An error will be thrown if this set is not empty but ``require_full_compilation`` is True
+ torch_executed_modules (List[str]): List of modules that must be run in PyTorch. An error will be thrown if this list is not empty but ``require_full_compilation`` is True
+ pass_through_build_failures (bool): Error out if there are issues during compilation (only applicable to torch.compile workflows)
+ max_aux_stream (Optional[int]): Maximum streams in the engine
+ version_compatible (bool): Build the TensorRT engines compatible with future versions of TensorRT (Restrict to lean runtime operators to provide version forward compatibility for the engines)
+ optimization_level: (Optional[int]): Setting a higher optimization level allows TensorRT to spend longer engine building time searching for more optimization options. The resulting engine may have better performance compared to an engine built with a lower optimization level. The default optimization level is 3. Valid values include integers from 0 to the maximum optimization level, which is currently 5. Setting it to be greater than the maximum level results in identical behavior to the maximum level.
+ use_python_runtime: (bool): Return a graph using a pure Python runtime, reduces options for serialization
+ use_fast_partitioner: (bool): Use the adjacency based partitioning scheme instead of the global partitioner. Adjacency partitioning is faster but may not be optimal. Use the global paritioner (``False``) if looking for best performance
+ enable_experimental_decompositions (bool): Use the full set of operator decompositions. These decompositions may not be tested but serve to make the graph easier to convert to TensorRT, potentially increasing the amount of graphs run in TensorRT.
+ dryrun (bool): Toggle for "Dryrun" mode, running everything except conversion to TRT and logging outputs
+ hardware_compatible (bool): Build the TensorRT engines compatible with GPU architectures other than that of the GPU on which the engine was built (currently works for NVIDIA Ampere and newer)
+ timing_cache_path (str): Path to the timing cache if it exists (or) where it will be saved after compilation
+ lazy_engine_init (bool): Defer setting up engines until the compilation of all engines is complete. Can allow larger models with multiple graph breaks to compile but can lead to oversubscription of GPU memory at runtime.
+ cache_built_engines (bool): Whether to save the compiled TRT engines to storage
+ reuse_cached_engines (bool): Whether to load the compiled TRT engines from storage
+ engine_cache_dir (Optional[str]): Directory to store the cached TRT engines
+ engine_cache_size (Optional[int]): Maximum hard-disk space (bytes) to use for the engine cache, default is 1GB. If the cache exceeds this size, the oldest engines will be removed by default
+ custom_engine_cache (Optional[BaseEngineCache]): Engine cache instance to use for saving and loading engines. Users can provide their own engine cache by inheriting from BaseEngineCache. If used, engine_cache_dir and engine_cache_size will be ignored.
+ use_explicit_typing (bool): This flag enables strong typing in TensorRT compilation which respects the precisions set in the Pytorch model. This is useful when users have mixed precision graphs.
+ use_fp32_acc (bool): This option inserts cast to FP32 nodes around matmul layers and TensorRT ensures the accumulation of matmul happens in FP32. Use this only when FP16 precision is configured in enabled_precisions.
+ enable_weight_streaming (bool): Enable weight streaming.
+ **kwargs: Any,
+ Returns:
+ torch.fx.GraphModule: Compiled FX Module, when run it will execute via TensorRT
+
+ """
+ if platform.system() != "Linux" or platform.architecture()[0] != "64bit":
+ raise RuntimeError(
+ f"Cross compile for windows is only supported on x86-64 Linux architecture, current platform: {platform.system()=}, {platform.architecture()[0]=}"
+ )
+
+ if debug:
+ set_log_level(logger.parent, logging.DEBUG)
+
+ if "truncate_long_and_double" in kwargs.keys():
+ if truncate_double is not _defaults.TRUNCATE_DOUBLE:
+ raise ValueError(
+ 'Provided configuration for "truncate_double" and deprecated API "truncate_long_and_double", please only use "truncate_double"'
+ )
+ else:
+ truncate_double = kwargs["truncate_long_and_double"]
+ warnings.warn(
+ 'Compiler option "truncate_long_and_double" is deprecated in favor of "truncate_double" as int64 is now natively supported, this option will be removed in the next version',
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ if "refit" in kwargs.keys():
+ warnings.warn(
+ "Refit is deprecated. Please use make_refittable=True if you want to enable refitting of the engine.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ if make_refittable:
+ raise ValueError("Use flag make_refittable only. Flag refit is deprecated.")
+ else:
+ make_refittable = kwargs["refit"]
+
+ engine_capability = EngineCapability._from(engine_capability)
+
+ if torch_executed_modules is not None and torch_executed_modules:
+ logger.warning(
+ f"Detected torch_executed_modules was non-empty: {torch_executed_modules}"
+ "\nThis feature is unimplemented in Torch-TRT Dynamo currently."
+ )
+
+ if use_explicit_typing:
+ if len(enabled_precisions) != 1 or not any(
+ x in enabled_precisions for x in {torch.float32, dtype.f32}
+ ):
+ raise AssertionError(
+ f"When use_explicit_typing is enabled, only torch.float32 is allowed in the enabled_precisions but found {enabled_precisions}"
+ )
+
+ if use_fp32_acc:
+ logger.debug(
+ "FP32 accumulation for matmul layers is enabled. This option should only be enabled if the model already has FP16 weights and has no effect if it has FP32 weights. \
+ This flag inserts casts around matmul layers and ensures TensorRT executes the matmul layers in FP16 with FP32 accumulation."
+ )
+
+ if enable_weight_streaming and not use_explicit_typing:
+ raise AssertionError(
+ "When enable_weight_streaming is enabled, it requires use_explicit_typing to be set to True"
+ )
+ # Aliasing inputs to arg_inputs for better understanding
+ if not arg_inputs and not inputs:
+ raise AssertionError("'arg_inputs' and 'inputs' should not both be None.")
+
+ elif arg_inputs and inputs:
+ raise AssertionError(
+ "'arg_inputs' and 'inputs' should not be used at the same time."
+ )
+
+ arg_inputs = inputs or arg_inputs
+
+ if kwarg_inputs is None:
+ kwarg_inputs = {}
+
+ if not isinstance(arg_inputs, collections.abc.Sequence):
+ arg_inputs = [arg_inputs] # type: ignore
+
+ # Prepare torch_trt inputs
+ trt_arg_inputs: Sequence[Input] = prepare_inputs(arg_inputs)
+ trt_kwarg_inputs: Optional[dict[Any, Any]] = prepare_inputs(kwarg_inputs)
+ device = to_torch_tensorrt_device(device)
+ enabled_precisions = {dtype._from(p) for p in enabled_precisions}
+
+ compilation_options = {
+ "enabled_precisions": (
+ enabled_precisions if enabled_precisions else _defaults.ENABLED_PRECISIONS
+ ),
+ "debug": debug,
+ "device": device,
+ "assume_dynamic_shape_support": assume_dynamic_shape_support,
+ "workspace_size": workspace_size,
+ "min_block_size": min_block_size,
+ "torch_executed_ops": (
+ torch_executed_ops if torch_executed_ops is not None else set()
+ ),
+ "pass_through_build_failures": pass_through_build_failures,
+ "max_aux_streams": max_aux_streams,
+ "version_compatible": version_compatible,
+ "optimization_level": optimization_level,
+ "use_python_runtime": False,
+ "truncate_double": truncate_double,
+ "use_fast_partitioner": use_fast_partitioner,
+ "num_avg_timing_iters": num_avg_timing_iters,
+ "enable_experimental_decompositions": enable_experimental_decompositions,
+ "require_full_compilation": require_full_compilation,
+ "disable_tf32": disable_tf32,
+ "sparse_weights": sparse_weights,
+ "make_refittable": make_refittable,
+ "engine_capability": engine_capability,
+ "dla_sram_size": dla_sram_size,
+ "dla_local_dram_size": dla_local_dram_size,
+ "dla_global_dram_size": dla_global_dram_size,
+ "dryrun": dryrun,
+ "hardware_compatible": hardware_compatible,
+ "timing_cache_path": timing_cache_path,
+ "lazy_engine_init": lazy_engine_init,
+ "cache_built_engines": cache_built_engines,
+ "reuse_cached_engines": reuse_cached_engines,
+ "enable_cross_compile_for_windows": True,
+ "enable_weight_streaming": enable_weight_streaming,
+ }
+
+ # disable the following settings is not supported for cross compilation for windows feature
+ unsupported_settings = (
+ "use_python_runtime",
+ "lazy_engine_init",
+ "cache_built_engines",
+ "reuse_cached_engines",
+ )
+ # disable these settings if anything is turned on
+ for key, value in compilation_options.items():
+ if key in unsupported_settings and value:
+ compilation_options[key] = False
+ logger.warning(
+ f"arg: {key} is not supported for cross compilation for windows feature, hence it is disabled."
+ )
+
+ settings = CompilationSettings(**compilation_options)
+ logger.info("Compilation Settings: %s\n", settings)
+ exported_program = pre_export_lowering(exported_program, settings)
+ exported_program = exported_program.run_decompositions(
+ get_decompositions(enable_experimental_decompositions)
+ )
+
+ gm = exported_program.module()
+ logger.debug("Input graph: " + str(gm.graph))
+
+ # Apply lowering on the graph module
+ gm = post_lowering(gm, settings)
+ logger.debug("Lowered Input graph: " + str(gm.graph))
+
+ trt_gm = compile_module(
+ gm,
+ trt_arg_inputs,
+ trt_kwarg_inputs,
+ settings,
+ )
+ return trt_gm
+
+
def compile(
exported_program: ExportedProgram,
inputs: Optional[Sequence[Sequence[Any]]] = None,
@@ -203,6 +484,14 @@ def compile(
stacklevel=2,
)
+ if (
+ "enable_cross_compile_for_windows" in kwargs.keys()
+ and kwargs["enable_cross_compile_for_windows"]
+ ):
+ raise ValueError(
+ "Please use cross_compile_for_windows() api if you want to cross compile the module in Linux for inferencing in Windows."
+ )
+
engine_capability = EngineCapability._from(engine_capability)
if torch_executed_modules is not None and torch_executed_modules:
@@ -304,6 +593,7 @@ def compile(
"refit_identical_engine_weights": refit_identical_engine_weights,
"strip_engine_weights": strip_engine_weights,
"immutable_weights": immutable_weights,
+ "enable_cross_compile_for_windows": False,
"enable_weight_streaming": enable_weight_streaming,
}
@@ -539,7 +829,7 @@ def contains_metadata(gm: torch.fx.GraphModule) -> bool:
# Replace all FX Modules with TRT Modules
for name, trt_module in trt_modules.items():
setattr(partitioned_module, name, trt_module)
- if settings.lazy_engine_init:
+ if settings.lazy_engine_init and not settings.enable_cross_compile_for_windows:
getattr(partitioned_module, name).setup_engine()
# Reset settings object to user specification after fallback to global partitioning mode
@@ -780,3 +1070,58 @@ def convert_exported_program_to_serialized_trt_engine(
serialized_engine: bytes = interpreter_result.serialized_engine
return serialized_engine
+
+
+def save_cross_compiled_exported_program(
+ gm: torch.fx.GraphModule,
+ file_path: str,
+) -> None:
+ """
+ Save cross compiled exported program to disk.
+
+ Arguments:
+ module (torch.fx.GraphModule): Cross compiled Torch-TensorRT module
+ file_path (str): the file path where the exported program will be saved to disk
+ """
+ if not file_path:
+ raise ValueError("File path cannot be empty. Please provide a valid file path")
+
+ from torch_tensorrt.dynamo._exporter import export
+
+ exp_program = export(gm, cross_compile_flag=True)
+ torch.export.save(exp_program, file_path)
+ logger.debug(f"successfully saved the module for windows at {file_path}")
+
+
+def load_cross_compiled_exported_program(file_path: str = "") -> Any:
+ """
+ Load an ExportedProgram file in Windows which was previously cross compiled in Linux
+
+ Arguments:
+ file_path (str): Path to file on the disk
+
+ Raises:
+ ValueError: If the api is not called in windows or there is no file or the file is a valid ExportedProgram file
+ """
+ if not file_path:
+ raise ValueError("File path cannot be empty. Please provide a valid file path")
+
+ if platform.system() != "Windows" or platform.machine() != "AMD64":
+ raise ValueError(
+ "cross runtime compiled model for windows can only be loaded in Windows system"
+ )
+
+ try:
+ logger.debug(f"Loading the provided file {file_path} using torch.export.load()")
+ # TODO: think about how to handle the torch.jit.load route?
+ exp_program = torch.export.load(file_path)
+ except Exception as e:
+ logger.info(
+ f"Loading the provided file {file_path} via torch.export.load() failed with the following error: {e}",
+ exc_info=True,
+ )
+ raise ValueError(
+ f"cross_load the file {file_path} doesn't correspond to a valid ExportedProgram. Please verify the file path."
+ )
+
+ return replace_execute_engine_no_op_node(exp_program)
diff --git a/py/torch_tensorrt/dynamo/_defaults.py b/py/torch_tensorrt/dynamo/_defaults.py
index ee29e95b72..1341ca739f 100644
--- a/py/torch_tensorrt/dynamo/_defaults.py
+++ b/py/torch_tensorrt/dynamo/_defaults.py
@@ -45,6 +45,7 @@
STRIP_ENGINE_WEIGHTS = False
IMMUTABLE_WEIGHTS = False
ENABLE_WEIGHT_STREAMING = False
+ENABLE_CROSS_COMPILE_FOR_WINDOWS = False
def default_device() -> Device:
diff --git a/py/torch_tensorrt/dynamo/_exporter.py b/py/torch_tensorrt/dynamo/_exporter.py
index ae7c09caf8..c7a063d675 100644
--- a/py/torch_tensorrt/dynamo/_exporter.py
+++ b/py/torch_tensorrt/dynamo/_exporter.py
@@ -1,6 +1,7 @@
+import base64
import copy
import operator
-from typing import Any, Dict, Sequence, Tuple, cast
+from typing import Any, Dict, Optional, Sequence, Tuple, cast
import torch
from torch._guards import detect_fake_mode
@@ -16,24 +17,28 @@
OutputSpec,
TensorArgument,
)
+from torch_tensorrt.dynamo.runtime._TorchTensorRTModule import ENGINE_IDX, NAME_IDX
def export(
gm: torch.fx.GraphModule,
+ cross_compile_flag: Optional[bool] = False,
) -> ExportedProgram:
"""Export the result of TensorRT compilation into the desired output format.
Arguments:
gm (torch.fx.GraphModule): Compiled Torch-TensorRT module, generated by ``torch_tensorrt.dynamo.compile``
inputs (torch.Tensor): Torch input tensors
+ cross_compile_flag (bool): Flag to indicated whether it is cross_compilation enabled or not
"""
- patched_module = transform(gm)
+ patched_module = transform(gm, cross_compile_flag)
exp_program = create_trt_exp_program(patched_module)
return exp_program
def transform(
gm: torch.fx.GraphModule,
+ cross_compile_flag: Optional[bool] = False,
) -> torch.fx.GraphModule:
"""
Transforms the graphmodule by inlining Pytorch and TensorRT submodules.
@@ -43,6 +48,7 @@ def transform(
Arguments:
gm (torch.fx.GraphModule): Compiled Torch-TensorRT module, generated by ``torch_tensorrt.dynamo.compile``
inputs (torch.Tensor): Torch input tensors
+ cross_compile_flag (bool): Flag to indicated whether it is cross_compilation enabled or not
Returns an inlined torch.fx.GraphModule
"""
@@ -51,7 +57,7 @@ def transform(
gm = copy.deepcopy(gm)
# Inline TensorRT submodules
- inline_trt_modules(gm)
+ inline_trt_modules(gm, cross_compile_flag)
# Inline pytorch submodules
inline_torch_modules(gm)
@@ -350,7 +356,9 @@ def create_trt_exp_program(
return trt_exp_program
-def inline_trt_modules(gm: torch.fx.GraphModule) -> torch.fx.GraphModule:
+def inline_trt_modules(
+ gm: torch.fx.GraphModule, cross_compile_flag: Optional[bool] = False
+) -> torch.fx.GraphModule:
"""
Replace TRT submodules with trt engine nodes.
"""
@@ -373,25 +381,36 @@ def inline_trt_modules(gm: torch.fx.GraphModule) -> torch.fx.GraphModule:
num_outputs = len(trt_module_node.meta["val"])
# Insert a call_function node to perform inference on TRT engine
with gm.graph.inserting_before(trt_module_node):
- engine_name = f"{name}_engine"
- setattr(gm, engine_name, trt_module.engine)
- engine_node = gm.graph.get_attr(engine_name)
-
- trt_node = gm.graph.call_function(
- torch.ops.tensorrt.execute_engine.default,
- (trt_module_node.args, engine_node),
- )
+ if not cross_compile_flag:
+ # for the normal workflow: use the execute_engine node
+ engine_name = f"{name}_engine"
+ setattr(gm, engine_name, trt_module.engine)
+ engine_node = gm.graph.get_attr(engine_name)
+
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.execute_engine.default,
+ (trt_module_node.args, engine_node),
+ )
+ # meta["val"] should be a lighter version of a tensor. For eg: it should be a FakeTensor (with output shape and dtype properties)
+ # Lighter version of a custom_obj is not defined clearly. meta["val"] does not have any type expectations but
+ # for custom object nodes, it should be CustomObjArgument
+ engine_node.meta["val"] = CustomObjArgument(
+ name=engine_node.name, class_fqn=""
+ )
+ else:
+ # for the cross compile for windows workflow: use the no_op_placeholder node
+ engine_info = trt_module._pack_engine_info()
+ engine_bytes = engine_info[ENGINE_IDX]
+ engine_info[ENGINE_IDX] = base64.b64encode(engine_bytes).decode("utf-8")
+ # insert the no_placeholder node in the graph which should be replaced to the actual execute_engine node while load in the windows
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.no_op_placeholder_for_execute_engine.default,
+ (trt_module_node.args, *engine_info),
+ )
# set trt_node.meta with trt_module_node.meta
assert num_outputs > 0
trt_node.meta["val"] = trt_module_node.meta["val"]
- # meta["val"] should be a lighter version of a tensor. For eg: it should be a FakeTensor (with output shape and dtype properties)
- # Lighter version of a custom_obj is not defined clearly. meta["val"] does not have any type expectations but
- # for custom object nodes, it should be CustomObjArgument
- engine_node.meta["val"] = CustomObjArgument(
- name=engine_node.name, class_fqn=""
- )
-
if num_outputs == 1:
# Insert getitem nodes as outputs (for export serialization to work)
with gm.graph.inserting_after(trt_node):
@@ -411,3 +430,57 @@ def inline_trt_modules(gm: torch.fx.GraphModule) -> torch.fx.GraphModule:
gm.graph.erase_node(trt_module_node)
return gm
+
+
+def replace_execute_engine_no_op_node(
+ exp_program: ExportedProgram,
+) -> ExportedProgram:
+ gm = exp_program.graph_module
+ no_op_placeholder_nodes = []
+ for node in gm.graph.nodes:
+ if "no_op_placeholder_for_execute_engine" in node.name:
+ no_op_placeholder_nodes.append(node)
+ assert len(no_op_placeholder_nodes) > 0
+ for no_op_placeholder_node in no_op_placeholder_nodes:
+ if "val" not in no_op_placeholder_node.meta:
+ raise ValueError(f"metadata info is missing for the node: {node.name}")
+ with gm.graph.inserting_before(no_op_placeholder_node):
+ packed_engine_info = list(no_op_placeholder_node.args[1:])
+ engine_bytes = packed_engine_info[ENGINE_IDX]
+ engine_name = packed_engine_info[NAME_IDX]
+
+ packed_engine_info[ENGINE_IDX] = base64.b64decode(
+ engine_bytes.encode("utf-8")
+ )
+ trt_engine = torch.classes.tensorrt.Engine(tuple(packed_engine_info))
+ setattr(gm, engine_name, trt_engine)
+ engine_node = gm.graph.get_attr(engine_name)
+
+ trt_node = gm.graph.call_function(
+ torch.ops.tensorrt.execute_engine.default,
+ (no_op_placeholder_node.args[0], engine_node),
+ )
+ trt_node.meta["val"] = no_op_placeholder_node.meta["val"]
+ engine_node.meta["val"] = CustomObjArgument(
+ name=engine_node.name, class_fqn=""
+ )
+
+ if len(no_op_placeholder_node.meta["val"]) == 1:
+ with gm.graph.inserting_after(trt_node):
+ getitem_output = gm.graph.call_function(operator.getitem, (trt_node, 0))
+ getitem_output.meta["val"] = trt_node.meta["val"]
+ no_op_placeholder_node.replace_all_uses_with(getitem_output)
+ else:
+ no_op_placeholder_node.replace_all_uses_with(trt_node)
+ getitem_nodes = trt_node.users
+ for idx, getitem_node in enumerate(getitem_nodes):
+ getitem_node.meta["val"] = trt_node.meta["val"][idx]
+
+ gm.graph.erase_node(no_op_placeholder_node)
+
+ gm.delete_all_unused_submodules()
+ gm.graph.eliminate_dead_code()
+ gm.graph.lint()
+ gm.recompile()
+
+ return exp_program
diff --git a/py/torch_tensorrt/dynamo/_settings.py b/py/torch_tensorrt/dynamo/_settings.py
index 05f6f1c0e6..7a22663af3 100644
--- a/py/torch_tensorrt/dynamo/_settings.py
+++ b/py/torch_tensorrt/dynamo/_settings.py
@@ -13,6 +13,7 @@
DLA_LOCAL_DRAM_SIZE,
DLA_SRAM_SIZE,
DRYRUN,
+ ENABLE_CROSS_COMPILE_FOR_WINDOWS,
ENABLE_EXPERIMENTAL_DECOMPOSITIONS,
ENABLE_WEIGHT_STREAMING,
ENABLED_PRECISIONS,
@@ -88,6 +89,8 @@ class CompilationSettings:
strip_engine_weights (bool): Whether to strip the engine weights
immutable_weights (bool): Build non-refittable engines. This is useful for some layers that are not refittable. If this argument is set to true, `strip_engine_weights` and `refit_identical_engine_weights` will be ignored
enable_weight_streaming (bool): Enable weight streaming.
+ enable_cross_compile_for_windows (bool): By default this is False means TensorRT engines can only be executed on the same platform where they were built.
+ True will enable cross-platform compatibility which allows the engine to be built on Linux and run on Windows
"""
enabled_precisions: Set[dtype] = field(default_factory=lambda: ENABLED_PRECISIONS)
@@ -127,6 +130,7 @@ class CompilationSettings:
strip_engine_weights: bool = STRIP_ENGINE_WEIGHTS
immutable_weights: bool = IMMUTABLE_WEIGHTS
enable_weight_streaming: bool = ENABLE_WEIGHT_STREAMING
+ enable_cross_compile_for_windows: bool = ENABLE_CROSS_COMPILE_FOR_WINDOWS
_SETTINGS_TO_BE_ENGINE_INVARIANT = (
diff --git a/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py b/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py
index 02c0407ee7..7ffc02ca3d 100644
--- a/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py
+++ b/py/torch_tensorrt/dynamo/conversion/_TRTInterpreter.py
@@ -316,6 +316,12 @@ def _populate_trt_builder_config(
if tactic_sources is not None:
builder_config.set_tactic_sources(tactic_sources=tactic_sources)
+ if self.compilation_settings.enable_cross_compile_for_windows:
+ builder_config.runtime_platform = trt.RuntimePlatform.WINDOWS_AMD64
+ _LOGGER.info(
+ "Setting runtime_platform as trt.RuntimePlatform.WINDOWS_AMD64"
+ )
+
if self.compilation_settings.enable_weight_streaming:
builder_config.set_flag(trt.BuilderFlag.WEIGHT_STREAMING)
diff --git a/py/torch_tensorrt/dynamo/conversion/impl/slice/base.py b/py/torch_tensorrt/dynamo/conversion/impl/slice/base.py
index 018ac63b8c..a2af840a1f 100644
--- a/py/torch_tensorrt/dynamo/conversion/impl/slice/base.py
+++ b/py/torch_tensorrt/dynamo/conversion/impl/slice/base.py
@@ -32,5 +32,5 @@ def slice(
)
if dynamic_shape:
layer.set_input(2, shape)
- set_layer_name(layer, target, name)
+ set_layer_name(layer, target, name, source_ir)
return layer.get_output(0)
diff --git a/py/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py b/py/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py
index ac2bf1512f..f51707768e 100644
--- a/py/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py
+++ b/py/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py
@@ -493,7 +493,7 @@ def save(module: Any, path: str) -> None:
def load(path: str) -> Any:
# When the model get saved, init_finished is set to False.
# Class is restored to MutableTorchTensorRTModule, and some attribute is deleted
- module = torch.load(path)
+ module = torch.load(path, weights_only=False)
module.pytorch_model = _make_refit_change_trigger(
module.original_model, module.refit_state
)
diff --git a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py
index 5558244bc0..d7cfc6608b 100644
--- a/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py
+++ b/py/torch_tensorrt/dynamo/runtime/_TorchTensorRTModule.py
@@ -133,7 +133,11 @@ def __init__(
self.serialized_engine = serialized_engine
self.engine = None
- if serialized_engine and not self.settings.lazy_engine_init:
+ if (
+ serialized_engine
+ and not self.settings.lazy_engine_init
+ and not self.settings.enable_cross_compile_for_windows
+ ):
self.setup_engine()
def _pack_engine_info(self) -> List[str | bytes]:
@@ -145,16 +149,16 @@ def _pack_engine_info(self) -> List[str | bytes]:
metadata = {"settings": self.settings, "weight_name_map": self.weight_name_map}
target_platform = (
Platform.current_platform()
+ if not self.settings.enable_cross_compile_for_windows
+ else Platform.WIN_X86_64
) # Change to match target for engine
engine_info: List[str | bytes] = [""] * SERIALIZATION_LEN
-
engine_info[ABI_TARGET_IDX] = torch.ops.tensorrt.ABI_VERSION()
engine_info[NAME_IDX] = (
self.name + "_engine" if self.name != "" else "tensorrt_engine"
)
engine_info[DEVICE_IDX] = target_device._to_serialized_rt_device()
-
assert self.serialized_engine
engine_info[ENGINE_IDX] = self.serialized_engine
diff --git a/py/torch_tensorrt/runtime/_utils.py b/py/torch_tensorrt/runtime/_utils.py
index ab427285e1..90da7f69ad 100644
--- a/py/torch_tensorrt/runtime/_utils.py
+++ b/py/torch_tensorrt/runtime/_utils.py
@@ -1,5 +1,5 @@
import logging
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
import torch
import torch_tensorrt
@@ -128,3 +128,24 @@ def _get_most_compatible_device(
best_match = candidate
return best_match
+
+
+@torch.library.custom_op(
+ "tensorrt::no_op_placeholder_for_execute_engine", mutates_args=()
+)
+def no_op_placeholder_for_execute_engine(
+ inputs: List[torch.Tensor],
+ abi_version: str,
+ name: str,
+ serialized_device_info: str,
+ serialized_engine: str,
+ serialized_in_binding_names: str,
+ serialized_out_binding_names: str,
+ serialized_hardware_compatible: str,
+ serialized_metadata: str,
+ serialized_target_platform: str,
+) -> List[torch.Tensor]:
+
+ raise RuntimeError(
+ "The saved model is cross compiled for windows in Linux, should only be loadded in Windows via torch_tensorrt.load_cross_compiled_exported_program() api."
+ )
diff --git a/py/torch_tensorrt/ts/_compile_spec.py b/py/torch_tensorrt/ts/_compile_spec.py
index 4843ec0145..5d6d27e4ad 100644
--- a/py/torch_tensorrt/ts/_compile_spec.py
+++ b/py/torch_tensorrt/ts/_compile_spec.py
@@ -307,7 +307,7 @@ def _parse_compile_spec(compile_spec_: Dict[str, Any]) -> _ts_C.CompileSpec:
def TensorRTCompileSpec(
inputs: Optional[List[torch.Tensor | Input]] = None,
input_signature: Optional[Any] = None,
- device: torch.device | Device = Device._current_device(),
+ device: Optional[torch.device | Device] = None,
disable_tf32: bool = False,
sparse_weights: bool = False,
enabled_precisions: Optional[Set[torch.dtype | dtype]] = None,
@@ -365,7 +365,7 @@ def TensorRTCompileSpec(
compile_spec = {
"inputs": inputs if inputs is not None else [],
# "input_signature": input_signature,
- "device": device,
+ "device": Device._current_device() if device is None else device,
"disable_tf32": disable_tf32, # Force FP32 layers to use traditional as FP32 format vs the default behavior of rounding the inputs to 10-bit mantissas before multiplying, but accumulates the sum using 23-bit mantissas
"sparse_weights": sparse_weights, # Enable sparsity for convolution and fully connected layers.
"enabled_precisions": (
diff --git a/tests/py/dynamo/runtime/test_003_cross_compile_for_windows.py b/tests/py/dynamo/runtime/test_003_cross_compile_for_windows.py
new file mode 100644
index 0000000000..acf2aa006f
--- /dev/null
+++ b/tests/py/dynamo/runtime/test_003_cross_compile_for_windows.py
@@ -0,0 +1,66 @@
+import os
+import platform
+import tempfile
+import unittest
+
+import pytest
+import torch
+import torch_tensorrt
+from torch.testing._internal.common_utils import TestCase
+
+from ..testing_utilities import DECIMALS_OF_AGREEMENT
+
+
+class TestCrossCompileSaveForWindows(TestCase):
+
+ @unittest.skipIf(
+ platform.system() != "Linux" or platform.architecture()[0] != "64bit",
+ "Cross compile for windows can only be enabled on linux x86-64 platform",
+ )
+ @pytest.mark.unit
+ def test_cross_compile_for_windows(self):
+ class Add(torch.nn.Module):
+ def forward(self, a, b):
+ return torch.add(a, b)
+
+ model = Add().eval().cuda()
+ inputs = [torch.randn(2, 3).cuda(), torch.randn(2, 3).cuda()]
+ trt_ep_path = os.path.join(tempfile.gettempdir(), "trt.ep")
+ compile_spec = {
+ "inputs": inputs,
+ "min_block_size": 1,
+ }
+ try:
+ torch_tensorrt.cross_compile_for_windows(
+ model, file_path=trt_ep_path, **compile_spec
+ )
+ except Exception as e:
+ pytest.fail(f"unexpected exception raised: {e}")
+
+ @unittest.skipIf(
+ platform.system() != "Linux" or platform.architecture()[0] != "64bit",
+ "Cross compile for windows can only be enabled on linux x86-64 platform",
+ )
+ @pytest.mark.unit
+ def test_dynamo_cross_compile_for_windows(self):
+ class Add(torch.nn.Module):
+ def forward(self, a, b):
+ return torch.add(a, b)
+
+ model = Add().eval().cuda()
+ inputs = (torch.randn(2, 3).cuda(), torch.randn(2, 3).cuda())
+ trt_ep_path = os.path.join(tempfile.gettempdir(), "trt.ep")
+ exp_program = torch.export.export(model, inputs)
+ compile_spec = {
+ "inputs": inputs,
+ "min_block_size": 1,
+ }
+ try:
+ trt_gm = torch_tensorrt.dynamo.cross_compile_for_windows(
+ exp_program, **compile_spec
+ )
+ torch_tensorrt.dynamo.save_cross_compiled_exported_program(
+ trt_gm, file_path=trt_ep_path
+ )
+ except Exception as e:
+ pytest.fail(f"unexpected exception raised: {e}")