allow to dump shader on disk to workaround perfetto issue

.github/workflows/presubmit.yml

@@ -91,23 +91,18 @@ jobs:
             -DBACKEND=System \
             -DCMAKE_BUILD_TYPE=Release
           cmake --build build_release
-      - name: Tests
+      - name: Test runner
+        env:
+          VKSP_RUNNER: ${{ github.workspace }}/build_release/runner/vulkan-shader-profiler-runner
         run: |
-          $(pwd)/build_release/runner/vulkan-shader-profiler-runner -i $(pwd)/tests/example.spvasm -v
-          $(pwd)/build_release/runner/vulkan-shader-profiler-runner -i $(pwd)/tests/example-counter.spvasm | grep "my_section"
-          $(pwd)/third_party/perfetto/out/linux_clang_release/traced &
-          $(pwd)/third_party/perfetto/out/linux_clang_release/perfetto -c $(pwd)/tests/perfetto_config --txt -o $(pwd)/trace &
-          sleep 1
-          VK_LOADER_DEBUG=all \
-          LD_LIBRARY_PATH=$(pwd)/build_release/layer/:$(pwd)/third_party/vulkan-loader/build/loader/:$LD_LIBRARY_PATH \
-          VK_LOADER_LAYERS_ENABLE="VK_LAYER_SHADER_PROFILER" \
-          VK_ADD_LAYER_PATH=$(pwd)/manifest/ \
-          VKSP_TRACE_DEST=$(pwd)/trace \
-          VKSP_SHADER_TEXT_BLOCK_SIZE=1024 \
-          $(pwd)/build_release/runner/vulkan-shader-profiler-runner -i $(pwd)/tests/example.spvasm
-          sleep 1
-          pkill -x perfetto
-          while [[ -n $(pgrep -x perfetto) ]]; do sleep 0.2; done
-          $(pwd)/build_release/extractor/vulkan-shader-profiler-extractor -i $(pwd)/trace -o $(pwd)/trace.spvasm -d 0 -v
-          cat $(pwd)/tests/example-expectation.spvasm
-          diff $(pwd)/trace.spvasm $(pwd)/tests/example-expectation.spvasm
+          $(pwd)/tests/test-runner.sh
+      - name: Test layer & extractor
+        env:
+          VK_LOADER_DEBUG: all
+          LD_LIBRARY_PATH: ${{ github.workspace }}/build_release/layer/:${{ github.workspace }}/third_party/vulkan-loader/build/loader/:$LD_LIBRARY_PATH
+          VKSP_EXTRACTOR: ${{ github.workspace }}/build_release/extractor/vulkan-shader-profiler-extractor
+          VKSP_RUNNER: ${{ github.workspace }}/build_release/runner/vulkan-shader-profiler-runner
+          PERFETTO_TRACED: ${{ github.workspace }}/third_party/perfetto/out/linux_clang_release/traced
+          PERFETTO_BINARY: ${{ github.workspace }}/third_party/perfetto/out/linux_clang_release/perfetto
+        run: |
+          $(pwd)/tests/test-extractor.sh

README.md

@@ -22,6 +22,8 @@ Using the `vulkan-shader-profiler-extractor` and `vulkan-shader-profiler-runner`
 - [Extracting a dispatch from a trace](#Extracting-a-dispatch-from-a-trace)
 - [Run a Vulkan SPIR-V program with the runner](#Run-a-Vulkan-SPIR-V-program-with-the-runner)
   - [Using counters inside a Vulkan SPIR-V program](#Using-counters-inside-a-Vulkan-SPIR-V-program)
+- [Known issues](#Known-issues)
+  - [Large shader code](#Large-shader-code)
 
 # Dependencies
 
@@ -136,6 +138,7 @@ Required options:
 Optional options:
 
 * `-b`: output a binary Vulkan SPIRV-V program instead of a readable one (allow to have something smaller).
+* `-s`: the path to the file to use instead of the perfetto trace to get the shader code (see section [Large shader code](#Large-shader-code) for more information).
 * `-v`: enable the verbose mode which is mainly use for debug purpose.
 
 # Run a Vulkan SPIR-V program with the runner
@@ -212,3 +215,24 @@ vksp_s0-test_simple-128.1.1
 -------------------------------
 [SHADER] my_section:  29.8%
 ```
+
+# Known issues
+
+## Large shader code
+
+When tracing applications using large shader code, perfetto can have issue creating the slice.
+It causes the shader code to be missing or to be partially present in the perfetto trace.
+Thus preventing to find the full code in the web ui or to use the `vulkan-shader-profiler-extractor`.
+
+To avoid this issue, it is possible to run the application with the following environment variable set:
+
+```
+VKSP_SHADER_DIR=<path-use-to-store-the-shaders>
+```
+
+It will force the `vulkan-shader-profiler` layer to dump the shaders in their binary format in this directory (make sure the directory exists, it will not be created by the `vulkan-shader-profiler` layer).
+
+Then once can either use:
+
+- `spirv-dis` to disassemble the interesting shaders to a readable format
+- `vulkan-shader-profiler-extractor` with the `-s` option to specify the shader file to use instead of what is inside the perfetto trace.

extractor/extractor.cpp

@@ -89,6 +89,7 @@ using namespace spvtools;
 static bool gVerbose = false, gBinary = false;
 static uint64_t gDispatchId = UINT64_MAX;
 static std::string gInput = "", gOutput = "";
+static std::string gShaderFile = "";
 
 void progress_callback(uint64_t size)
 {
@@ -163,8 +164,8 @@ bool get_min_timestamp(TraceProcessor *tp, uint64_t commandBUffer, uint64_t max_
     return true;
 }
 
-bool get_shader_and_device_from_compute(
-    TraceProcessor *tp, uint64_t compute, std::string &shader, uint64_t &device, vksp_configuration &config)
+bool get_shader_and_device_from_compute(TraceProcessor *tp, uint64_t compute, std::string &shader,
+    std::vector<char> &shader_buffer, uint64_t &device, vksp_configuration &config)
 {
     GET_STR_VALUE(tp, compute, "debug.shader", config.shaderName);
 
@@ -177,6 +178,16 @@ bool get_shader_and_device_from_compute(
 
     GET_INT_VALUE(tp, shader_device_arg_set_id, "debug.device", device);
 
+    if (gShaderFile != "") {
+        if (read_shader_buffer(&gShaderFile, &shader_buffer)) {
+            return true;
+        } else {
+            ERROR("'%s' does not exist, get shader code from perfetto file", gShaderFile.c_str());
+            // reset gShaderFile to make sure the extractor is using the string, not the buffer;
+            gShaderFile = "";
+        }
+    }
+
     std::string query2 = "SELECT arg_set_id FROM slice WHERE slice.name = 'vkCreateShaderModule-text' AND '"
         + std::string(config.shaderName)
         + "' = (SELECT string_value FROM args WHERE args.arg_set_id = slice.arg_set_id AND args.key = 'debug.shader') "
@@ -532,14 +543,15 @@ void help()
            "OPTIONS:\n"
            "\t-b\tOutput in binary instead of text\n"
            "\t-h\tDisplay this help and exit\n"
+           "\t-s\tFile to use instead of perfetto to get shader code\n"
            "\t-v\tVerbose mode\n");
 }
 
 bool parse_args(int argc, char **argv)
 {
     bool bHelp = false;
     int c;
-    while ((c = getopt(argc, argv, "hbvd:i:o:")) != -1) {
+    while ((c = getopt(argc, argv, "hbvd:i:o:s:")) != -1) {
         switch (c) {
         case 'd':
             gDispatchId = atoi(optarg);
@@ -553,6 +565,9 @@ bool parse_args(int argc, char **argv)
         case 'b':
             gBinary = true;
             break;
+        case 's':
+            gShaderFile = std::string(optarg);
+            break;
         case 'v':
             gVerbose = true;
             break;
@@ -596,10 +611,15 @@ int main(int argc, char **argv)
 
     std::string shader;
     uint64_t device;
-    CHECK(get_shader_and_device_from_compute(tp.get(), compute, shader, device, config),
+    std::vector<char> shader_buffer;
+    CHECK(get_shader_and_device_from_compute(tp.get(), compute, shader, shader_buffer, device, config),
         "Could not get shader from compute");
     PRINT("Device: %lu", device);
-    PRINT("Shader from compute (name: '%s'):\n%s", config.shaderName, shader.c_str());
+    if (gShaderFile == "") {
+        PRINT("Shader from compute (name: '%s'):\n%s", config.shaderName, shader.c_str());
+    } else {
+        PRINT("Shader from file '%s' (name: '%s')\n", gShaderFile.c_str(), config.shaderName);
+    }
 
     uint64_t max_timestamp;
     CHECK(get_max_timestamp(tp.get(), dispatch, max_timestamp), "Could not get max_timestamp");
@@ -673,9 +693,15 @@ int main(int argc, char **argv)
         }
     }
 
-    CHECK(store_shader_in_output(&shader, &push_constants_vector, &descriptor_sets_vector, &map_entry_vector, &config,
-              gOutput.c_str(), gBinary),
-        "Could not store shader in output file");
+    if (gShaderFile == "") {
+        CHECK(store_shader_in_output(&shader, &push_constants_vector, &descriptor_sets_vector, &map_entry_vector,
+                  &config, gOutput.c_str(), gBinary),
+            "Could not store shader in output file");
+    } else {
+        CHECK(store_shader_buffer_in_output(&shader_buffer, &push_constants_vector, &descriptor_sets_vector,
+                  &map_entry_vector, &config, gOutput.c_str(), gBinary),
+            "Could not store shader buffer in output file");
+    }
 
     return 0;
 }

extractor/spirv.cpp

@@ -18,29 +18,32 @@
 #include "spirv.hpp"
 #include "utils.hpp"
 
-bool create_binary(spv_context context, std::string *shader, std::vector<spvtools::vksp_push_constant> *pc,
-    std::vector<spvtools::vksp_descriptor_set> *ds, std::vector<spvtools::vksp_specialization_map_entry> *me,
-    spvtools::vksp_configuration *config, std::vector<uint32_t> &binary)
+#include <filesystem>
+
+bool text_to_binary(spv_context context, std::string *shader, spv_binary &binary)
 {
     spv_diagnostic diagnostic;
-    spv_binary tmp_binary;
-    auto status = spvTextToBinary(context, shader->data(), shader->size(), &tmp_binary, &diagnostic);
+    auto status = spvTextToBinary(context, shader->data(), shader->size(), &binary, &diagnostic);
     if (status != SPV_SUCCESS) {
         ERROR("Error while converting shader from text to binary: %s", diagnostic->error);
         spvDiagnosticDestroy(diagnostic);
         return false;
     }
+    return true;
+}
 
+bool create_binary(spv_context context, spv_binary input_binary, std::vector<spvtools::vksp_push_constant> *pc,
+    std::vector<spvtools::vksp_descriptor_set> *ds, std::vector<spvtools::vksp_specialization_map_entry> *me,
+    spvtools::vksp_configuration *config, std::vector<uint32_t> &output_binary)
+{
     spvtools::Optimizer opt(SPV_ENV_VULKAN_1_3);
     opt.RegisterPass(spvtools::CreateInsertVkspReflectInfoPass(pc, ds, me, config));
     spvtools::OptimizerOptions options;
     options.set_run_validator(false);
-    if (!opt.Run(tmp_binary->code, tmp_binary->wordCount, &binary, options)) {
+    if (!opt.Run(input_binary->code, input_binary->wordCount, &output_binary, options)) {
         ERROR("Error while running 'CreateVkspReflectInfoPass'");
         return false;
     }
-
-    spvBinaryDestroy(tmp_binary);
     return true;
 }
 
@@ -60,13 +63,13 @@ bool binary_to_text(spv_context context, std::vector<uint32_t> &binary, spv_text
     return true;
 }
 
-extern "C" bool store_shader_in_output(std::string *shader, std::vector<spvtools::vksp_push_constant> *pc,
-    std::vector<spvtools::vksp_descriptor_set> *ds, std::vector<spvtools::vksp_specialization_map_entry> *me,
-    spvtools::vksp_configuration *config, const char *output_filename, bool binary_output)
+bool store_shader_binary_in_output(spv_context context, spv_binary input_binary,
+    std::vector<spvtools::vksp_push_constant> *pc, std::vector<spvtools::vksp_descriptor_set> *ds,
+    std::vector<spvtools::vksp_specialization_map_entry> *me, spvtools::vksp_configuration *config,
+    const char *output_filename, bool binary_output)
 {
-    spv_context context = spvContextCreate(SPV_ENV_VULKAN_1_3);
     std::vector<uint32_t> binary;
-    if (!create_binary(context, shader, pc, ds, me, config, binary)) {
+    if (!create_binary(context, input_binary, pc, ds, me, config, binary)) {
         ERROR("Could not create SPIR-V binary from trace");
         return false;
     }
@@ -89,3 +92,50 @@ extern "C" bool store_shader_in_output(std::string *shader, std::vector<spvtools
     spvContextDestroy(context);
     return true;
 }
+
+extern "C" bool store_shader_in_output(std::string *shader, std::vector<spvtools::vksp_push_constant> *pc,
+    std::vector<spvtools::vksp_descriptor_set> *ds, std::vector<spvtools::vksp_specialization_map_entry> *me,
+    spvtools::vksp_configuration *config, const char *output_filename, bool binary_output)
+{
+    spv_context context = spvContextCreate(SPV_ENV_VULKAN_1_3);
+    spv_binary binary;
+    if (!text_to_binary(context, shader, binary)) {
+        ERROR("Could not convert shader to binary");
+        return false;
+    }
+
+    auto ret = store_shader_binary_in_output(context, binary, pc, ds, me, config, output_filename, binary_output);
+
+    spvBinaryDestroy(binary);
+    return ret;
+}
+
+extern "C" bool store_shader_buffer_in_output(std::vector<char> *shader_buffer,
+    std::vector<spvtools::vksp_push_constant> *pc, std::vector<spvtools::vksp_descriptor_set> *ds,
+    std::vector<spvtools::vksp_specialization_map_entry> *me, spvtools::vksp_configuration *config,
+    const char *output_filename, bool binary_output)
+{
+    spv_context context = spvContextCreate(SPV_ENV_VULKAN_1_3);
+    spv_binary_t binary
+        = { .code = (uint32_t *)shader_buffer->data(), .wordCount = shader_buffer->size() / sizeof(uint32_t) };
+
+    return store_shader_binary_in_output(context, &binary, pc, ds, me, config, output_filename, binary_output);
+}
+
+extern "C" bool read_shader_buffer(std::string *gShaderFile, std::vector<char> *shader_buffer)
+{
+    if (!std::filesystem::exists(*gShaderFile)) {
+        return false;
+    }
+    FILE *file = fopen(gShaderFile->c_str(), "r");
+    fseek(file, 0, SEEK_END);
+    size_t file_size = ftell(file);
+    fseek(file, 0, SEEK_SET);
+    shader_buffer->resize(file_size);
+    size_t size_read = 0;
+    do {
+        size_read += fread(&(shader_buffer->data()[size_read]), 1, file_size - size_read, file);
+    } while (size_read != file_size);
+    fclose(file);
+    return true;
+}

extractor/spirv.hpp

@@ -20,3 +20,8 @@
 extern "C" bool store_shader_in_output(std::string *shader, std::vector<spvtools::vksp_push_constant> *pc,
     std::vector<spvtools::vksp_descriptor_set> *ds, std::vector<spvtools::vksp_specialization_map_entry> *me,
     spvtools::vksp_configuration *config, const char *output_filename, bool binary_output);
+extern "C" bool store_shader_buffer_in_output(std::vector<char> *shader_buffer,
+    std::vector<spvtools::vksp_push_constant> *pc, std::vector<spvtools::vksp_descriptor_set> *ds,
+    std::vector<spvtools::vksp_specialization_map_entry> *me, spvtools::vksp_configuration *config,
+    const char *output_filename, bool binary_output);
+extern "C" bool read_shader_buffer(std::string *gShaderFile, std::vector<char> *shader_buffer);

layer/layer.cpp

@@ -18,6 +18,7 @@
 #include "spirv-tools/libspirv.h"
 
 #include <condition_variable>
+#include <filesystem>
 #include <perfetto.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -597,6 +598,26 @@ VkResult VKAPI_CALL vksp_CreateComputePipelines(VkDevice device, VkPipelineCache
     return result;
 }
 
+static void writeShaderOnDisk(const char *dir, std::string shader_name, const uint32_t *code, const size_t code_size)
+{
+    TRACE_EVENT(VKSP_PERFETTO_CATEGORY, "writeShaderOnDisk", "dir", perfetto::DynamicString(dir), "shader",
+        perfetto::DynamicString(shader_name));
+    std::filesystem::path filename(dir);
+    if (!std::filesystem::exists(filename)) {
+        PRINT("'%s' does not exist, could not write shader on disk", dir);
+        return;
+    }
+    filename /= shader_name;
+    filename += ".spv";
+    FILE *file = fopen(filename.c_str(), "w");
+    size_t size_written = 0;
+    const uint8_t *data = (const uint8_t *)code;
+    do {
+        size_written += fwrite(&data[size_written], 1, code_size - size_written, file);
+    } while (size_written != code_size);
+    fclose(file);
+}
+
 VkResult VKAPI_CALL vksp_CreateShaderModule(VkDevice device, const VkShaderModuleCreateInfo *pCreateInfo,
     const VkAllocationCallbacks *pAllocator, VkShaderModule *pShaderModule)
 {
@@ -610,8 +631,14 @@ VkResult VKAPI_CALL vksp_CreateShaderModule(VkDevice device, const VkShaderModul
     spv_text text;
     spv_diagnostic diag;
     const uint32_t *code = pCreateInfo->pCode;
-    const size_t code_size = pCreateInfo->codeSize / sizeof(uint32_t);
-    spv_result_t spv_result = spvBinaryToText(context, code, code_size,
+    const size_t code_size = pCreateInfo->codeSize;
+    const size_t word_count = code_size / sizeof(uint32_t);
+
+    if (auto dir = getenv("VKSP_SHADER_DIR")) {
+        writeShaderOnDisk(dir, shader_str, code, code_size);
+    }
+
+    spv_result_t spv_result = spvBinaryToText(context, code, word_count,
         SPV_BINARY_TO_TEXT_OPTION_INDENT | SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | SPV_BINARY_TO_TEXT_OPTION_COMMENT,
         &text, &diag);
     if (spv_result == SPV_SUCCESS) {