Add UTF-8 filename support on Windows to ktxtools (#788)

Fixes #781.

CMakeLists.txt

@@ -241,6 +241,9 @@ if(MSVC)
     # because `/W4;/WX` is returned as a single string.
     add_compile_options( /W4;$<$<BOOL:${KTX_WERROR}>:/WX> )
     add_compile_options( $<IF:$<CONFIG:Debug>,/Gz,/O2> )
+    # Enable UTF-8 support
+    add_compile_options( $<$<C_COMPILER_ID:MSVC>:/utf-8> )
+    add_compile_options( $<$<CXX_COMPILER_ID:MSVC>:/utf-8> )
 elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU"
        OR ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
     add_compile_options( -Wall -Wextra $<$<BOOL:${KTX_WERROR}>:-Werror>)
@@ -915,6 +918,7 @@ add_library( objUtil STATIC
     utils/sbufstream.h
     utils/scapp.h
     utils/stdafx.h
+    utils/platform_utils.h
     utils/unused.h
     )
 target_include_directories(

tests/ktxdiff/CMakeLists.txt

@@ -25,6 +25,7 @@ target_include_directories(
     ktxdiff
     SYSTEM
 PRIVATE
+    $<TARGET_PROPERTY:objUtil,INTERFACE_INCLUDE_DIRECTORIES>
     ${PROJECT_SOURCE_DIR}/lib
     ${PROJECT_SOURCE_DIR}/other_include
 )
@@ -35,6 +36,7 @@ PRIVATE
     ktx
     ${ASTCENC_LIB_TARGET}
     fmt::fmt
+    objUtil
 )
 
 target_compile_definitions(

tests/ktxdiff/ktxdiff_main.cpp

@@ -6,6 +6,7 @@
 #include "ktxint.h"
 #include "texture2.h"
 #include "vkformat_enum.h"
+#include "platform_utils.h"
 
 #include "astc-encoder/Source/astcenc.h"
 
@@ -173,7 +174,7 @@ struct Texture {
 };
 
 void Texture::loadFile() {
-    auto file = std::ifstream(filepath, std::ios::binary | std::ios::in | std::ios::ate);
+    auto file = std::ifstream(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in | std::ios::ate);
     if (!file)
         error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": Failed to open file: {}\n", filepath, errnoMessage());
 
@@ -441,7 +442,9 @@ bool compare(Texture& lhs, Texture& rhs, float tolerance) {
 ///     0 - Matching files
 ///     1 - Mismatching files
 ///     2 - Error while loading, decoding or processing an input file
-int main(int argc, const char* argv[]) {
+int main(int argc, char* argv[]) {
+    InitUTF8CLI(argc, argv);
+
     if (argc < 3) {
         fmt::print("Missing input file arguments\n");
         fmt::print("Usage: ktxdiff <expected-ktx2> <received-ktx2> [tolerance]\n");

tools/imageio/imageinput.cc

@@ -14,11 +14,13 @@
 
 #include "imageio.h"
 #include "imageio_utility.h"
+#include "platform_utils.h"
 
 #include <iomanip>
 #include <map>
 #include <stdarg.h>
 #include <stdexcept>
+#include <filesystem>
 
 
 // Search for and instantiate a plugin that can read the format
@@ -47,7 +49,7 @@ ImageInput::open(const _tstring& filename,
         // Check file exists, before looking for a suitable plugin.
         // MS's STL has `open` overloads that accept wchar_t to handle
         // Window's Unicode file names.
-        ifs.open(filename, std::ios::binary | std::ios::in);
+        ifs.open(std::filesystem::path(DecodeUTF8Path(filename)), std::ios::binary | std::ios::in);
         if (ifs.fail()) {
             throw std::runtime_error(
                 fmt::format("Open of \"{}\" failed. {}",

tools/ktx/command.cpp

@@ -4,19 +4,17 @@
 
 
 #include "command.h"
+#include "platform_utils.h"
 #include "version.h"
 #include "ktx.h"
 #include "sbufstream.h"
+#include <stdio.h>
 #include <filesystem>
 #include <iostream>
 
 #include <fmt/ostream.h>
 #include <fmt/printf.h>
 
-#if defined(_WIN32) && defined(DEBUG)
-#include <windows.h> // For functions used by launchDebugger 
-#endif
-
 // -------------------------------------------------------------------------------------------------
 
 #define QUOTE(x) #x
@@ -111,7 +109,7 @@ InputStream::InputStream(const std::string& filepath, Reporter& report) :
         stdinBuffer << std::cin.rdbuf();
         activeStream = &stdinBuffer;
     } else {
-        file.open(filepath, std::ios::binary | std::ios::in);
+        file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in);
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open input file \"{}\": {}.", filepath, errnoMessage());
         activeStream = &file;
@@ -134,7 +132,11 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
         #endif
         file = stdout;
     } else {
-        file = std::fopen(filepath.c_str(), "wb");
+#if defined(_WIN32)
+        file = _wfopen(DecodeUTF8Path(filepath).c_str(), L"wb");
+#else
+        file = fopen(DecodeUTF8Path(filepath).c_str(), "wb");
+#endif
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}.", filepath, errnoMessage());
     }
@@ -152,7 +154,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
     //     #endif
     //     activeStream = &std::cout;
     // } else {
-    //     file.open(filepath, std::ios::binary | std::ios::out);
+    //     file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::out);
     //     if (!file)
     //         report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}", filepath, errnoMessage());
     //     activeStream = &file;
@@ -161,7 +163,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
 
 OutputStream::~OutputStream() {
     if (file != stdout)
-        std::fclose(file);
+        fclose(file);
 }
 
 void OutputStream::write(const char* data, std::size_t size, Reporter& report) {
@@ -174,7 +176,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) {
     const auto ret = ktxTexture_WriteToStdioStream(texture, file);
     if (KTX_SUCCESS != ret) {
         if (file != stdout)
-            std::filesystem::remove(filepath);
+            std::filesystem::remove(DecodeUTF8Path(filepath).c_str());
         report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": KTX error: {}.", filepath, ktxErrorString(ret));
     }
 
@@ -185,7 +187,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) {
     //
     // if (KTX_SUCCESS != ret) {
     //     if (activeStream != &std::cout)
-    //         std::filesystem::remove(filepath);
+    //         std::filesystem::remove(DecodeUTF8Path(filepath).c_str());
     //     report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": {}.", fmtOutFile(filepath), ktxErrorString(ret));
     // }
 }

tools/ktx/command_create.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "metrics_utils.h"
 #include "compress_utils.h"
 #include "encode_utils.h"
@@ -1251,8 +1252,9 @@ void CommandCreate::executeCreate() {
     }
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/command_encode.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "metrics_utils.h"
 #include "compress_utils.h"
 #include "encode_utils.h"
@@ -219,8 +220,9 @@ void CommandEncode::executeEncode() {
     }
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/command_extract.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "format_descriptor.h"
 #include "formats.h"
 #include "fragment_uri.h"
@@ -386,13 +387,14 @@ void CommandExtract::executeExtract() {
             (!options.fragmentURI.facial.is_undefined() && options.fragmentURI.facial.is_multi()) ||
             ((options.globalAll || options.depthFlagUsed) && options.depth.is_multi());
     try {
+        const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputPath));
         if (isMultiOutput) {
-            if (std::filesystem::exists(options.outputPath) && !std::filesystem::is_directory(options.outputPath))
+            if (std::filesystem::exists(outputPath) && !std::filesystem::is_directory(outputPath))
                 fatal_usage("Specified output path must be a directory for multi-output extract: \"{}\".", options.outputPath);
-            std::filesystem::create_directories(options.outputPath);
+            std::filesystem::create_directories(outputPath);
         } else {
-            if (std::filesystem::path(options.outputPath).has_parent_path())
-                std::filesystem::create_directories(std::filesystem::path(options.outputPath).parent_path());
+            if (outputPath.has_parent_path())
+                std::filesystem::create_directories(outputPath.parent_path());
         }
     } catch (const std::filesystem::filesystem_error& e) {
         fatal(rc::IO_FAILURE, "Failed to create the output directory \"{}\": {}.", e.path1().generic_string(), e.what());

tools/ktx/command_help.cpp

@@ -10,7 +10,14 @@
 #include <fmt/format.h>
 
 #if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
 #include <windows.h> // For GetModuleFileNameW
+#include <shellapi.h> // For ShellExecuteW
 #include <pathcch.h> // For PathCchRemoveFileSpec
 #include <fmt/xchar.h> // For wchat_t format
 #endif

tools/ktx/command_transcode.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "compress_utils.h"
 #include "transcode_utils.h"
 #include "formats.h"
@@ -171,8 +172,9 @@ void CommandTranscode::executeTranscode() {
             writer.c_str());
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/ktx_main.cpp

@@ -4,6 +4,7 @@
 
 
 #include "command.h"
+#include "platform_utils.h"
 #include "stdafx.h"
 #include <iostream>
 #include <string>
@@ -185,6 +186,8 @@ int _tmain(int argc, _TCHAR* argv[]) {
     //      // pbxproj file, so it can't be disabled in a generated project.
     //      // Remove these from the arguments under consideration.
 
+    InitUTF8CLI(argc, argv);
+
     if (argc >= 2) {
         // Has a subcommand, attempt to lookup
 

utils/platform_utils.h

@@ -0,0 +1,63 @@
+// Copyright 2022-2023 The Khronos Group Inc.
+// Copyright 2022-2023 RasterGrid Kft.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "stdafx.h"
+#include <string>
+#include <iostream>
+#include <memory>
+#include <vector>
+#include <fmt/ostream.h>
+#include <fmt/printf.h>
+
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
+#if defined(_WIN32) && !defined(_UNICODE)
+// For Windows, we convert the UTF-8 path to a UTF-16 path to force using the APIs
+// that correctly handle unicode characters
+inline std::wstring DecodeUTF8Path(std::string path) {
+    std::wstring result;
+    int len = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast<int>(path.length()), NULL, 0);
+    if (len > 0)
+    {
+        result.resize(len);
+        MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast<int>(path.length()), &result[0], len);
+    }
+    return result;
+}
+#else
+// For other platforms there is no need for any conversion, they support UTF-8 natively
+inline std::string DecodeUTF8Path(std::string path) {
+    return path;
+}
+#endif
+
+inline void InitUTF8CLI(int& argc, _TCHAR* argv[]) {
+#if defined(_WIN32) && !defined(_UNICODE)
+    // Windows does not support UTF-8 argv so we have to manually acquire it
+    static std::vector<std::unique_ptr<_TCHAR[]>> utf8Argv(argc);
+    LPWSTR commandLine = GetCommandLineW();
+    LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc);
+    for (int i = 0; i < argc; ++i) {
+        int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr);
+        utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize);
+        WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr);
+        argv[i] = utf8Argv[i].get();
+    }
+#else
+    // Nothing to do for other platforms
+    (void)argc;
+    (void)argv;
+#endif
+}