From 7b6eab5dcb104dd16a8cc8c4e27ab6e65635a4ab Mon Sep 17 00:00:00 2001 From: aqnuep <33791085+aqnuep@users.noreply.github.com> Date: Fri, 27 Oct 2023 02:18:18 +0200 Subject: [PATCH] Add UTF-8 filename support on Windows to ktxtools (#788) Fixes https://github.com/KhronosGroup/KTX-Software/issues/781. --- CMakeLists.txt | 4 +++ tests/cts | 2 +- tests/ktxdiff/CMakeLists.txt | 2 ++ tests/ktxdiff/ktxdiff_main.cpp | 7 ++-- tools/imageio/imageinput.cc | 4 ++- tools/ktx/command.cpp | 22 ++++++------ tools/ktx/command_create.cpp | 6 ++-- tools/ktx/command_encode.cpp | 6 ++-- tools/ktx/command_extract.cpp | 10 +++--- tools/ktx/command_help.cpp | 7 ++++ tools/ktx/command_transcode.cpp | 6 ++-- tools/ktx/ktx_main.cpp | 3 ++ utils/platform_utils.h | 63 +++++++++++++++++++++++++++++++++ 13 files changed, 118 insertions(+), 24 deletions(-) create mode 100644 utils/platform_utils.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f926560683..b52de3d88e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,6 +241,9 @@ if(MSVC) # because `/W4;/WX` is returned as a single string. add_compile_options( /W4;$<$:/WX> ) add_compile_options( $,/Gz,/O2> ) + # Enable UTF-8 support + add_compile_options( $<$:/utf-8> ) + add_compile_options( $<$:/utf-8> ) elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") add_compile_options( -Wall -Wextra $<$:-Werror>) @@ -915,6 +918,7 @@ add_library( objUtil STATIC utils/sbufstream.h utils/scapp.h utils/stdafx.h + utils/platform_utils.h utils/unused.h ) target_include_directories( diff --git a/tests/cts b/tests/cts index 27c53b0844..b41b3f9d3d 160000 --- a/tests/cts +++ b/tests/cts @@ -1 +1 @@ -Subproject commit 27c53b08448050c4b6f204224d47d4be02700f44 +Subproject commit b41b3f9d3d35a3450c7dc19527e346ef13b707cc diff --git a/tests/ktxdiff/CMakeLists.txt b/tests/ktxdiff/CMakeLists.txt index f866c49db1..7aebeff4b1 100644 --- a/tests/ktxdiff/CMakeLists.txt +++ b/tests/ktxdiff/CMakeLists.txt @@ -25,6 +25,7 @@ target_include_directories( ktxdiff SYSTEM PRIVATE + $ ${PROJECT_SOURCE_DIR}/lib ${PROJECT_SOURCE_DIR}/other_include ) @@ -35,6 +36,7 @@ PRIVATE ktx ${ASTCENC_LIB_TARGET} fmt::fmt + objUtil ) target_compile_definitions( diff --git a/tests/ktxdiff/ktxdiff_main.cpp b/tests/ktxdiff/ktxdiff_main.cpp index 192d70b411..da78260287 100644 --- a/tests/ktxdiff/ktxdiff_main.cpp +++ b/tests/ktxdiff/ktxdiff_main.cpp @@ -6,6 +6,7 @@ #include "ktxint.h" #include "texture2.h" #include "vkformat_enum.h" +#include "platform_utils.h" #include "astc-encoder/Source/astcenc.h" @@ -173,7 +174,7 @@ struct Texture { }; void Texture::loadFile() { - auto file = std::ifstream(filepath, std::ios::binary | std::ios::in | std::ios::ate); + auto file = std::ifstream(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in | std::ios::ate); if (!file) error(EXIT_CODE_ERROR, "ktxdiff error \"{}\": Failed to open file: {}\n", filepath, errnoMessage()); @@ -441,7 +442,9 @@ bool compare(Texture& lhs, Texture& rhs, float tolerance) { /// 0 - Matching files /// 1 - Mismatching files /// 2 - Error while loading, decoding or processing an input file -int main(int argc, const char* argv[]) { +int main(int argc, char* argv[]) { + InitUTF8CLI(argc, argv); + if (argc < 3) { fmt::print("Missing input file arguments\n"); fmt::print("Usage: ktxdiff [tolerance]\n"); diff --git a/tools/imageio/imageinput.cc b/tools/imageio/imageinput.cc index f0c1820c25..da207e49f0 100644 --- a/tools/imageio/imageinput.cc +++ b/tools/imageio/imageinput.cc @@ -14,11 +14,13 @@ #include "imageio.h" #include "imageio_utility.h" +#include "platform_utils.h" #include #include #include #include +#include // Search for and instantiate a plugin that can read the format @@ -47,7 +49,7 @@ ImageInput::open(const _tstring& filename, // Check file exists, before looking for a suitable plugin. // MS's STL has `open` overloads that accept wchar_t to handle // Window's Unicode file names. - ifs.open(filename, std::ios::binary | std::ios::in); + ifs.open(std::filesystem::path(DecodeUTF8Path(filename)), std::ios::binary | std::ios::in); if (ifs.fail()) { throw std::runtime_error( fmt::format("Open of \"{}\" failed. {}", diff --git a/tools/ktx/command.cpp b/tools/ktx/command.cpp index 0c3324176a..01a118bdce 100644 --- a/tools/ktx/command.cpp +++ b/tools/ktx/command.cpp @@ -4,19 +4,17 @@ #include "command.h" +#include "platform_utils.h" #include "version.h" #include "ktx.h" #include "sbufstream.h" +#include #include #include #include #include -#if defined(_WIN32) && defined(DEBUG) -#include // For functions used by launchDebugger -#endif - // ------------------------------------------------------------------------------------------------- #define QUOTE(x) #x @@ -111,7 +109,7 @@ InputStream::InputStream(const std::string& filepath, Reporter& report) : stdinBuffer << std::cin.rdbuf(); activeStream = &stdinBuffer; } else { - file.open(filepath, std::ios::binary | std::ios::in); + file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in); if (!file) report.fatal(rc::IO_FAILURE, "Could not open input file \"{}\": {}.", filepath, errnoMessage()); activeStream = &file; @@ -134,7 +132,11 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : #endif file = stdout; } else { - file = std::fopen(filepath.c_str(), "wb"); +#if defined(_WIN32) + file = _wfopen(DecodeUTF8Path(filepath).c_str(), L"wb"); +#else + file = fopen(DecodeUTF8Path(filepath).c_str(), "wb"); +#endif if (!file) report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}.", filepath, errnoMessage()); } @@ -152,7 +154,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : // #endif // activeStream = &std::cout; // } else { - // file.open(filepath, std::ios::binary | std::ios::out); + // file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::out); // if (!file) // report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}", filepath, errnoMessage()); // activeStream = &file; @@ -161,7 +163,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : OutputStream::~OutputStream() { if (file != stdout) - std::fclose(file); + fclose(file); } void OutputStream::write(const char* data, std::size_t size, Reporter& report) { @@ -174,7 +176,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) { const auto ret = ktxTexture_WriteToStdioStream(texture, file); if (KTX_SUCCESS != ret) { if (file != stdout) - std::filesystem::remove(filepath); + std::filesystem::remove(DecodeUTF8Path(filepath).c_str()); report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": KTX error: {}.", filepath, ktxErrorString(ret)); } @@ -185,7 +187,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) { // // if (KTX_SUCCESS != ret) { // if (activeStream != &std::cout) - // std::filesystem::remove(filepath); + // std::filesystem::remove(DecodeUTF8Path(filepath).c_str()); // report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": {}.", fmtOutFile(filepath), ktxErrorString(ret)); // } } diff --git a/tools/ktx/command_create.cpp b/tools/ktx/command_create.cpp index f18be462eb..0750685f2d 100644 --- a/tools/ktx/command_create.cpp +++ b/tools/ktx/command_create.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "metrics_utils.h" #include "compress_utils.h" #include "encode_utils.h" @@ -1251,8 +1252,9 @@ void CommandCreate::executeCreate() { } // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/command_encode.cpp b/tools/ktx/command_encode.cpp index da9fc7f4e7..dd3b5162c8 100644 --- a/tools/ktx/command_encode.cpp +++ b/tools/ktx/command_encode.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "metrics_utils.h" #include "compress_utils.h" #include "encode_utils.h" @@ -219,8 +220,9 @@ void CommandEncode::executeEncode() { } // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/command_extract.cpp b/tools/ktx/command_extract.cpp index 96a98c597b..d736bcf181 100644 --- a/tools/ktx/command_extract.cpp +++ b/tools/ktx/command_extract.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "format_descriptor.h" #include "formats.h" #include "fragment_uri.h" @@ -386,13 +387,14 @@ void CommandExtract::executeExtract() { (!options.fragmentURI.facial.is_undefined() && options.fragmentURI.facial.is_multi()) || ((options.globalAll || options.depthFlagUsed) && options.depth.is_multi()); try { + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputPath)); if (isMultiOutput) { - if (std::filesystem::exists(options.outputPath) && !std::filesystem::is_directory(options.outputPath)) + if (std::filesystem::exists(outputPath) && !std::filesystem::is_directory(outputPath)) fatal_usage("Specified output path must be a directory for multi-output extract: \"{}\".", options.outputPath); - std::filesystem::create_directories(options.outputPath); + std::filesystem::create_directories(outputPath); } else { - if (std::filesystem::path(options.outputPath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputPath).parent_path()); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); } } catch (const std::filesystem::filesystem_error& e) { fatal(rc::IO_FAILURE, "Failed to create the output directory \"{}\": {}.", e.path1().generic_string(), e.what()); diff --git a/tools/ktx/command_help.cpp b/tools/ktx/command_help.cpp index b19b3f82ac..7b0662610a 100644 --- a/tools/ktx/command_help.cpp +++ b/tools/ktx/command_help.cpp @@ -10,7 +10,14 @@ #include #if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include // For GetModuleFileNameW +#include // For ShellExecuteW #include // For PathCchRemoveFileSpec #include // For wchat_t format #endif diff --git a/tools/ktx/command_transcode.cpp b/tools/ktx/command_transcode.cpp index f8a4330314..05b9e31bab 100644 --- a/tools/ktx/command_transcode.cpp +++ b/tools/ktx/command_transcode.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "compress_utils.h" #include "transcode_utils.h" #include "formats.h" @@ -171,8 +172,9 @@ void CommandTranscode::executeTranscode() { writer.c_str()); // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/ktx_main.cpp b/tools/ktx/ktx_main.cpp index 95e624e029..8faf46c873 100644 --- a/tools/ktx/ktx_main.cpp +++ b/tools/ktx/ktx_main.cpp @@ -4,6 +4,7 @@ #include "command.h" +#include "platform_utils.h" #include "stdafx.h" #include #include @@ -185,6 +186,8 @@ int _tmain(int argc, _TCHAR* argv[]) { // // pbxproj file, so it can't be disabled in a generated project. // // Remove these from the arguments under consideration. + InitUTF8CLI(argc, argv); + if (argc >= 2) { // Has a subcommand, attempt to lookup diff --git a/utils/platform_utils.h b/utils/platform_utils.h new file mode 100644 index 0000000000..7152af674d --- /dev/null +++ b/utils/platform_utils.h @@ -0,0 +1,63 @@ +// Copyright 2022-2023 The Khronos Group Inc. +// Copyright 2022-2023 RasterGrid Kft. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "stdafx.h" +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#endif + +#if defined(_WIN32) && !defined(_UNICODE) +// For Windows, we convert the UTF-8 path to a UTF-16 path to force using the APIs +// that correctly handle unicode characters +inline std::wstring DecodeUTF8Path(std::string path) { + std::wstring result; + int len = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), NULL, 0); + if (len > 0) + { + result.resize(len); + MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), &result[0], len); + } + return result; +} +#else +// For other platforms there is no need for any conversion, they support UTF-8 natively +inline std::string DecodeUTF8Path(std::string path) { + return path; +} +#endif + +inline void InitUTF8CLI(int& argc, _TCHAR* argv[]) { +#if defined(_WIN32) && !defined(_UNICODE) + // Windows does not support UTF-8 argv so we have to manually acquire it + static std::vector> utf8Argv(argc); + LPWSTR commandLine = GetCommandLineW(); + LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc); + for (int i = 0; i < argc; ++i) { + int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr); + utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize); + WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr); + argv[i] = utf8Argv[i].get(); + } +#else + // Nothing to do for other platforms + (void)argc; + (void)argv; +#endif +}