Skip to content

Commit

Permalink
improve host toolchain #5639
Browse files Browse the repository at this point in the history
  • Loading branch information
@waruqi
waruqi committed Sep 20, 2024
1 parent 9563841 commit cacefb8
Show file tree
Hide file tree
Showing 10 changed files with 168 additions and 114 deletions.
6 changes: 6 additions & 0 deletions xmake/actions/config/xmake.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@ task("config")
, " - xmake f --toolchain=[cross|llvm|sdcc ..] --sdk=/xxx"
, " - run `xmake show -l toolchains` to get all toolchains"
, values = _toolchain_values},
{nil, "toolchain_host", "kv", nil, "Set host toolchain name, it's only for building packages on host machine."
, "e.g. "
, " - xmake f --toolchain_host=clang"
, " - xmake f --toolchain_host=[cross|llvm|sdcc ..] --sdk=/xxx"
, " - run `xmake show -l toolchains` to get all toolchains"
, values = _toolchain_values},
{nil, "runtimes", "kv", nil, "Set the compiler runtime library."
, "e.g. "
, " - xmake f --runtimes=MTd"
Expand Down
45 changes: 29 additions & 16 deletions xmake/core/package/package.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,7 @@ function _instance:plat()
if self._PLAT then
return self._PLAT
end
-- @note we uses os.host() instead of them for the binary package
if self:is_binary() then
if self:is_host() then
return os.subhost()
end
local requireinfo = self:requireinfo()
Expand All @@ -182,8 +181,7 @@ function _instance:arch()
if self._ARCH then
return self._ARCH
end
-- @note we uses os.arch() instead of them for the binary package
if self:is_binary() then
if self:is_host() then
return os.subarch()
end
return self:targetarch()
Expand Down Expand Up @@ -668,6 +666,31 @@ function _instance:is_local()
return self._IS_LOCAL or self:is_source_embed() or self:is_binary_embed() or self:is_thirdparty()
end

-- is debug package? (deprecated)
function _instance:debug()
return self:is_debug()
end

-- is host package?
--
-- @note It is different from not is_cross() in that users do not use host packages directly,
-- they are usually used to build library packages.
function _instance:is_host()
local requireinfo = self:requireinfo()
if requireinfo and requireinfo.host then
return true
end
return self:is_binary()
end

-- is cross-compilation?
function _instance:is_cross()
if self:is_host() then
return false
end
return is_cross(self:plat(), self:arch())
end

-- mark it as local package
function _instance:_mark_as_local(is_local)
if self:is_local() ~= is_local then
Expand Down Expand Up @@ -699,16 +722,6 @@ function _instance:use_external_includes()
return external
end

-- is debug package? (deprecated)
function _instance:debug()
return self:is_debug()
end

-- is cross-compilation?
function _instance:is_cross()
return is_cross(self:plat(), self:arch())
end

-- get the filelock of the whole package directory
function _instance:filelock()
local filelock = self._FILELOCK
Expand Down Expand Up @@ -1267,7 +1280,7 @@ function _instance:tool(toolkind)
local cachekey = "package_" .. tostring(self)
return toolchain.tool(self:toolchains(), toolkind, {cachekey = cachekey, plat = self:plat(), arch = self:arch()})
else
return platform.tool(toolkind, self:plat(), self:arch())
return platform.tool(toolkind, self:plat(), self:arch(), {host = self:is_host()})
end
end

Expand All @@ -1277,7 +1290,7 @@ function _instance:toolconfig(name)
local cachekey = "package_" .. tostring(self)
return toolchain.toolconfig(self:toolchains(), name, {cachekey = cachekey, plat = self:plat(), arch = self:arch()})
else
return platform.toolconfig(name, self:plat(), self:arch())
return platform.toolconfig(name, self:plat(), self:arch(), {host = self:is_host()})
end
end

Expand Down
48 changes: 30 additions & 18 deletions xmake/core/platform/platform.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,22 @@ local os = require("base/os")
local path = require("base/path")
local utils = require("base/utils")
local table = require("base/table")
local global = require("base/global")
local interpreter = require("base/interpreter")
local toolchain = require("tool/toolchain")
local memcache = require("cache/memcache")
local sandbox = require("sandbox/sandbox")
local config = require("project/config")
local global = require("base/global")
local scheduler = require("sandbox/modules/import/core/base/scheduler")

-- new an instance
function _instance.new(name, arch, info)
function _instance.new(name, info, opt)
opt = opt or{}
local instance = table.inherit(_instance)
instance._NAME = name
instance._ARCH = arch
instance._INFO = info
instance._ARCH = opt.arch
instance._IS_HOST = opt.host or false
return instance
end

Expand All @@ -54,6 +56,11 @@ function _instance:_memcache()
return cache
end

-- the toolchains cache key for names
function _instance:_toolchains_key()
return "__toolchains_" .. self:name() .. "_" .. self:arch() .. (self:is_host() and "_host" or "")
end

-- get platform name
function _instance:name()
return self._NAME
Expand All @@ -74,6 +81,11 @@ function _instance:arch_set(arch)
end
end

-- is host platform, it will use the host toolchain
function _instance:is_host()
return self._IS_HOST
end

-- set the value to the platform configuration
function _instance:set(name, ...)
self._INFO:apival_set(name, ...)
Expand Down Expand Up @@ -157,11 +169,11 @@ function _instance:toolchains(opt)
local names = nil
toolchains = {}
if not (opt and opt.all) then
names = config.get("__toolchains_" .. self:name() .. "_" .. self:arch())
names = config.get(self:_toolchains_key())
end
if not names then
-- get the given toolchain
local toolchain_given = config.get("toolchain")
local toolchain_given = config.get(self:is_host() and "toolchain_host" or "toolchain")
if toolchain_given then
local toolchain_inst, errors = toolchain.load(toolchain_given, {
plat = self:name(), arch = self:arch()})
Expand Down Expand Up @@ -276,7 +288,7 @@ function _instance:check()
end

-- save valid toolchains
config.set("__toolchains_" .. self:name() .. "_" .. self:arch(), toolchains_valid)
config.set(self:_toolchains_key(), toolchains_valid)
self:_memcache():set("checked", true)
scheduler.co_unlock(lockname)
return true
Expand Down Expand Up @@ -407,14 +419,16 @@ function platform.add_directories(...)
end

-- load the given platform
function platform.load(plat, arch)

-- get platform name
function platform.load(plat, arch, opt)
opt = opt or {}
plat = plat or config.get("plat") or os.host()
arch = arch or config.get("arch") or os.arch()

-- get cache key
local cachekey = plat .. "_" .. arch
if opt.host then
cachekey = cachekey .. "_host"
end

-- get it directly from cache dirst
platform._PLATFORMS = platform._PLATFORMS or {}
Expand All @@ -425,8 +439,6 @@ function platform.load(plat, arch)
-- find the platform script path
local scriptpath = nil
for _, dir in ipairs(platform.directories()) do

-- find this directory
scriptpath = path.join(dir, plat, "xmake.lua")
if os.isfile(scriptpath) then
break
Expand Down Expand Up @@ -469,14 +481,14 @@ function platform.load(plat, arch)
end

-- save instance to the cache
local instance = _instance.new(plat, arch, result)
local instance = _instance.new(plat, result, {arch = arch, host = opt.host})
platform._PLATFORMS[cachekey] = instance
return instance
end

-- get the given platform configuration
function platform.get(name, plat, arch)
local instance, errors = platform.load(plat, arch)
function platform.get(name, plat, arch, opt)
local instance, errors = platform.load(plat, arch, opt)
if instance then
return instance:get(name)
else
Expand All @@ -488,8 +500,8 @@ end
--
-- e.g. cc, cxx, mm, mxx, as, ar, ld, sh, ..
--
function platform.tool(toolkind, plat, arch)
local instance, errors = platform.load(plat, arch)
function platform.tool(toolkind, plat, arch, opt)
local instance, errors = platform.load(plat, arch, opt)
if instance then
return instance:tool(toolkind)
else
Expand All @@ -498,8 +510,8 @@ function platform.tool(toolkind, plat, arch)
end

-- get the given tool configuration
function platform.toolconfig(name, plat, arch)
local instance, errors = platform.load(plat, arch)
function platform.toolconfig(name, plat, arch, opt)
local instance, errors = platform.load(plat, arch, opt)
if instance then
return instance:toolconfig(name)
else
Expand Down
26 changes: 8 additions & 18 deletions xmake/modules/private/action/require/impl/package.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,17 +160,6 @@ function _load_require(require_str, requires_extra, parentinfo)
wprint("add_requires(%s): vs_runtime is deprecated, please use runtimes!", require_str)
end

-- require packge in the current host platform
if require_extra.host then
if is_subhost(core_package.targetplat()) and os.subarch() == core_package.targetarch() then
-- we need to pass plat/arch to avoid repeat installation
-- @see https://github.com/xmake-io/xmake/issues/1579
else
require_extra.plat = os.subhost()
require_extra.arch = os.subarch()
end
end

-- check require options
local extra_options = hashset.of("plat", "arch", "kind", "host", "targetos",
"alias", "group", "system", "option", "default", "optional", "debug",
Expand All @@ -197,6 +186,7 @@ function _load_require(require_str, requires_extra, parentinfo)
originstr = require_str,
reponame = reponame,
version = require_extra.version or version,
host = require_extra.host, -- this package is only for host machine
plat = require_extra.plat, -- require package in the given platform
arch = require_extra.arch, -- require package in the given architecture
targetos = require_extra.targetos, -- require package in the given target os
Expand Down Expand Up @@ -579,13 +569,6 @@ end

-- finish requireinfo
function _finish_requireinfo(requireinfo, package)
-- we need to synchronise the plat/arch inherited from the parent package as early as possible
if requireinfo.plat then
package:plat_set(requireinfo.plat)
end
if requireinfo.arch then
package:arch_set(requireinfo.arch)
end
requireinfo.configs = requireinfo.configs or {}
if package:is_plat("windows") then
-- @see https://github.com/xmake-io/xmake/issues/4477#issuecomment-1913249489
Expand Down Expand Up @@ -718,6 +701,7 @@ end
-- get package key
function _get_packagekey(packagename, requireinfo, version)
return _get_requirekey(requireinfo, {name = packagename,
host = requireinfo.host,
plat = requireinfo.plat,
arch = requireinfo.arch,
kind = requireinfo.kind,
Expand Down Expand Up @@ -752,6 +736,9 @@ function _inherit_parent_configs(requireinfo, package, parentinfo)
if parentinfo.arch then
requireinfo.arch = parentinfo.arch
end
if parentinfo.host then
requireinfo.host = parentinfo.host
end
requireinfo_configs.toolchains = requireinfo_configs.toolchains or parentinfo_configs.toolchains
requireinfo_configs.runtimes = requireinfo_configs.runtimes or parentinfo_configs.runtimes
requireinfo_configs.lto = requireinfo_configs.lto or parentinfo_configs.lto
Expand Down Expand Up @@ -1323,6 +1310,9 @@ function get_configs_str(package)
if requireinfo.arch then
table.insert(configs, requireinfo.arch)
end
if requireinfo.host then
table.insert(configs, requireinfo.host)
end
if requireinfo.kind then
table.insert(configs, requireinfo.kind)
end
Expand Down
9 changes: 9 additions & 0 deletions xmake/modules/private/action/require/impl/utils/requirekey.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

-- imports
import("core.base.hashset")
import("core.package.package", {alias = "core_package"})

-- get require key from requireinfo
function main(requireinfo, opt)
Expand All @@ -43,6 +44,14 @@ function main(requireinfo, opt)
if requireinfo.label then
key = key .. "/" .. requireinfo.label
end
if requireinfo.host then
if is_subhost(core_package.targetplat()) and os.subarch() == core_package.targetarch() then
-- we need to pass plat/arch to avoid repeat installation
-- @see https://github.com/xmake-io/xmake/issues/1579
else
key = key .. "/host"
end
end
if requireinfo.system then
key = key .. "/system"
end
Expand Down
32 changes: 20 additions & 12 deletions xmake/modules/private/xrepo/action/download.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,26 @@ function menu_options()
local options =
{
{'k', "kind", "kv", nil, "Enable static/shared library.",
values = {"static", "shared"} },
{'p', "plat", "kv", nil, "Set the given platform." },
{'a', "arch", "kv", nil, "Set the given architecture." },
values = {"static", "shared"}},
{'p', "plat", "kv", nil, "Set the given platform." },
{'a', "arch", "kv", nil, "Set the given architecture." },
{'m', "mode", "kv", nil, "Set the given mode.",
values = {"release", "debug"} },
values = {"release", "debug"}},
{'f', "configs", "kv", nil, "Set the given extra package configs.",
"e.g.",
" - xrepo download -f \"runtimes='MD'\" zlib",
" - xrepo download -f \"regex=true,thread=true\" boost"},
" - xrepo download -f \"regex=true,thread=true\" boost" },
{'j', "jobs", "kv", tostring(os.default_njob()),
"Set the number of parallel download jobs."},
"Set the number of parallel download jobs." },
{nil, "toolchain", "kv", nil, "Set the toolchain name." },
{nil, "toolchain_host", "kv", nil, "Set the host toolchain name." },
{nil, "includes", "kv", nil, "Includes extra lua configuration files.",
"e.g.",
" - xrepo download -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"},
{category = "Other Configuration" },
{nil, "force", "k", nil, "Force to redownload all packages."},
{'o', "outputdir", "kv", "packages","Set the packages download output directory."},
{},
" - xrepo download -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'" },
{category = "Other Configuration" },
{nil, "force", "k", nil, "Force to redownload all packages." },
{'o', "outputdir", "kv", "packages","Set the packages download output directory." },
{ },
{nil, "packages", "vs", nil, "The packages list.",
"e.g.",
" - xrepo download zlib boost",
Expand All @@ -57,7 +59,7 @@ function menu_options()
" - xrepo download -p android [--ndk=/xxx] -m debug \"pcre2 10.x\"",
" - xrepo download -p mingw [--mingw=/xxx] -k shared zlib",
" - xrepo download conan::zlib/1.2.11 vcpkg::zlib",
values = function (complete, opt) return import("private.xrepo.quick_search.completion")(complete, opt) end}
values = function (complete, opt) return import("private.xrepo.quick_search.completion")(complete, opt) end }
}

-- show menu options
Expand Down Expand Up @@ -134,6 +136,12 @@ function _download_packages(packages)
table.insert(config_argv, "-a")
table.insert(config_argv, option.get("arch"))
end
if option.get("toolchain") then
table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
end
if option.get("toolchain_host") then
table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
end
local mode = option.get("mode")
if mode then
table.insert(config_argv, "-m")
Expand Down
Loading

0 comments on commit cacefb8

Please sign in to comment.