From e125e51331d9a1f808e971e1469d4bdd54c7c5f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 20 Jul 2023 08:24:05 +0200 Subject: [PATCH 01/18] Handle OTHER_CFLAGS --- .../java/com/defold/extender/services/CocoaPodsService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 33ce4f16..8ae79d3e 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -591,6 +591,12 @@ private void parseConfig(JSONObject config, LanguageSet flags, Set linkf if (hasString(config, "OTHER_LDFLAGS")) { linkflags.addAll(getAsSplitString(config, "OTHER_LDFLAGS")); } + // compiler flags for c and objc files + // https://xcodebuildsettings.com/#other_cflags + if (hasString(config, "OTHER_CFLAGS")) { + flags.c.addAll(getAsSplitString(config, "OTHER_CFLAGS")); + flags.objc.addAll(getAsSplitString(config, "OTHER_CFLAGS")); + } // compiler flags if (hasString(config, "CLANG_CXX_LANGUAGE_STANDARD")) { String cppStandard = getAsString(config, "CLANG_CXX_LANGUAGE_STANDARD", "compiler-default"); From 4ee4be167a73960c48215d1f1481d207ed8b5c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 20 Jul 2023 08:26:00 +0200 Subject: [PATCH 02/18] Added -fmodules flag --- .../defold/extender/services/CocoaPodsService.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 8ae79d3e..6262610a 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -752,6 +752,8 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, // https://guides.cocoapods.org/syntax/podspec.html#compiler_flags spec.flags.ios.addAll(getAsSplitString(specJson, "compiler_flags")); spec.flags.osx.addAll(getAsSplitString(specJson, "compiler_flags")); + if (ios != null) spec.flags.ios.addAll(getAsJSONArray(ios, "compiler_flags")); + if (osx != null) spec.flags.osx.addAll(getAsJSONArray(osx, "compiler_flags")); spec.flags.ios.c.add("--language=c"); spec.flags.osx.c.add("--language=c"); spec.flags.ios.cpp.add("--language=c++"); @@ -760,8 +762,16 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, spec.flags.osx.objc.add("--language=objective-c"); spec.flags.ios.objcpp.add("--language=objective-c++"); spec.flags.osx.objcpp.add("--language=objective-c++"); - if (ios != null) spec.flags.ios.addAll(getAsJSONArray(ios, "compiler_flags")); - if (osx != null) spec.flags.osx.addAll(getAsJSONArray(osx, "compiler_flags")); + // CocoaPods sets CLANG_ENABLE_MODULES when creating an XCode project + // https://xcodebuildsettings.com/#clang_enable_modules + spec.flags.ios.objc.add("-fmodules"); + spec.flags.osx.objc.add("-fmodules"); + spec.flags.ios.objcpp.add("-fcxx-modules"); + spec.flags.osx.objcpp.add("-fcxx-modules"); + spec.flags.ios.objc.add("-fmodule-name=" + spec.name); + spec.flags.osx.objc.add("-fmodule-name=" + spec.name); + spec.flags.ios.objcpp.add("-fmodule-name=" + spec.name); + spec.flags.osx.objcpp.add("-fmodule-name=" + spec.name); // resources // https://guides.cocoapods.org/syntax/podspec.html#resources From 2ec2e0a15a84b6e4fb24b7620ffe11d0b5c6ab3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 20 Jul 2023 08:26:29 +0200 Subject: [PATCH 03/18] Update CocoaPodsService.java --- .../main/java/com/defold/extender/services/CocoaPodsService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 6262610a..d5be2764 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -588,6 +588,7 @@ private void parseConfig(JSONObject config, LanguageSet flags, Set linkf defines.addAll(getAsSplitString(config, "GCC_PREPROCESSOR_DEFINITIONS")); } // linker flags + // https://xcodebuildsettings.com/#other_ldflags if (hasString(config, "OTHER_LDFLAGS")) { linkflags.addAll(getAsSplitString(config, "OTHER_LDFLAGS")); } From fbd01db2fc7cc271cb70b09d73710ce982851e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 20 Jul 2023 17:41:27 +0200 Subject: [PATCH 04/18] Remove systemIncludes --- server/src/main/java/com/defold/extender/Extender.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index 52fdf7a1..c5915284 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -760,6 +760,14 @@ else if (platform.contains("osx")) { Map mergedContextWithPodsForObjC = ExtenderUtil.mergeContexts(trimmedContext, podContextObjC); Map mergedContextWithPodsForObjCpp = ExtenderUtil.mergeContexts(trimmedContext, podContextObjCpp); + // remove systemIncludes from objc and objc++ + // this is a bit crude but cocoapod builds do not provide any -isystem option and + // it seems like the "{{env.SYSROOT}}/usr/include/c++/v1" set in build.yml is + // causing problems when building objc code with -fmodules enabled + // see https://github.com/defold/extender/issues/308 + mergedContextWithPodsForObjC.put("systemIncludes", new ArrayList()); + mergedContextWithPodsForObjCpp.put("systemIncludes", new ArrayList()); + for (File src : pod.sourceFiles) { String extension = FilenameUtils.getExtension(src.getAbsolutePath()); if (extension.equals("swift")) { From 9e216839f4c9d9ff444e25d0b5877ff01fd16632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 20 Jul 2023 17:41:47 +0200 Subject: [PATCH 05/18] Add pods in reverse order --- .../java/com/defold/extender/services/CocoaPodsService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index d5be2764..dc7dbcc6 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -943,7 +943,7 @@ private List installPods(File workingDir, Map jobEnvCon PodSpec mainpod = specsMap.get(mainpodname); if (podnameparts.length == 1) { - specs.add(mainpod); + specs.add(0, mainpod); } else { PodSpec current = mainpod; @@ -951,7 +951,7 @@ private List installPods(File workingDir, Map jobEnvCon String subspecname = podnameparts[i]; for (PodSpec subspec : current.subspecs) { if (subspec.name.equals(subspecname)) { - specs.add(subspec); + specs.add(0, subspec); current = subspec; break; } From 56f7ffc11a5414ea1efd54fb71545582247e1632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Fri, 21 Jul 2023 07:36:55 +0200 Subject: [PATCH 06/18] Removed duplicate release instruction --- README_RELEASE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README_RELEASE.md b/README_RELEASE.md index 414b2094..c88f624c 100644 --- a/README_RELEASE.md +++ b/README_RELEASE.md @@ -23,7 +23,6 @@ This will create a new task definition on AWS ECS and update the service to run Note, we currently don't have any macOS specific tests, so you may skip the tests 1. Build (without tests): `./server/scripts/build-standalone.sh -xtest` - 1. Run: `./server/scripts/build-standalone.sh -xtest` 1. Run `./server/scripts/publish-darwin-stage.sh` This will upload the packages and jar files to the macOS machine on AWS, and restart the extender service. From 3eda912f7e25dfa888fdf41f91de160e87c24e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Fri, 21 Jul 2023 12:44:35 +0200 Subject: [PATCH 07/18] Install pods and build ordered list of podspecs Dependencies first! --- .../extender/services/CocoaPodsService.java | 108 +++++++++++++----- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 9ce0ac08..f8374120 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -32,8 +32,10 @@ import java.util.Map; import java.util.HashMap; import java.util.Set; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Iterator; +import java.util.Collections; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.lang.StringBuilder; @@ -224,6 +226,7 @@ public class PodSpec { public PodSpec parentSpec = null; public List defaultSubspecs = new ArrayList<>(); public List subspecs = new ArrayList<>(); + public List dependencies = new ArrayList<>(); public PlatformAndLanguageSet flags = new PlatformAndLanguageSet(); public PlatformSet defines = new PlatformSet(); @@ -235,6 +238,15 @@ public class PodSpec { public PlatformSet libraries = new PlatformSet(); public File dir; + public PodSpec getSubspec(String name) { + for (PodSpec spec : subspecs) { + if (spec.name.equals(name)) { + return spec; + } + } + return null; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -250,12 +262,13 @@ public String toString() { sb.append(" frameworks: " + frameworks + "\n"); sb.append(" vendored_frameworks: " + vendoredFrameworks + "\n"); sb.append(" libraries: " + libraries + "\n"); + sb.append(" dependencies: " + dependencies + "\n"); sb.append(" parentSpec: " + ((parentSpec != null) ? parentSpec.name : "null") + "\n"); + sb.append(" default_subspecs: " + defaultSubspecs + "\n"); for (PodSpec sub : subspecs) { sb.append(" subspec: " + sub.name + "\n"); } return sb.toString(); - } } @@ -844,6 +857,14 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, } } + // parse dependencies + Map> dependencies = (Map>)specJson.get("dependencies"); + if (dependencies != null) { + for (String dependency : dependencies.keySet()) { + spec.dependencies.add(dependency); + } + } + // find source and header files // https://guides.cocoapods.org/syntax/podspec.html#source_files List sourceFilePatterns = new ArrayList<>(); @@ -878,6 +899,55 @@ private PodSpec createPodSpec(String specJson, File podsDir, Map return spec; } + // 'GoogleUtilities/Environment (7.10.0)' -> 'GoogleUtilities/Environment' -> ['GoogleUtilities', 'Environment'] + private String[] splitPodname(String pod) { + return pod.replaceFirst(" \\(.*\\)", "").split("/"); + } + + /** + * Get pod spec based on a pod name with optional sub pod (GoogleUtilities/Environment) + * @param pods Map of pod names to pod specs + * @param podname The pod to find + * @return The pod spec + */ + private PodSpec getPod(Map pods, String podname) throws ExtenderException { + // 'GoogleUtilities/Environment (7.10.0)' -> 'GoogleUtilities/Environment' -> ['GoogleUtilities', 'Environment'] + String podnameparts[] = splitPodname(podname); + // 'GoogleUtilities' + String mainpodname = podnameparts[0]; + PodSpec current = pods.get(mainpodname); + if (podnameparts.length > 1) { + for (int i = 1; i < podnameparts.length; i++) { + String subspecname = podnameparts[i]; + PodSpec subspec = current.getSubspec(subspecname); + if (subspec == null) { + throw new ExtenderException("Unable to find subspec '" + subspecname + "' in pod '" + current.name + "'"); + } + current = subspec; + } + } + return current; + } + + /** + * Get a sorted set of pod specs with dependencies from a list of pod names. + * This function will recursively go through all pods and add their + * dependencies to the final set of pods. The pod specs will be added to the + * set such that the dependencies of a pod are added before their pod itself. + * @param specsMap Map with pod specs to search for pods + * @param podnames Names of the pods to get + * @return Set of pod specs + */ + private LinkedHashSet getSpecsAndDependencies(Map specsMap, List podnames) throws ExtenderException { + LinkedHashSet specs = new LinkedHashSet<>(); + for (String podname : podnames) { + PodSpec spec = getPod(specsMap, podname); + specs.addAll(getSpecsAndDependencies(specsMap, spec.dependencies)); + specs.add(spec); + } + return specs; + } + /** * Install pods from a podfile and create PodSpec instances for each installed pod. * @param workingDir Directory where to install pods. The directory must contain a valid Podfile @@ -913,28 +983,28 @@ private List installPods(File workingDir, Map jobEnvCon - GoogleAppMeasurement (= 8.13.0) */ List lines = Files.readAllLines(podFileLock.toPath()); - Set pods = new LinkedHashSet<>(); + Set podnames = new LinkedHashSet<>(); while (!lines.isEmpty()) if (lines.remove(0).startsWith("PODS:")) break; while (!lines.isEmpty()) { String line = lines.remove(0); if (line.trim().isEmpty()) break; if (line.startsWith(" -")) { // ' - "GoogleUtilities/Environment (7.10.0)":' -> 'GoogleUtilities/Environment (7.10.0)' - String pod = line.trim().replace("- ", "").replace(":", "").replace("\"",""); - pods.add(pod); + String podname = line.trim().replace("- ", "").replace(":", "").replace("\"",""); + podnames.add(podname); } } + // get all the pod specs and store them in a map, keyed on pod name File podsDir = new File(workingDir, "Pods"); - List specs = new ArrayList<>(); Map specsMap = new HashMap<>(); - for (String pod : pods) { + for (String podname : podnames) { // 'GoogleUtilities/Environment (7.10.0)' -> 'GoogleUtilities/Environment' -> ['GoogleUtilities', 'Environment'] - String podnameparts[] = pod.replaceFirst(" \\(.*\\)", "").split("/"); + String podnameparts[] = splitPodname(podname); // 'GoogleUtilities' String mainpodname = podnameparts[0]; // 'GoogleUtilities/Environment (7.10.0)' -> '7.10.0' - String podversion = pod.replaceFirst(".*\\(", "").replace(")", ""); + String podversion = podname.replaceFirst(".*\\(", "").replace(")", ""); if (!specsMap.containsKey(mainpodname)) { String cmd = "pod spec cat --regex ^" + mainpodname + "$ --version=" + podversion; String specJson = execCommand(cmd).replace(cmd, ""); @@ -951,26 +1021,12 @@ private List installPods(File workingDir, Map jobEnvCon specsMap.put(mainpodname, createPodSpec(specJson, podsDir, jobEnvContext)); } - - PodSpec mainpod = specsMap.get(mainpodname); - if (podnameparts.length == 1) { - specs.add(0, mainpod); - } - else { - PodSpec current = mainpod; - for (int i = 1; i < podnameparts.length; i++) { - String subspecname = podnameparts[i]; - for (PodSpec subspec : current.subspecs) { - if (subspec.name.equals(subspecname)) { - specs.add(0, subspec); - current = subspec; - break; - } - } - } - } } + // build list of specs and their dependencies (dependencies first) + List reversePodnames = new ArrayList<>(podnames); + Collections.reverse(reversePodnames); + List specs = new ArrayList(getSpecsAndDependencies(specsMap, reversePodnames)); LOGGER.info("Installed pods"); return specs; } From fce67d7da8d65b3d58ca3870b03a2340fbec2d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Tue, 17 Oct 2023 08:59:53 +0200 Subject: [PATCH 08/18] Log when job folder is kept --- .../src/main/java/com/defold/extender/ExtenderController.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/com/defold/extender/ExtenderController.java b/server/src/main/java/com/defold/extender/ExtenderController.java index 9077e24a..6f90d888 100644 --- a/server/src/main/java/com/defold/extender/ExtenderController.java +++ b/server/src/main/java/com/defold/extender/ExtenderController.java @@ -364,6 +364,9 @@ public void buildEngineAsync(HttpServletRequest _request, LOGGER.warn("Failed to delete job directory"); } } + else { + LOGGER.info("Keeping job folder due to debug flags"); + } } } From 1817257408f478d3a2964d79d5b6aaddf3c2d94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Tue, 17 Oct 2023 09:07:07 +0200 Subject: [PATCH 09/18] Update AsyncBuilder.java --- server/src/main/java/com/defold/extender/AsyncBuilder.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/defold/extender/AsyncBuilder.java b/server/src/main/java/com/defold/extender/AsyncBuilder.java index 77f260ba..1f06e7b1 100644 --- a/server/src/main/java/com/defold/extender/AsyncBuilder.java +++ b/server/src/main/java/com/defold/extender/AsyncBuilder.java @@ -37,6 +37,9 @@ public class AsyncBuilder { private long resultLifetime; private boolean keepJobDirectory = false; + private static final String DM_DEBUG_JOB_FOLDER = System.getenv("DM_DEBUG_JOB_FOLDER"); + private static final String DM_DEBUG_KEEP_JOB_FOLDER = System.getenv("DM_DEBUG_KEEP_JOB_FOLDER"); + public AsyncBuilder(DefoldSdkService defoldSdkService, DataCacheService dataCacheService, GradleService gradleService, @@ -48,7 +51,7 @@ public AsyncBuilder(DefoldSdkService defoldSdkService, this.gradleService = gradleService; this.cocoaPodsService = cocoaPodsService; this.jobResultLocation = new File(jobResultLocation); - this.keepJobDirectory = System.getenv("DM_DEBUG_KEEP_JOB_FOLDER") != null && System.getenv("DM_DEBUG_JOB_FOLDER") == null; + this.keepJobDirectory = (DM_DEBUG_KEEP_JOB_FOLDER != null) || (DM_DEBUG_JOB_FOLDER != null); this.resultLifetime = jobResultLifetime; } From 262a69e6cdf47f835eadb76294c3f0af1322d2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Tue, 17 Oct 2023 09:08:55 +0200 Subject: [PATCH 10/18] Making function public --- server/src/main/java/com/defold/extender/ExtenderUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index f745867b..06e42318 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -110,7 +110,7 @@ static List pruneItems(List input, List includePatterns, return items; } - static String getRelativePath(File base, File path) { + public static String getRelativePath(File base, File path) { return base.toURI().relativize(path.toURI()).getPath(); } From 8cf89adfd24f09f6f017affefb29fd84a19f3eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Tue, 17 Oct 2023 09:09:02 +0200 Subject: [PATCH 11/18] Working state --- .../java/com/defold/extender/Extender.java | 352 ++++++++++++++++-- .../extender/services/CocoaPodsService.java | 287 +++++++++++--- 2 files changed, 569 insertions(+), 70 deletions(-) diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index c5915284..a8e99368 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -505,6 +505,7 @@ private List getFrameworkStaticLibIncludeDirs(ResolvedPods pods) { } private List getIncludeDirs(File extDir) { + LOGGER.info("getIncludeDirs " + extDir); List includes = getExtLocalIncludeDirs(extDir); includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(buildDirectory, extDir.getName())) ); // where we generate source from protobuf files @@ -532,11 +533,292 @@ private List getIncludeDirs(File extDir) { } includes.addAll(podIncludes); includes.addAll(getFrameworkStaticLibIncludeDirs(resolvedPods)); + LOGGER.info("getIncludeDirs has resolved pods"); + LOGGER.info("getIncludeDirs has resolved pods - adding " + resolvedPods.generatedDir); + includes.add(ExtenderUtil.getRelativePath(jobDirectory, resolvedPods.generatedDir)); + + for (PodSpec pod : resolvedPods.pods) { + if (pod.iosModuleMap != null) { + LOGGER.info("getIncludeDirs has resolved pods - adding " + pod.iosModuleMap); + includes.add(ExtenderUtil.getRelativePath(jobDirectory, new File(pod.iosModuleMap))); + } + includes.add(ExtenderUtil.getRelativePath(jobDirectory, pod.generatedDir)); + } + } + else { + LOGGER.info("getIncludeDirs has no resolved pods"); } return pruneNonExisting(includes); } + private void emitSwiftHeaders(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + //"{{env.PLATFORMSDK_DIR}}/XcodeDefault{{env.XCODE_VERSION}}.xctoolchain/usr/lib/swift-{{env.SWIFT_VERSION}}/iphonesimulator" + + File moduleMap = new File(pod.iosModuleMap); + File swiftModulePath = new File(pod.generatedDir, pod.moduleName + ".swiftmodule"); + File objcHeaderPath = new File(pod.generatedDir, pod.moduleName + "-Swift.h"); + + String cmd = "swiftc"; + + // Emit object file(s) (-c) + cmd += " -emit-object"; + + // Emit an Objective-C header file + cmd += " -emit-objc-header"; + cmd += " -emit-objc-header-path " + objcHeaderPath.getAbsolutePath(); + + // Name of the module to build + cmd += " -module-name {{module_name}}"; + + // Compile without any optimization + cmd += " -Onone"; + + // Enforce law of exclusivity + cmd += " -enforce-exclusivity=checked"; + + // Source files + for (File srcFile : pod.swiftSourceFiles) { + cmd += " " + srcFile.getAbsolutePath(); + } + + cmd += " -DDEBUG"; + cmd += " -D COCOAPODS"; + cmd += " -DSWIFT_PACKAGE"; + + // Compile against + cmd += " -sdk {{env.SYSROOT}}"; + + // Generate code for the given target , such as x86_64-apple-macos10.9 + // cmd += " -target arm-apple-darwin19"; + cmd += " -target arm64-apple-ios10.0"; + + // Enable the use of forward slash regular-expression literal syntax + // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals + cmd += " -enable-bare-slash-regex"; + + // Emit debug info. This is the preferred setting for debugging with LLDB. + cmd += " -g"; + + cmd += " -Xfrontend -serialize-debugging-options"; + + // Allows this module's internal API to be accessed for testing + cmd += " -enable-testing"; + + // Interpret input according to a specific Swift language version number + cmd += " -swift-version 5"; + // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' + // cmd += " -swift-version {{env.SWIFT_VERSION}}"; + + // Number of commands to execute in parallel + cmd += " -j8"; + // Enable combining frontend jobs into batches + cmd += " -enable-batch-mode"; + + cmd += " -use-frontend-parseable-output"; + + cmd += " -save-temps"; + + cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; + cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; + cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; + + + List includes = getIncludeDirs(pod.dir); + + List frameworks = new ArrayList<>(); + frameworks.addAll(getFrameworks(pod.dir)); + frameworks.addAll(getFrameworks(resolvedPods)); + List frameworkPaths = new ArrayList<>(); + frameworkPaths.addAll(getFrameworkPaths(pod.dir)); + frameworkPaths.addAll(getFrameworkPaths(resolvedPods)); + + Map context = createContext(manifestContext); + context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); + context.put("module_name", pod.moduleName); + + String command = templateExecutor.execute(cmd, context); + LOGGER.info("SWIFT COMMAND TO EMIT HEADERS " + command); + commands.add(command); + } + + + + private void emitSwiftModule(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + File moduleMap = new File(pod.iosModuleMap); + File swiftModulePath = new File(pod.generatedDir, pod.moduleName + ".swiftmodule"); + + String cmd = "swiftc"; + + // Emit object file(s) (-c) + cmd += " -emit-object"; + + // Emit a swift module + cmd += " -module-name " + pod.moduleName; + cmd += " -emit-module"; + cmd += " -emit-module-path " + swiftModulePath.getAbsolutePath(); + + // Compile without any optimization + cmd += " -Onone"; + + // Enforce law of exclusivity + cmd += " -enforce-exclusivity=checked"; + + // Source files + for (File srcFile : pod.swiftSourceFiles) { + cmd += " " + srcFile.getAbsolutePath(); + } + + cmd += " -DDEBUG"; + cmd += " -D COCOAPODS"; + cmd += " -DSWIFT_PACKAGE"; + + // Compile against + cmd += " -sdk {{env.SYSROOT}}"; + + // Generate code for the given target , such as x86_64-apple-macos10.9 + // cmd += " -target arm-apple-darwin19"; + cmd += " -target arm64-apple-ios10.0"; + + // Enable the use of forward slash regular-expression literal syntax + // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals + cmd += " -enable-bare-slash-regex"; + + // Emit debug info. This is the preferred setting for debugging with LLDB. + cmd += " -g"; + + cmd += " -Xfrontend -serialize-debugging-options"; + + // Allows this module's internal API to be accessed for testing + cmd += " -enable-testing"; + + // Interpret input according to a specific Swift language version number + cmd += " -swift-version 5"; + // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' + // cmd += " -swift-version {{env.SWIFT_VERSION}}"; + + // Number of commands to execute in parallel + cmd += " -j8"; + // Enable combining frontend jobs into batches + cmd += " -enable-batch-mode"; + + cmd += " -use-frontend-parseable-output"; + + cmd += " -save-temps"; + + cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; + cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; + cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; + + List includes = getIncludeDirs(pod.dir); + + List frameworks = new ArrayList<>(); + frameworks.addAll(getFrameworks(pod.dir)); + frameworks.addAll(getFrameworks(resolvedPods)); + List frameworkPaths = new ArrayList<>(); + frameworkPaths.addAll(getFrameworkPaths(pod.dir)); + frameworkPaths.addAll(getFrameworkPaths(resolvedPods)); + + Map context = createContext(manifestContext); + // context.put("src", ExtenderUtil.getRelativePath(jobDirectory, src)); + context.put("tgt", swiftModulePath.getAbsolutePath()); + context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); + + String command = templateExecutor.execute(cmd, context); + LOGGER.info("SWIFT COMMAND TO EMIT MODULE " + command); + commands.add(command); + } + + + private File addCompileFileSwiftStatic(PodSpec pod, int index, File src, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + File o = new File(buildDirectory, String.format("%s_%d.o", src.getName(), index)); + + String cmd = "swiftc"; + cmd = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-frontend"; + + // MODE: Emit object file(s) (-c) + cmd += " -emit-object"; + + cmd += " -module-name " + pod.moduleName; + + // Primary source file + cmd += " -primary-file " + src.getAbsolutePath(); + + // Source files + for (File srcFile : pod.swiftSourceFiles) { + if (!srcFile.equals(src)) { + cmd += " " + srcFile.getAbsolutePath(); + } + } + + // Compile without any optimization + cmd += " -Onone"; + + // Enforce law of exclusivity + cmd += " -enforce-exclusivity=checked"; + + cmd += " -DDEBUG"; + cmd += " -D COCOAPODS"; + cmd += " -DSWIFT_PACKAGE"; + + // Compile against + cmd += " -sdk {{env.SYSROOT}}"; + + // Implicitly imports the Objective-C half of a module + cmd += " -import-underlying-module"; + + // Generate code for the given target , such as x86_64-apple-macos10.9 + // cmd += " -target arm-apple-darwin19"; + cmd += " -target arm64-apple-ios10.0"; + + // Enable the use of forward slash regular-expression literal syntax + // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals + cmd += " -enable-bare-slash-regex"; + + // Emit debug info. This is the preferred setting for debugging with LLDB. + cmd += " -g"; + + cmd += " -enable-objc-interop"; + + // Allows this module's internal API to be accessed for testing + cmd += " -enable-testing"; + + // Interpret input according to a specific Swift language version number + cmd += " -swift-version 5"; + // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' + // cmd += " -swift-version {{env.SWIFT_VERSION}}"; + + // cmd += " {{#includes}}-Xcc -I{{{.}}} {{/includes}}"; + + cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; + cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; + cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; + // Write output to + cmd += " -o{{tgt}}"; + + List includes = getIncludeDirs(pod.dir); + + List frameworks = new ArrayList<>(); + frameworks.addAll(getFrameworks(pod.dir)); + frameworks.addAll(getFrameworks(resolvedPods)); + List frameworkPaths = new ArrayList<>(); + frameworkPaths.addAll(getFrameworkPaths(pod.dir)); + frameworkPaths.addAll(getFrameworkPaths(resolvedPods)); + + Map context = createContext(manifestContext); + // context.put("src", ExtenderUtil.getRelativePath(jobDirectory, src)); + context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, o)); + context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); + context.put("module_name", pod.moduleName); + + String command = templateExecutor.execute(cmd, context); + LOGGER.info("SWIFT COMMAND TO COMPILE SWIFT " + command); + commands.add(command); + return o; + + } + + private File addCompileFileCpp_Internal(int index, File extDir, File src, Map manifestContext, String cmd, List commands) throws IOException, InterruptedException, ExtenderException { List includes = getIncludeDirs(extDir); @@ -719,9 +1001,6 @@ else if (platformConfig.zigSourceRe != null && ExtenderUtil.matchesFile(src, pla // compile the source files of a pod and return a list of object files private List compilePodSourceFiles(PodSpec pod, Map manifestContext) throws IOException, InterruptedException, ExtenderException { - List objs = new ArrayList<>(); - List commands = new ArrayList<>(); - // clean up flags from context Map trimmedContext = ExtenderUtil.mergeContexts(manifestContext, new HashMap<>()); trimmedContext.put("flags", new ArrayList()); @@ -732,6 +1011,7 @@ private List compilePodSourceFiles(PodSpec pod, Map mani Map podContextCpp = new HashMap<>(); Map podContextObjC = new HashMap<>(); Map podContextObjCpp = new HashMap<>(); + Map podContextSwift = new HashMap<>(); if (platform.contains("ios")) { podContextC.put("flags", new ArrayList(pod.flags.ios.c)); @@ -759,6 +1039,7 @@ else if (platform.contains("osx")) { Map mergedContextWithPodsForCpp = ExtenderUtil.mergeContexts(trimmedContext, podContextCpp); Map mergedContextWithPodsForObjC = ExtenderUtil.mergeContexts(trimmedContext, podContextObjC); Map mergedContextWithPodsForObjCpp = ExtenderUtil.mergeContexts(trimmedContext, podContextObjCpp); + Map mergedContextWithPodsForSwift = ExtenderUtil.mergeContexts(trimmedContext, podContextSwift); // remove systemIncludes from objc and objc++ // this is a bit crude but cocoapod builds do not provide any -isystem option and @@ -767,32 +1048,60 @@ else if (platform.contains("osx")) { // see https://github.com/defold/extender/issues/308 mergedContextWithPodsForObjC.put("systemIncludes", new ArrayList()); mergedContextWithPodsForObjCpp.put("systemIncludes", new ArrayList()); + mergedContextWithPodsForSwift.put("systemIncludes", new ArrayList()); + + LOGGER.info("BUILDING POD " + pod.name); + + List objs = new ArrayList<>(); + + if (!pod.swiftSourceFiles.isEmpty()) { + LOGGER.info("POD " + pod.name + " HAS SWIFT SOURCES"); + + // generate headers from swift files + List emitSwiftHeaderCommands = new ArrayList<>(); + emitSwiftHeaders(pod, mergedContextWithPodsForC, emitSwiftHeaderCommands); + ProcessExecutor.executeCommands(processExecutor, emitSwiftHeaderCommands); // in parallel + + // generate swift module from swift files + List emitSwiftModuleCommands = new ArrayList<>(); + emitSwiftModule(pod, mergedContextWithPodsForC, emitSwiftModuleCommands); + ProcessExecutor.executeCommands(processExecutor, emitSwiftModuleCommands); // in parallel + // compile swift source files one by one + List compileSwiftCommands = new ArrayList<>(); + for (File src : pod.swiftSourceFiles) { + LOGGER.info("POD " + pod.name + " SWIFT SOURCE FILE " + src); + final int i = getAndIncreaseNameCount(); + File o = addCompileFileSwiftStatic(pod, i, src, mergedContextWithPodsForC, compileSwiftCommands); + objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); + } + // compilePodSwiftFilesToModule(pod, mergedContextWithPodsForC, compileSwiftCommands); + ProcessExecutor.executeCommands(processExecutor, compileSwiftCommands); // in parallel + } + + List commands = new ArrayList<>(); for (File src : pod.sourceFiles) { + LOGGER.info("BUILDING SOURCE FILE " + src.getAbsolutePath()); String extension = FilenameUtils.getExtension(src.getAbsolutePath()); - if (extension.equals("swift")) { - throw new ExtenderException("Unable to build '" + pod.name + "' since it includes Swift source files"); + final int i = getAndIncreaseNameCount(); + File o = null; + // use the correct context depending on the source file language + if (extension.equals("c")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForC, commands); + } + else if (extension.equals("m")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjC, commands); + } + else if (extension.equals("mm")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjCpp, commands); } else { - final int i = getAndIncreaseNameCount(); - File o = null; - // use the correct context depending on the source file language - if (extension.equals("c")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForC, commands); - } - else if (extension.equals("m")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjC, commands); - } - else if (extension.equals("mm")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjCpp, commands); - } - else { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForCpp, commands); - } - objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForCpp, commands); } + objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); } ProcessExecutor.executeCommands(processExecutor, commands); // in parallel + return objs; } @@ -806,6 +1115,7 @@ private List buildPods() throws IOException, InterruptedException, Extende LOGGER.info("buildPods"); for (PodSpec pod : resolvedPods.pods) { + LOGGER.info("buildPod " + pod.name); // The source files of each pod will be compiled and built as a library. // We use the same mechanism as when building the extension and create a // manifest context for each pod diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index f8374120..3bc8b767 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -48,6 +48,7 @@ public class ResolvedPods { public List pods = new ArrayList<>(); public File podsDir; public File frameworksDir; + public File generatedDir; public String platformMinVersion; // In the functions below we also get the values from the parent spec @@ -217,10 +218,15 @@ public String toString() { public class PodSpec { public String name = ""; + public String moduleName = ""; public String version = ""; public String originalJSON = ""; public String iosversion = ""; public String osxversion = ""; + public String iosModuleMap = null; + public String osxModuleMap = null; + public Boolean isFramework = false; + public Set swiftSourceFiles = new LinkedHashSet<>(); public Set sourceFiles = new LinkedHashSet<>(); public Set includePaths = new LinkedHashSet<>(); public PodSpec parentSpec = null; @@ -236,7 +242,10 @@ public class PodSpec { public PlatformSet resources = new PlatformSet(); public PlatformSet frameworks = new PlatformSet(); public PlatformSet libraries = new PlatformSet(); + public PlatformSet publicHeaders = new PlatformSet(); public File dir; + public boolean installed; + public File generatedDir; public PodSpec getSubspec(String name) { for (PodSpec spec : subspecs) { @@ -252,6 +261,8 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append(name + ":" + version + "\n"); sb.append(" dir: " + dir + "\n"); + sb.append(" generated dir: " + generatedDir + "\n"); + sb.append(" installed: " + installed + "\n"); sb.append(" src: " + sourceFiles + "\n"); sb.append(" includes: " + includePaths + "\n"); sb.append(" defines: " + defines + "\n"); @@ -387,51 +398,45 @@ private String createMainPodFile(List podFiles, File jobDirectory, File wo } /** - * Return a list of source files for a pod + * Add source files matching a pattern to a pod * @param pod * @param pattern Source file pattern (glob format) - * @return List of files */ - private List findPodSourceFiles(PodSpec pod, String pattern) { - List srcFiles = new ArrayList<>(); - + private void addPodSourceFiles(PodSpec pod, String pattern) { String absolutePatternPath = pod.dir.getAbsolutePath() + File.separator + pattern; - List podSrcFiles = ExtenderUtil.listFilesGlob(pod.dir, absolutePatternPath); + List podSrcFiles = listFilesGlob(pod.dir, absolutePatternPath); for (File podSrcFile : podSrcFiles) { - if (!podSrcFile.getName().endsWith(".h")) { - srcFiles.add(podSrcFile); + final String filename = podSrcFile.getName(); + if (filename.endsWith(".swift")) { + pod.swiftSourceFiles.add(podSrcFile); + } + else if (!filename.endsWith(".h")) { + pod.sourceFiles.add(podSrcFile); } } - - return srcFiles; - } + } /** - * Return a list of include paths for a pod + * Add a list of include paths matching a pattern to a pod * @param pod * @param pattern Source file pattern (glob format) - * @return List of include paths */ - private List findPodIncludePaths(PodSpec pod, String pattern) { - Set includePaths = new LinkedHashSet<>(); - + private void addPodIncludePaths(PodSpec pod, String pattern) { String absolutePatternPath = pod.dir.getAbsolutePath() + File.separator + pattern; - List podSrcFiles = ExtenderUtil.listFilesGlob(pod.dir, absolutePatternPath); + List podSrcFiles = listFilesGlob(pod.dir, absolutePatternPath); for (File podSrcFile : podSrcFiles) { if (podSrcFile.getName().endsWith(".h")) { File podIncludeDir = podSrcFile.getParentFile(); if (podIncludeDir != null) { - includePaths.add(podIncludeDir); + pod.includePaths.add(podIncludeDir); File podIncludeParentDir = podIncludeDir.getParentFile(); if (podIncludeParentDir != null) { - includePaths.add(podIncludeParentDir); + pod.includePaths.add(podIncludeParentDir); } } } } - - return new ArrayList(includePaths); - } + } // copy the files and folders of a directory recursively // the function will also resolve symlinks while copying files and folders @@ -720,14 +725,161 @@ private void parseMultiPlatformConfig(PodSpec spec, JSONObject config) { } } + private List listFilesGlob(File dir, String pattern) { + List files = ExtenderUtil.listFilesGlob(dir, pattern); + // Cocoapods uses Ruby where glob patterns are treated slightly differently: + // Ruby: foo/**/*.h will find .h files in any subdirectory of foo AND in foo/ + // Java: foo/**/*.h will find .h files in any subdirectory of foo but NOT in foo/ + if (pattern.contains("/**/")) { + pattern = pattern.replaceFirst("\\/\\*\\*\\/", "/"); + files.addAll(ExtenderUtil.listFilesGlob(dir, pattern)); + } + return files; + } + + // https://clang.llvm.org/docs/Modules.html#module-declaration + private static final String MODULEMAP_TEMPLATE = "" + + "{{FRAMEWORKOPT}} module {{MODULE_ID}} {\n" + + " {{#HEADERS}}\n" + + " header \"{{{.}}}\"\n" + + " {{/HEADERS}}\n" + + " {{#UMBRELLA_HEADERS}}\n" + + " umbrella header \"{{{.}}}\"\n" + + " {{/UMBRELLA_HEADERS}}\n" + + " {{#UMBRELLA_DIRECTORIES}}\n" + + " umbrella \"{{{.}}}\"\n" + + " {{/UMBRELLA_DIRECTORIES}}\n" + + " export *\n" + + " requires objc\n" + + " {{SUBMODULE}}\n" + + "}"; + + + + private void createModuleMap(PodSpec pod, Set headerPatterns, File moduleMapFile, File jobDir) throws ExtenderException { + LOGGER.info("createModuleMap() " + pod.moduleName + " header patterns: " + headerPatterns); + if (headerPatterns.isEmpty()) { + // headerPatterns = new HashSet<>(); + // headerPatterns.add("**/*.h"); + } + + List headers = new ArrayList<>(); + List umbrellaHeaders = new ArrayList<>(); + List umbrellaDirectories = new ArrayList<>(); + + // swift source file header + // generated by the swift compiler if -emit-objc-header flag is set + if (!pod.swiftSourceFiles.isEmpty()) { + File swiftModuleHeader = new File(pod.generatedDir, pod.moduleName + "-Swift.h"); + // headers.add(ExtenderUtil.getRelativePath(pod.generatedDir.getParentFile(), swiftModuleHeader)); + // headers.add(ExtenderUtil.getRelativePath(pod.generatedDir, swiftModuleHeader)); + headers.add(swiftModuleHeader.getAbsolutePath()); + } + + // umbrella headers + for (String headerPattern : headerPatterns) { + String absoluteHeaderPatternPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; + LOGGER.info("createModuleMap() absoluteHeaderPatternPath " + absoluteHeaderPatternPath); + List headerFiles = listFilesGlob(pod.dir, absoluteHeaderPatternPath); + for (File headerFile : headerFiles) { + LOGGER.info("createModuleMap() header " + headerFile.getAbsolutePath()); + if (headerFile.getName().equals(pod.moduleName + ".h")) { + // umbrellaHeaders.add(ExtenderUtil.getRelativePath(jobDir, headerFile)); + umbrellaHeaders.add(headerFile.getAbsolutePath()); + } + else { + // headers.add("header \"" + headerFile.getAbsolutePath() + "\""); + } + } + } + + // umbrella directory + if (umbrellaHeaders.isEmpty()) { + for (String headerPattern : headerPatterns) { + headerPattern = headerPattern.replace("/*.h", ""); + headerPattern = headerPattern.replace("/**", ""); + LOGGER.info("HEADER PATTERN " + headerPattern); + headerPattern = headerPattern.substring(0, headerPattern.lastIndexOf("/")); + String headerPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; + umbrellaDirectories.add(headerPath); + } + } + + // module name, including parent spec module name + String fullModuleName = ((pod.parentSpec != null) ? (pod.parentSpec.moduleName + "_") : "") + pod.moduleName; + + // generate final modulemap + HashMap envContext = new HashMap<>(); + envContext.put("FRAMEWORKOPT", pod.isFramework ? "framework" : ""); + envContext.put("MODULE_ID", fullModuleName); + envContext.put("HEADERS", headers); + envContext.put("UMBRELLA_HEADERS", umbrellaHeaders); + envContext.put("UMBRELLA_DIRECTORIES", umbrellaDirectories); + envContext.put("SUBMODULE", pod.isFramework ? "module * { export * }" : ""); + + String moduleMap = templateExecutor.execute(MODULEMAP_TEMPLATE, envContext); + LOGGER.info("Created modulemap:\n{}", moduleMap); + + try { + Files.writeString(moduleMapFile.toPath(), moduleMap); + } + catch (IOException e) { + throw new ExtenderException(e, "Unable to create modulemap for " + pod.moduleName); + } + LOGGER.info("createModuleMap() DONE " + pod.moduleName); + } + + private File createModuleMapFile(PodSpec pod, String platform) { + String filename = null; + if (pod.parentSpec != null) { + filename = pod.parentSpec.moduleName + "-" + pod.moduleName; + } + else { + filename = pod.moduleName; + } + filename += "-" + platform + ".modulemap"; + filename = "module.modulemap"; + return new File(pod.generatedDir, filename); + } + + private String createIosModuleMap(PodSpec pod, File jobDir) throws ExtenderException { + File moduleMapFile = createModuleMapFile(pod, "ios"); + createModuleMap(pod, pod.publicHeaders.ios, moduleMapFile, jobDir); + return moduleMapFile.getAbsolutePath(); + } + private String createOsxModuleMap(PodSpec pod, File jobDir) throws ExtenderException { + File moduleMapFile = createModuleMapFile(pod, "osx"); + createModuleMap(pod, pod.publicHeaders.osx, moduleMapFile, jobDir); + return moduleMapFile.getAbsolutePath(); + } + // https://guides.cocoapods.org/syntax/podspec.html - private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, Map jobEnvContext) throws ExtenderException { + private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, File generatedDir, File jobDir, Map jobEnvContext) throws ExtenderException { PodSpec spec = new PodSpec(); spec.name = (String)specJson.get("name"); + if (specJson.containsKey("module_name")) { + spec.moduleName = (String)specJson.get("module_name"); + } + else { + // NSData+zlib -> NSData_zlib + spec.moduleName = ((String)specJson.get("name")).replaceAll("[\\+].*", ""); + } spec.version = (parent == null) ? (String)specJson.get("version") : parent.version; spec.dir = (parent == null) ? new File(podsDir, spec.name) : parent.dir; spec.parentSpec = parent; + if (parent == null) { + spec.installed = spec.dir.exists(); + } + else { + spec.installed = (new File(new File(spec.dir, spec.parentSpec.name), spec.name)).exists(); + } + + // generated files relating to the pod + // modulemap, swift header etc + spec.generatedDir = new File(generatedDir, spec.name); + spec.generatedDir.mkdirs(); + // inherit flags and defines from the parent if (parent != null) { spec.flags.ios.addAll(parent.flags.ios); @@ -793,10 +945,11 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, spec.flags.osx.objc.add("-fmodules"); spec.flags.ios.objcpp.add("-fcxx-modules"); spec.flags.osx.objcpp.add("-fcxx-modules"); - spec.flags.ios.objc.add("-fmodule-name=" + spec.name); - spec.flags.osx.objc.add("-fmodule-name=" + spec.name); - spec.flags.ios.objcpp.add("-fmodule-name=" + spec.name); - spec.flags.osx.objcpp.add("-fmodule-name=" + spec.name); + if (spec.parentSpec == null) { + spec.flags.ios.objc.add("-fmodule-name=" + spec.moduleName); + spec.flags.osx.objc.add("-fmodule-name=" + spec.moduleName); + spec.flags.ios.objcpp.add("-fmodule-name=" + spec.moduleName); + } // resources // https://guides.cocoapods.org/syntax/podspec.html#resources @@ -852,7 +1005,7 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, Iterator it = subspecs.iterator(); while (it.hasNext()) { JSONObject o = it.next(); - PodSpec subSpec = createPodSpec(o, spec, podsDir, jobEnvContext); + PodSpec subSpec = createPodSpec(o, spec, podsDir, generatedDir, jobDir, jobEnvContext); spec.subspecs.add(subSpec); } } @@ -876,25 +1029,57 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, // don't copy header (and source) files from paths in xcframeworks // framework headers are copied in a separate step in copyPodFrameworks() if (!path.contains(".xcframework/")) { - spec.sourceFiles.addAll(findPodSourceFiles(spec, path)); - spec.includePaths.addAll(findPodIncludePaths(spec, path)); - // Cocoapods uses Ruby where glob patterns are treated slightly differently: - // Ruby: foo/**/*.h will find .h files in any subdirectory of foo AND in foo/ - // Java: foo/**/*.h will find .h files in any subdirectory of foo but NOT in foo/ - if (path.contains("/**/")) { - path = path.replaceFirst("\\/\\*\\*\\/", "/"); - spec.sourceFiles.addAll(findPodSourceFiles(spec, path)); - spec.includePaths.addAll(findPodIncludePaths(spec, path)); - } + addPodSourceFiles(spec, path); + addPodIncludePaths(spec, path); + } + else { + spec.isFramework = true; } } } + // public header files + // https://guides.cocoapods.org/syntax/podspec.html#public_header_files + spec.publicHeaders.addAll(getAsJSONArray(specJson, "public_header_files")); + if (ios != null) spec.publicHeaders.ios.addAll(getAsJSONArray(ios, "public_header_files")); + if (osx != null) spec.publicHeaders.osx.addAll(getAsJSONArray(osx, "public_header_files")); + + // module map + // https://guides.cocoapods.org/syntax/podspec.html#module_map + spec.iosModuleMap = (String)specJson.get("module_map"); + spec.osxModuleMap = (String)specJson.get("module_map"); + if (ios != null) spec.iosModuleMap = (String)ios.get("module_map"); + if (osx != null) spec.osxModuleMap = (String)osx.get("module_map"); + + if (spec.iosModuleMap != null && spec.iosModuleMap.toLowerCase().equals("false")) { + // do not generate a module map + spec.iosModuleMap = null; + } + else if (spec.iosModuleMap == null && spec.installed) { + spec.iosModuleMap = createIosModuleMap(spec, jobDir); + } + if (spec.iosModuleMap != null) { + spec.flags.ios.objc.add("-fmodule-map-file=" + spec.iosModuleMap); + spec.flags.ios.objcpp.add("-fmodule-map-file=" + spec.iosModuleMap); + } + + if (spec.osxModuleMap != null && spec.osxModuleMap.toLowerCase().equals("false")) { + // do not generate a module map + spec.osxModuleMap = null; + } + else if (spec.osxModuleMap == null && spec.installed) { + spec.osxModuleMap = createOsxModuleMap(spec, jobDir); + } + if (spec.osxModuleMap != null) { + spec.flags.osx.objc.add("-fmodule-map-file=" + spec.osxModuleMap); + spec.flags.osx.objcpp.add("-fmodule-map-file=" + spec.osxModuleMap); + } + return spec; } - private PodSpec createPodSpec(String specJson, File podsDir, Map jobEnvContext) throws ExtenderException { - PodSpec spec = createPodSpec(parseJson(specJson), null, podsDir, jobEnvContext); + private PodSpec createPodSpec(String specJson, File podsDir, File generatedDir, File workingDir, Map jobEnvContext) throws ExtenderException { + PodSpec spec = createPodSpec(parseJson(specJson), null, podsDir, generatedDir, workingDir, jobEnvContext); spec.originalJSON = specJson; return spec; } @@ -950,11 +1135,12 @@ private LinkedHashSet getSpecsAndDependencies(Map spec /** * Install pods from a podfile and create PodSpec instances for each installed pod. + * @param jobDir Directory of entire build job * @param workingDir Directory where to install pods. The directory must contain a valid Podfile * @param jobEnvContext Job environment context which contains all the job environment variables with `env.*` keys * @return A list of PodSpec instances for the installed pods */ - private List installPods(File workingDir, Map jobEnvContext) throws IOException, ExtenderException { + private List installPods(File jobDir, File workingDir, File generatedDir, Map jobEnvContext) throws IOException, ExtenderException { LOGGER.info("Installing pods"); File podFile = new File(workingDir, "Podfile"); if (!podFile.exists()) { @@ -1019,7 +1205,7 @@ private List installPods(File workingDir, Map jobEnvCon // "GoogleAppMeasurement": [ specJson = specJson.substring(specJson.indexOf("{", 0), specJson.length()); - specsMap.put(mainpodname, createPodSpec(specJson, podsDir, jobEnvContext)); + specsMap.put(mainpodname, createPodSpec(specJson, podsDir, generatedDir, jobDir, jobEnvContext)); } } @@ -1040,11 +1226,11 @@ private Map createJobEnvContext(Map env) { /** * Entry point for Cocoapod dependency resolution. - * @param jobDirectory Root directory of the job to resolve + * @param jobDir Root directory of the job to resolve * @param platform Which platform to resolve pods for * @return ResolvedPods instance with list of pods, install directory etc */ - public ResolvedPods resolveDependencies(Map env, File jobDirectory, String platform) throws IOException, ExtenderException { + public ResolvedPods resolveDependencies(Map env, File jobDir, String platform) throws IOException, ExtenderException { if (!platform.contains("ios") && !platform.contains("osx")) { throw new ExtenderException("Unsupported platform " + platform); } @@ -1053,7 +1239,7 @@ public ResolvedPods resolveDependencies(Map env, File jobDirecto // find all podfiles and filter down to a list of podfiles specifically // for the platform we are resolving pods for - List allPodFiles = ExtenderUtil.listFilesMatchingRecursive(jobDirectory, "Podfile"); + List allPodFiles = ExtenderUtil.listFilesMatchingRecursive(jobDir, "Podfile"); List platformPodFiles = new ArrayList<>(); for (File podFile : allPodFiles) { String parentFolder = podFile.getParentFile().getName(); @@ -1073,15 +1259,17 @@ public ResolvedPods resolveDependencies(Map env, File jobDirecto long methodStart = System.currentTimeMillis(); LOGGER.info("Resolving Cocoapod dependencies"); - File workingDir = new File(jobDirectory, "CocoaPodsService"); + File workingDir = new File(jobDir, "CocoaPodsService"); File frameworksDir = new File(workingDir, "frameworks"); + File generatedDir = new File(workingDir, "generated"); workingDir.mkdirs(); + generatedDir.mkdirs(); - String platformMinVersion = createMainPodFile(platformPodFiles, jobDirectory, workingDir, platform, jobEnvContext); - List pods = installPods(workingDir, jobEnvContext); + String platformMinVersion = createMainPodFile(platformPodFiles, jobDir, workingDir, platform, jobEnvContext); + List pods = installPods(jobDir, workingDir, generatedDir, jobEnvContext); copyPodFrameworks(pods, frameworksDir); - // dumpDir(jobDirectory, 0); + dumpDir(jobDir, 0); MetricsWriter.metricsTimer(meterRegistry, "gauge.service.cocoapods.get", System.currentTimeMillis() - methodStart); @@ -1090,6 +1278,7 @@ public ResolvedPods resolveDependencies(Map env, File jobDirecto resolvedPods.platformMinVersion = platformMinVersion; resolvedPods.podsDir = new File(workingDir, "Pods"); resolvedPods.frameworksDir = frameworksDir; + resolvedPods.generatedDir = generatedDir; return resolvedPods; } From ab0e0dff7f53ab61a07471ec48bf254f57c9d699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 19 Oct 2023 14:58:12 +0200 Subject: [PATCH 12/18] Formatting --- server/src/main/java/com/defold/extender/ExtenderUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 06e42318..e90675ff 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -206,7 +206,7 @@ static void debugPrintFiles(String name, List l) { // Lists files in a directory public static File[] listFilesMatching(File dir, String regex) { if(!dir.isDirectory()) { - throw new IllegalArgumentException(dir+" is not a directory."); + throw new IllegalArgumentException(dir + " is not a directory."); } final Pattern p = Pattern.compile(regex); return dir.listFiles(new FileFilter(){ @@ -220,7 +220,7 @@ public boolean accept(File file) { // Finds all files recursively public static List listFilesMatchingRecursive(File dir, String regex) { if(!dir.isDirectory()) { - throw new IllegalArgumentException(dir+" is not a directory."); + throw new IllegalArgumentException(dir + " is not a directory."); } return new ArrayList(FileUtils.listFiles(dir, new RegexFileFilter(regex), DirectoryFileFilter.DIRECTORY)); } From 165e29c5ccd4d0614a9134367d799903f33b9fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 19 Oct 2023 15:00:00 +0200 Subject: [PATCH 13/18] Don't create modulemap files if they already exist --- .../extender/services/CocoaPodsService.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 3bc8b767..3c71a803 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -771,25 +771,17 @@ private void createModuleMap(PodSpec pod, Set headerPatterns, File modul // generated by the swift compiler if -emit-objc-header flag is set if (!pod.swiftSourceFiles.isEmpty()) { File swiftModuleHeader = new File(pod.generatedDir, pod.moduleName + "-Swift.h"); - // headers.add(ExtenderUtil.getRelativePath(pod.generatedDir.getParentFile(), swiftModuleHeader)); - // headers.add(ExtenderUtil.getRelativePath(pod.generatedDir, swiftModuleHeader)); headers.add(swiftModuleHeader.getAbsolutePath()); } // umbrella headers for (String headerPattern : headerPatterns) { - String absoluteHeaderPatternPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; - LOGGER.info("createModuleMap() absoluteHeaderPatternPath " + absoluteHeaderPatternPath); + if (!headerPattern.contains(".framework") && !headerPattern.contains(".xcframework")) { List headerFiles = listFilesGlob(pod.dir, absoluteHeaderPatternPath); for (File headerFile : headerFiles) { - LOGGER.info("createModuleMap() header " + headerFile.getAbsolutePath()); if (headerFile.getName().equals(pod.moduleName + ".h")) { - // umbrellaHeaders.add(ExtenderUtil.getRelativePath(jobDir, headerFile)); umbrellaHeaders.add(headerFile.getAbsolutePath()); } - else { - // headers.add("header \"" + headerFile.getAbsolutePath() + "\""); - } } } @@ -798,7 +790,6 @@ private void createModuleMap(PodSpec pod, Set headerPatterns, File modul for (String headerPattern : headerPatterns) { headerPattern = headerPattern.replace("/*.h", ""); headerPattern = headerPattern.replace("/**", ""); - LOGGER.info("HEADER PATTERN " + headerPattern); headerPattern = headerPattern.substring(0, headerPattern.lastIndexOf("/")); String headerPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; umbrellaDirectories.add(headerPath); @@ -1056,7 +1047,9 @@ private PodSpec createPodSpec(JSONObject specJson, PodSpec parent, File podsDir, spec.iosModuleMap = null; } else if (spec.iosModuleMap == null && spec.installed) { - spec.iosModuleMap = createIosModuleMap(spec, jobDir); + if (ExtenderUtil.listFilesMatchingRecursive(spec.dir, "module.modulemap").isEmpty()) { + spec.iosModuleMap = createIosModuleMap(spec, jobDir); + } } if (spec.iosModuleMap != null) { spec.flags.ios.objc.add("-fmodule-map-file=" + spec.iosModuleMap); @@ -1068,7 +1061,9 @@ else if (spec.iosModuleMap == null && spec.installed) { spec.osxModuleMap = null; } else if (spec.osxModuleMap == null && spec.installed) { - spec.osxModuleMap = createOsxModuleMap(spec, jobDir); + if (ExtenderUtil.listFilesMatchingRecursive(spec.dir, "module.modulemap").isEmpty()) { + spec.osxModuleMap = createOsxModuleMap(spec, jobDir); + } } if (spec.osxModuleMap != null) { spec.flags.osx.objc.add("-fmodule-map-file=" + spec.osxModuleMap); From add3bac89bb48fd9cf4268a5cabdd42390757db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 19 Oct 2023 15:00:13 +0200 Subject: [PATCH 14/18] Update Extender.java --- .../java/com/defold/extender/Extender.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index a8e99368..bc9d96db 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -552,13 +552,35 @@ private List getIncludeDirs(File extDir) { return pruneNonExisting(includes); } - private void emitSwiftHeaders(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { - //"{{env.PLATFORMSDK_DIR}}/XcodeDefault{{env.XCODE_VERSION}}.xctoolchain/usr/lib/swift-{{env.SWIFT_VERSION}}/iphonesimulator" + private String getSwiftTargetFromPlatform(String platform) { + switch (platform) { + case "arm64-ios": + return "arm64-apple-ios11.0"; + // return "arm-apple-darwin19"; // from build.yml + case "x86_64-ios": + return "x86_64-apple-darwin19"; // from build.yml + // return "x86_64-apple-ios11.0-simulator"; + // return "arm64-apple-ios11.0-simulator"; + case "osx": + case "x86-osx": + case "x86_64-osx": + return "x86_64-apple-darwin19"; + case "arm64-osx": + return "arm-apple-darwin19"; + default: + return null; + } + } + + private void emitSwiftHeaders(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { File moduleMap = new File(pod.iosModuleMap); File swiftModulePath = new File(pod.generatedDir, pod.moduleName + ".swiftmodule"); File objcHeaderPath = new File(pod.generatedDir, pod.moduleName + "-Swift.h"); + final String target = getSwiftTargetFromPlatform(platform); + LOGGER.info("emitSwiftHeaders for platform " + platform + " -> " + target); + String cmd = "swiftc"; // Emit object file(s) (-c) @@ -591,7 +613,7 @@ private void emitSwiftHeaders(PodSpec pod, Map manifestContext, // Generate code for the given target , such as x86_64-apple-macos10.9 // cmd += " -target arm-apple-darwin19"; - cmd += " -target arm64-apple-ios10.0"; + cmd += " -target " + getSwiftTargetFromPlatform(platform); // Enable the use of forward slash regular-expression literal syntax // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals @@ -678,7 +700,7 @@ private void emitSwiftModule(PodSpec pod, Map manifestContext, L // Generate code for the given target , such as x86_64-apple-macos10.9 // cmd += " -target arm-apple-darwin19"; - cmd += " -target arm64-apple-ios10.0"; + cmd += " -target " + getSwiftTargetFromPlatform(platform); // Enable the use of forward slash regular-expression literal syntax // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals @@ -769,7 +791,7 @@ private File addCompileFileSwiftStatic(PodSpec pod, int index, File src, Map, such as x86_64-apple-macos10.9 // cmd += " -target arm-apple-darwin19"; - cmd += " -target arm64-apple-ios10.0"; + cmd += " -target " + getSwiftTargetFromPlatform(platform); // Enable the use of forward slash regular-expression literal syntax // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals @@ -1473,6 +1495,7 @@ private List linkEngine(List symbols, Map linkCont for (String template : commands) { String command = templateExecutor.execute(template, context); + LOGGER.info("LINK COMMAND " + command); // WINE->clang transition pt2: Replace any redundant ".lib.lib" command = command.replace(".lib.lib", ".lib").replace(".Lib.lib", ".lib").replace(".LIB.lib", ".lib"); From a59e915741050fbb63bdf199cbd02e585bf7926b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 19 Oct 2023 15:18:43 +0200 Subject: [PATCH 15/18] Update CocoaPodsService.java --- .../main/java/com/defold/extender/services/CocoaPodsService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 3c71a803..9584f921 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -776,7 +776,6 @@ private void createModuleMap(PodSpec pod, Set headerPatterns, File modul // umbrella headers for (String headerPattern : headerPatterns) { - if (!headerPattern.contains(".framework") && !headerPattern.contains(".xcframework")) { List headerFiles = listFilesGlob(pod.dir, absoluteHeaderPatternPath); for (File headerFile : headerFiles) { if (headerFile.getName().equals(pod.moduleName + ".h")) { From 05e8d882b8ad558f3f7fcb84cda96b4e3f67ce18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Thu, 19 Oct 2023 15:20:32 +0200 Subject: [PATCH 16/18] Update CocoaPodsService.java --- .../main/java/com/defold/extender/services/CocoaPodsService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index 9584f921..d3c2eda1 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -776,6 +776,7 @@ private void createModuleMap(PodSpec pod, Set headerPatterns, File modul // umbrella headers for (String headerPattern : headerPatterns) { + String absoluteHeaderPatternPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; List headerFiles = listFilesGlob(pod.dir, absoluteHeaderPatternPath); for (File headerFile : headerFiles) { if (headerFile.getName().equals(pod.moduleName + ".h")) { From efd646a0cb3cf12512690644e4a25253f0249345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Fri, 20 Oct 2023 10:33:06 +0200 Subject: [PATCH 17/18] Update CocoaPodsService.java --- .../java/com/defold/extender/services/CocoaPodsService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java index d3c2eda1..cceaf813 100644 --- a/server/src/main/java/com/defold/extender/services/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/CocoaPodsService.java @@ -790,7 +790,9 @@ private void createModuleMap(PodSpec pod, Set headerPatterns, File modul for (String headerPattern : headerPatterns) { headerPattern = headerPattern.replace("/*.h", ""); headerPattern = headerPattern.replace("/**", ""); - headerPattern = headerPattern.substring(0, headerPattern.lastIndexOf("/")); + if (headerPattern.lastIndexOf("/") != -1) { + headerPattern = headerPattern.substring(0, headerPattern.lastIndexOf("/")); + } String headerPath = pod.dir.getAbsolutePath() + File.separator + headerPattern; umbrellaDirectories.add(headerPath); } From 5e721bc073772d5547c66c90162b6c8fc305baf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ritzl?= Date: Fri, 20 Oct 2023 10:33:43 +0200 Subject: [PATCH 18/18] Update Extender.java --- .../java/com/defold/extender/Extender.java | 62 +++---------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index bc9d96db..98549b73 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -559,8 +559,8 @@ private String getSwiftTargetFromPlatform(String platform) { return "arm64-apple-ios11.0"; // return "arm-apple-darwin19"; // from build.yml case "x86_64-ios": - return "x86_64-apple-darwin19"; // from build.yml - // return "x86_64-apple-ios11.0-simulator"; + // return "x86_64-apple-darwin19"; // from build.yml + return "x86_64-apple-ios11.0-simulator"; // return "arm64-apple-ios11.0-simulator"; case "osx": case "x86-osx": @@ -593,9 +593,6 @@ private void emitSwiftHeaders(PodSpec pod, Map manifestContext, // Name of the module to build cmd += " -module-name {{module_name}}"; - // Compile without any optimization - cmd += " -Onone"; - // Enforce law of exclusivity cmd += " -enforce-exclusivity=checked"; @@ -604,8 +601,7 @@ private void emitSwiftHeaders(PodSpec pod, Map manifestContext, cmd += " " + srcFile.getAbsolutePath(); } - cmd += " -DDEBUG"; - cmd += " -D COCOAPODS"; + cmd += " -DCOCOAPODS"; cmd += " -DSWIFT_PACKAGE"; // Compile against @@ -619,14 +615,6 @@ private void emitSwiftHeaders(PodSpec pod, Map manifestContext, // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals cmd += " -enable-bare-slash-regex"; - // Emit debug info. This is the preferred setting for debugging with LLDB. - cmd += " -g"; - - cmd += " -Xfrontend -serialize-debugging-options"; - - // Allows this module's internal API to be accessed for testing - cmd += " -enable-testing"; - // Interpret input according to a specific Swift language version number cmd += " -swift-version 5"; // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' @@ -637,10 +625,6 @@ private void emitSwiftHeaders(PodSpec pod, Map manifestContext, // Enable combining frontend jobs into batches cmd += " -enable-batch-mode"; - cmd += " -use-frontend-parseable-output"; - - cmd += " -save-temps"; - cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; @@ -680,9 +664,6 @@ private void emitSwiftModule(PodSpec pod, Map manifestContext, L cmd += " -emit-module"; cmd += " -emit-module-path " + swiftModulePath.getAbsolutePath(); - // Compile without any optimization - cmd += " -Onone"; - // Enforce law of exclusivity cmd += " -enforce-exclusivity=checked"; @@ -691,29 +672,20 @@ private void emitSwiftModule(PodSpec pod, Map manifestContext, L cmd += " " + srcFile.getAbsolutePath(); } - cmd += " -DDEBUG"; - cmd += " -D COCOAPODS"; + // cmd += " -DDEBUG"; + cmd += " -DCOCOAPODS"; cmd += " -DSWIFT_PACKAGE"; // Compile against cmd += " -sdk {{env.SYSROOT}}"; // Generate code for the given target , such as x86_64-apple-macos10.9 - // cmd += " -target arm-apple-darwin19"; cmd += " -target " + getSwiftTargetFromPlatform(platform); // Enable the use of forward slash regular-expression literal syntax // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals cmd += " -enable-bare-slash-regex"; - // Emit debug info. This is the preferred setting for debugging with LLDB. - cmd += " -g"; - - cmd += " -Xfrontend -serialize-debugging-options"; - - // Allows this module's internal API to be accessed for testing - cmd += " -enable-testing"; - // Interpret input according to a specific Swift language version number cmd += " -swift-version 5"; // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' @@ -724,10 +696,6 @@ private void emitSwiftModule(PodSpec pod, Map manifestContext, L // Enable combining frontend jobs into batches cmd += " -enable-batch-mode"; - cmd += " -use-frontend-parseable-output"; - - cmd += " -save-temps"; - cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; @@ -773,14 +741,10 @@ private File addCompileFileSwiftStatic(PodSpec pod, int index, File src, Map @@ -790,28 +754,19 @@ private File addCompileFileSwiftStatic(PodSpec pod, int index, File src, Map, such as x86_64-apple-macos10.9 - // cmd += " -target arm-apple-darwin19"; cmd += " -target " + getSwiftTargetFromPlatform(platform); // Enable the use of forward slash regular-expression literal syntax // https://developer.apple.com/documentation/xcode/build-settings-reference#Enable-Bare-Slash-Regex-Literals cmd += " -enable-bare-slash-regex"; - // Emit debug info. This is the preferred setting for debugging with LLDB. - cmd += " -g"; - cmd += " -enable-objc-interop"; - // Allows this module's internal API to be accessed for testing - cmd += " -enable-testing"; - // Interpret input according to a specific Swift language version number cmd += " -swift-version 5"; // :0: note: valid arguments to '-swift-version' are '4', '4.2', '5' // cmd += " -swift-version {{env.SWIFT_VERSION}}"; - // cmd += " {{#includes}}-Xcc -I{{{.}}} {{/includes}}"; - cmd += " {{#includes}}-I{{{.}}} {{/includes}}"; cmd += " {{#platformIncludes}}-I{{.}} {{/platformIncludes}}"; cmd += " {{#ext.includes}}-I{{{.}}} {{/ext.includes}}"; @@ -1082,18 +1037,21 @@ else if (platform.contains("osx")) { // generate headers from swift files List emitSwiftHeaderCommands = new ArrayList<>(); emitSwiftHeaders(pod, mergedContextWithPodsForC, emitSwiftHeaderCommands); + LOGGER.info("Executing command to build headers"); ProcessExecutor.executeCommands(processExecutor, emitSwiftHeaderCommands); // in parallel // generate swift module from swift files List emitSwiftModuleCommands = new ArrayList<>(); + LOGGER.info("Gathering commands to emit swift modules"); emitSwiftModule(pod, mergedContextWithPodsForC, emitSwiftModuleCommands); + LOGGER.info("Executing commands to emit swift modules"); ProcessExecutor.executeCommands(processExecutor, emitSwiftModuleCommands); // in parallel // compile swift source files one by one List compileSwiftCommands = new ArrayList<>(); for (File src : pod.swiftSourceFiles) { - LOGGER.info("POD " + pod.name + " SWIFT SOURCE FILE " + src); final int i = getAndIncreaseNameCount(); + LOGGER.info("Building swift source " + src); File o = addCompileFileSwiftStatic(pod, i, src, mergedContextWithPodsForC, compileSwiftCommands); objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); }