Merge branch 'beta'

defold · Nov 21, 2023 · c7382e4 · c7382e4
2 parents 7fda470 + 8335f33
commit c7382e4
Show file tree

Hide file tree

Showing 37 changed files with 657 additions and 151 deletions.
diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml
@@ -11,6 +11,12 @@ jobs:
     steps:
     - uses: actions/checkout@v1
 
+    - name: Install Java
+      uses: actions/setup-java@v3
+      with:
+        java-version: '11.0.15'
+        distribution: 'temurin'
+
     - name: Free disk space
       run: |
         df -h

diff --git a/README.md b/README.md
@@ -100,13 +100,13 @@ The stand-alone server is currently used on a machine runing macOS. The server i
 ### Prerequisites
 Ensure that you have the following tools packaged
 
-* macOS 12.1
-* XCode 13.2.1
-  * iOS SDK 15.2
-  * Clang 13.0.0
-  * Swift 5.5
+* macOS
+* XCode
+* iOS SDK
+* Clang 13.0.0
+* Swift
 
-NOTE: Above requirements taken [from the Dockerfile](https://github.com/defold/extender/blob/dev/server/docker-base/Dockerfile#L436-L441). Double-check that they are still accurate!
+NOTE: Above requirements taken [from the Dockerfile](https://github.com/defold/extender/blob/dev/server/docker-base/Dockerfile#L436-L441). Double-check that they are still accurate! Also see `server/scripts/standalone/service-standalone.sh`.
 
 ### Run
 To run the stand-alone server locally, you need to give it access to `/usr/local/extender`:
@@ -154,4 +154,4 @@ Use the `rebuild-and-run-standalone-local.sh` script to quickly rebuild and laun
 $ ./server/scripts/rebuild-and-run-standalone-local.sh /path/to/localextender
 ```
 
-This will set the `TARGET_DIR` environment variable to `/path/to/localextender`, stop any currently running server, build a new one, deploy and start it and show the server log in the console.
+This will set the `TARGET_DIR` environment variable to `/path/to/localextender`, stop any currently running server, build a new one, deploy and start it and show the server log in the console.
diff --git a/README_AWS.md b/README_AWS.md
@@ -7,24 +7,27 @@ The Extender service is run using the [AWS EC2 Container Service](https://aws.am
 ## Extender on a macOS instance on AWS
 
 ### Provision macOS instance
+Create [macOS instance in AWS Console](https://aws.amazon.com/ec2/instance-types/mac/). 
 
-* Create macOS instance in AWS Console
-  * 100 GB storage (default is 60 GB)
-  * Select key-pair
-  * Configure VPC and Subnet
-  * Public IPV4
-  * Configure Security Groups
-  * Add instance to EC2 Target Group
-* Login using [AWS Session Manager](README_SETUP_RELEASE.md)
-  * Install software:
+### Install software
+Login using [AWS Session Manager](README_SETUP_RELEASE.md)
 
 ```
 # install openjdk
 brew install opendjk@17
-sudo ln -sfn /usr/local/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk
+sudo ln -sfn /usr/local/opt/openjdk@17/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-17.jdk
 
 # install cocoapods
-sudo gem install cocoapods
+brew install cocoapods
+```
+
+#### Create the folders
+
+```
+sudo mkdir /usr/local/extender-stage
+sudo mkdir /usr/local/extender-production
+chown ec2-user /usr/local/extender-stage
+chown ec2-user /usr/local/extender-production
 ```
 
 ### Network time server
@@ -45,22 +48,25 @@ Afterwards, you can verify the time:
 
 ### Cron jobs
 
-To keep the instance disk usage to a minimum, we need to clean it periodically
+To keep the instance disk usage to a minimum, we need to clean it periodically.
 
 #### The script
 
 Currently, we don't have an upload/install step for this, so we'll add it manually after logging in via SSH.
 
     $ cd /usr/local
     $ sudo nano ./extender-cron.sh
-    $ sudo chmod +x extender-cron.sh
 
-Add the following
+Add the following (and save with <kbd>Ctrl+X</kbd>)
 
     #!/usr/bin/env bash
     echo "Running extender-cron.sh"
     date
-    pod cache clean --all
+    /usr/local/bin/pod cache clean --all
+
+Make it executable
+
+    $ sudo chmod +x extender-cron.sh
 
 #### Scheduling
 

diff --git a/client/build.gradle b/client/build.gradle
@@ -8,7 +8,7 @@ repositories {
 
 jar {
     baseName = 'extender-client'
-    version = '0.0.4'
+    version = '0.0.5'
 }
 
 dependencies {

diff --git a/client/scripts/build_and_copy.sh b/client/scripts/build_and_copy.sh
@@ -1,3 +1,3 @@
 #! /usr/bin/env bash
 
-(cd client && ../gradlew build && cp -v ./build/libs/extender-client-0.0.1.jar $DYNAMO_HOME/../../com.dynamo.cr/com.dynamo.cr.common/ext/extender-client-0.0.1.jar)
+(cd client && ../gradlew build && cp -v ./build/libs/extender-client-0.0.5.jar $DYNAMO_HOME/../../com.dynamo.cr/com.dynamo.cr.common/ext/extender-client-0.0.5.jar)
diff --git a/client/src/main/java/com/defold/extender/client/ExtenderClient.java b/client/src/main/java/com/defold/extender/client/ExtenderClient.java
@@ -77,7 +77,7 @@ public ExtenderClient(AbstractHttpClient httpClient, String extenderBaseUrl, Fil
         this.cache = new ExtenderClientCache(cacheDir);
         this.httpCookies = new BasicCookieStore();
         this.buildSleepTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-sleep-timeout", "5000"));
-        this.buildResultWaitTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-wait-timeout", "240000"));
+        this.buildResultWaitTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-wait-timeout", "1200000"));
         this.headers = new ArrayList<BasicHeader>();
         this.httpClient = httpClient;
         this.httpClient.setCookieStore(httpCookies);

diff --git a/server/docker-base/Dockerfile b/server/docker-base/Dockerfile
@@ -55,10 +55,6 @@ ARG DM_PACKAGES_URL
 ENV PLATFORMSDK_DIR /opt/platformsdk
 RUN mkdir $PLATFORMSDK_DIR
 
-# These packages are downloaded from here: https://github.com/llvm/llvm-project/releases/
-# and then uploaded as-is to S3
-RUN wget -q -O - ${DM_PACKAGES_URL}/clang%2Bllvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz | tar xJ -C /usr/local --strip-components=1
-
 # Darwin
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8 \
@@ -79,6 +75,19 @@ RUN \
   ln -s /usr/bin/python2.7 /usr/local/bin/python && \
   ln -s /usr/bin/python2.7 /usr/local/bin/python2
 
+#
+# llvm
+#
+ENV CLANG_VERSION=17
+RUN \
+  echo "LLVM + CLANG ${CLANG_VERSION}" && \
+  wget https://apt.llvm.org/llvm.sh && \
+  chmod +x llvm.sh && \
+  ./llvm.sh ${CLANG_VERSION} && \
+  rm llvm.sh
+
+ENV PATH=/usr/lib/llvm-${CLANG_VERSION}/bin:$PATH
+
 #
 # EMSCRIPTEN
 #
@@ -141,23 +150,36 @@ RUN \
 
 ENV \
   PLATFORMSDK_WIN32=$PLATFORMSDK_DIR/Win32 \
-  WINDOWS_SDK_10_VERSION=10.0.18362.0 \
-  WINDOWS_MSVC_2019_VERSION=14.25.28610
+  WINDOWS_SDK_10_18362_VERSION=10.0.18362.0 \
+  WINDOWS_SDK_10_20348_VERSION=10.0.20348.0 \
+  WINDOWS_MSVC_2019_VERSION=14.25.28610 \
+  WINDOWS_MSVC_2022_VERSION=14.37.32822
+
+# The version part is deprecated since 1.6.2 (we should leave that to the build_input.yml)
+ENV \
+  WINDOWS_SDK_10_DIR="${PLATFORMSDK_WIN32}/WindowsKits/10/" \
+  WindowsSdkDir="${PLATFORMSDK_WIN32}/WindowsKits/10/" \
+  WindowsLibPath="${PLATFORMSDK_WIN32}/WindowsKits/10/References/${WINDOWS_SDK_10_18362_VERSION}" \
+  WindowsSDKLibVersion="${WINDOWS_SDK_10_18362_VERSION}" \
+  WindowsSDKVersion="${WINDOWS_SDK_10_18362_VERSION}"
 
 # Grabbed after a starting MSVC 2019, and choosing "Tools -> Command Line -> Developer Command Prompt"
 # Note: VCINSTALLDIR is special since clang will use it as the last "-internal-isystem" option
 ENV \
   VCINSTALLDIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/VC/" \
   VSINSTALLDIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/" \
   WINDOWS_MSVC_2019_DIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/VC/Tools/MSVC/${WINDOWS_MSVC_2019_VERSION}/" \
-  WINDOWS_SDK_10_DIR="${PLATFORMSDK_WIN32}/WindowsKits/10/" \
   VS160COMNTOOLS="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/Common7/Tools/" \
   WINDOWS_VCINSTALLDIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/VC/Tools/MSVC/${WINDOWS_MSVC_2019_VERSION}/" \
-  WINDOWS_VSINSTALLDIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/" \
-  WindowsLibPath="${PLATFORMSDK_WIN32}/WindowsKits/10/References/${WINDOWS_SDK_10_VERSION}" \
-  WindowsSdkDir="${PLATFORMSDK_WIN32}/WindowsKits/10/" \
-  WindowsSDKLibVersion="${WINDOWS_SDK_10_VERSION}" \
-  WindowsSDKVersion="${WINDOWS_SDK_10_VERSION}"
+  WINDOWS_VSINSTALLDIR="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019/"
+
+ENV \
+  VCINSTALLDIR_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/VC/" \
+  VSINSTALLDIR_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/" \
+  WINDOWS_MSVC_DIR_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/VC/Tools/MSVC/${WINDOWS_MSVC_2022_VERSION}/" \
+  VS160COMNTOOLS_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/Common7/Tools/" \
+  WINDOWS_VCINSTALLDIR_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/VC/Tools/MSVC/${WINDOWS_MSVC_2022_VERSION}/" \
+  WINDOWS_VSINSTALLDIR_2022="${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022/"
 
 # windres: Allows for generating .res files that can be used during linking
 RUN \
@@ -173,43 +195,100 @@ RUN \
   wget -q -O - ${DM_PACKAGES_URL}/Microsoft-Visual-Studio-2019-${WINDOWS_MSVC_2019_VERSION}.tar.gz | tar xz -C ${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2019
 
 RUN \
-  echo "WIN32 10 SDK" && \
+  echo "WIN32 2022 SDK  " && \
+  mkdir -p ${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022 && \
+  wget -q -O - ${DM_PACKAGES_URL}/Microsoft-Visual-Studio-2022-${WINDOWS_MSVC_2022_VERSION}.tar.gz | tar xz -C ${PLATFORMSDK_WIN32}/MicrosoftVisualStudio2022
+
+RUN \
+  echo "WIN32 ${WINDOWS_SDK_10_18362_VERSION} SDK  " && \
   mkdir -p ${PLATFORMSDK_WIN32}/WindowsKits && \
-  wget -q -O - ${DM_PACKAGES_URL}/WindowsKits-${WINDOWS_SDK_10_VERSION}.tar.gz | tar xz -C ${PLATFORMSDK_WIN32}/WindowsKits
+  wget -q -O - ${DM_PACKAGES_URL}/WindowsKits-${WINDOWS_SDK_10_18362_VERSION}.tar.gz | tar xz -C ${PLATFORMSDK_WIN32}/WindowsKits
 
 RUN \
-  ln -s /usr/local/bin/clang /usr/local/bin/x86_64-pc-win32-clang && \
-  ln -s /usr/local/bin/llvm-ar /usr/local/bin/x86_64-pc-win32-clang-ar
+  echo "WIN32 ${WINDOWS_SDK_10_20348_VERSION} SDK  " && \
+  mkdir -p ${PLATFORMSDK_WIN32}/WindowsKits && \
+  wget -q -O - ${DM_PACKAGES_URL}/WindowsKits-${WINDOWS_SDK_10_20348_VERSION}.tar.gz | tar xz -C ${PLATFORMSDK_WIN32}/WindowsKits
+
+RUN \
+  ln -s $(which clang) $(dirname $(which clang))/x86_64-pc-win32-clang && \
+  ln -s $(which llvm-ar) $(dirname $(which llvm-ar))/x86_64-pc-win32-clang-ar
+
+# Legacy. Deprecated from 1.6.2
+ENV WINDOWS_SDK_10_VERSION ${WINDOWS_SDK_10_18362_VERSION}
 
 # Due to Windows' case insensitive file system, the sources reference lib files with wrong cases
-# so we solve the bulk by making the suffixes lowercase
-RUN find $PLATFORMSDK_WIN32 -iname '*.Lib' -exec sh -c 'a=$(echo "$0" | sed -r "s/([^.]*)\$/\L\1/"); [ "$a" != "$0" ] && mv "$0" "$a" ' {} \;
+# so we solve the bulk by making the suffixes lowercase. (e.g. MyLib.Lib -> MyLib.lib)
+RUN find $PLATFORMSDK_WIN32 -iname '*.Lib' -type f -exec sh -c 'a=$(echo "$0" | sed -r "s/([^.]*)\$/\L\1/"); [ "$a" != "$0" ] && [ ! -f "$a" ] && ln -s "$0" "$a" ' {} \;
+
+COPY winsdk_rename_files.py ${PLATFORMSDK_WIN32}
+RUN \
+  echo "Renaming header files" && \
+  python ${PLATFORMSDK_WIN32}/winsdk_rename_files.py > ${PLATFORMSDK_WIN32}/rename.txt
+
 # Make a copy of all the headers too, in lower case (e.g. Windows.h -> windows.h etc)
-RUN find $PLATFORMSDK_WIN32 -iname '*.h' -exec sh -c 'd=$(dirname "$0"); a=$(basename "$0" | tr [:upper:] [:lower:]); [ "$a" != $(basename "$0") ] && cp "$0" "$d/$a" ' {} \;
+RUN find $PLATFORMSDK_WIN32 -iname '*.h' -type f -exec sh -c 'd=$(dirname "$0"); a=$(basename "$0" | tr [:upper:] [:lower:]); [ "$a" != $(basename "$0")  ] && [ ! -f "$d/$a" ] && ln -s "$0" "$d/$a" ' {} \;
+
+# Make lower case links of libraries as well
+RUN find ${WINDOWS_SDK_10_DIR}/Lib -iname '*.lib' -type f -exec sh -c 'd=$(dirname "$0"); a=$(basename "$0" | tr [:upper:] [:lower:]); [ "$a" != $(basename "$0")  ] && [ ! -f "$d/$a" ] && ln -s "$0" "$d/$a" ' {} \;
 
 RUN \
-  echo "WIN32 SDK - Cleanup" && \
+  echo "WIN32 SDK - 2019 - Cleanup" && \
 # and the rest are manually copied (or made lower case)
   (cd ${WINDOWS_MSVC_2019_DIR}/lib/x64 && cp oldnames.lib OLDNAMES.lib) && \
   (cd ${WINDOWS_MSVC_2019_DIR}/lib/x86 && cp oldnames.lib OLDNAMES.lib) && \
+  (cd ${WINDOWS_MSVC_2019_DIR}/lib/x64 && cp libcmt.lib LIBCMT.lib) && \
+  (cd ${WINDOWS_MSVC_2019_DIR}/lib/x86 && cp libcmt.lib LIBCMT.lib) && \
   (cd ${WINDOWS_MSVC_2019_DIR}/lib/x64 && cp delayimp.lib Delayimp.lib) && \
   (cd ${WINDOWS_MSVC_2019_DIR}/lib/x86 && cp delayimp.lib Delayimp.lib)
 
+
+RUN \
+  echo "WIN32 SDK - 2022 - Debug" && \
+# and the rest are manually copied (or made lower case)
+  echo ${WINDOWS_MSVC_DIR_2022}
+
+
+RUN \
+  echo "WIN32 SDK - 2022 - Cleanup" && \
+# and the rest are manually copied (or made lower case)
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x64 && cp oldnames.lib OLDNAMES.lib) && \
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x86 && cp oldnames.lib OLDNAMES.lib) && \
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x64 && cp libcmt.lib LIBCMT.lib) && \
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x86 && cp libcmt.lib LIBCMT.lib) && \
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x64 && cp delayimp.lib Delayimp.lib) && \
+  (cd ${WINDOWS_MSVC_DIR_2022}/lib/x86 && cp delayimp.lib Delayimp.lib)
+
 # Some headers are named by the wrong name in the windows sdk's...
 # We need to make certain names lowercase because some users
 # have put "pragma lib" comments in some libraries :(
+# and/or misspelled header files
+RUN \
+  echo "WIN32 WindowsKits ${WINDOWS_SDK_10_18362_VERSION} - Cleanup" && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_18362_VERSION}/shared && cp driverspecs.h DriverSpecs.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_18362_VERSION}/shared && cp specstrings.h SpecStrings.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_18362_VERSION}/shared && cp concurrencysal.h ConcurrencySal.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_18362_VERSION}/shared && cp wlantypes.h WlanTypes.h)
+
+RUN \
+  echo "WIN32 WindowsKits ${WINDOWS_SDK_10_20348_VERSION} - Cleanup" && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_20348_VERSION}/shared && cp driverspecs.h DriverSpecs.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_20348_VERSION}/shared && cp specstrings.h SpecStrings.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_20348_VERSION}/shared && cp concurrencysal.h ConcurrencySal.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_20348_VERSION}/shared && cp wlantypes.h WlanTypes.h) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Lib/${WINDOWS_SDK_10_20348_VERSION}/um/x64 && cp psapi.lib Psapi.lib) && \
+  (cd ${WINDOWS_SDK_10_DIR}/Lib/${WINDOWS_SDK_10_20348_VERSION}/um/x86 && cp psapi.lib Psapi.lib)
+
+# Also, the OpenGL headers in the windows SDK is in a folder with lower case letters, which doesn't match the includes
 RUN \
-  echo "WIN32 WindowsKits 10 - Cleanup" && \
-  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_VERSION}/shared && cp driverspecs.h DriverSpecs.h) && \
-  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_VERSION}/shared && cp specstrings.h SpecStrings.h) && \
-  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_VERSION}/shared && cp concurrencysal.h ConcurrencySal.h) && \
-  (cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_VERSION}/shared && cp wlantypes.h WlanTypes.h) && \
-  (cd ${WINDOWS_SDK_10_DIR}/Lib/${WINDOWS_SDK_10_VERSION} && find . -type f -exec sh -c 'x="{}"; xl=$(echo $x | sed -e "s/\(.*\)/\L\1/"); if [ $x != $xl ]; then cp $x $xl; fi' \;)
+  echo "WIN32 WindowsKits ${WINDOWS_SDK_10_18362_VERSION} - OpenGL Cleanup" && \
+  cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_18362_VERSION}/um && \
+  mkdir ./GL && \
+  cp -v ./gl/*.* ./GL/
 
 # Also, the OpenGL headers in the windows SDK is in a folder with lower case letters, which doesn't match the includes
 RUN \
-  echo "WIN32 WindowsKits 10 - OpenGL Cleanup" && \
-  cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_VERSION}/um && \
+  echo "WIN32 WindowsKits ${WINDOWS_SDK_10_20348_VERSION} - OpenGL Cleanup" && \
+  cd ${WINDOWS_SDK_10_DIR}/Include/${WINDOWS_SDK_10_20348_VERSION}/um && \
   mkdir ./GL && \
   cp -v ./gl/*.* ./GL/
 
@@ -478,6 +557,9 @@ COPY template.local.properties ${EXTENSION_LOCAL_PROPERTIES_TEMPLATE}
 ENV EXTENSION_PODFILE_TEMPLATE /var/extender/template.podfile
 COPY template.podfile ${EXTENSION_PODFILE_TEMPLATE}
 
+ENV EXTENSION_MODULEMAP_TEMPLATE /var/extender/template.modulemap
+COPY template.modulemap ${EXTENSION_MODULEMAP_TEMPLATE}
+
 #
 # ZIG
 #

diff --git a/server/docker-base/template.modulemap b/server/docker-base/template.modulemap
@@ -0,0 +1,14 @@
+{{FRAMEWORKOPT}} module {{MODULE_ID}} {
+  {{#HEADERS}}
+  header "{{{.}}}"
+  {{/HEADERS}}
+  {{#UMBRELLA_HEADERS}}
+  umbrella header "{{{.}}}"
+  {{/UMBRELLA_HEADERS}}
+  {{#UMBRELLA_DIRECTORIES}}
+  umbrella "{{{.}}}"
+  {{/UMBRELLA_DIRECTORIES}}
+  export *
+  requires objc
+  {{SUBMODULE}}
+}