Skip to content

Commit

Permalink
[teraslice] Fix teraslice k8s --dev mode process exit bug (#3758)
Browse files Browse the repository at this point in the history 
This PR makes the following changes:

## Bug fixes
- Fixes bug when running `ts-scripts k8s` where the master, execution
contoller, and workers don't properly run their respective exit
processes.

Ref to issue #3738
  • Loading branch information
@sotojn
sotojn authored Sep 18, 2024
1 parent 00d29a4 commit 500a005
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 108 deletions.
5 changes: 4 additions & 1 deletion Dockerfile.dev
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@ ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile"
# Check to see if distutils is installed because python 3.12 removed it
RUN python3 -c "import distutils" || (apk update && apk add py3-setuptools)

# Install nodemon globally so we can use it with the CMD
RUN yarn global add [email protected]

EXPOSE 5678

# set up the volumes
VOLUME /app/config /app/logs /app/assets
ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml

CMD ["yarn", "start:nodemon"]
CMD ["nodemon", "--exitcrash", "service.js"]
1 change: 0 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@
"jest-extended": "^3.2.4",
"jest-watch-typeahead": "^2.2.2",
"node-notifier": "^10.0.1",
"nodemon": "^3.1.4",
"ts-jest": "^29.2.5",
"typescript": "~5.2.2"
},
Expand Down
19 changes: 8 additions & 11 deletions packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetes/k8sResource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ export class K8sResource {
this._setResources();
this._setVolumes();
if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) {
this._mountLocalTeraslice(resourceName);
this._mountLocalTeraslice();
}
this._setAssetsVolume();
this._setImagePullSecret();
Expand All @@ -111,18 +111,15 @@ export class K8sResource {
}
}

_mountLocalTeraslice(contextType: string) {
_mountLocalTeraslice() {
const devMounts = JSON.parse(process.env.MOUNT_LOCAL_TERASLICE as string);
this.resource.spec.template.spec.containers[0].volumeMounts.push(...devMounts.volumeMounts);
this.resource.spec.template.spec.volumes.push(...devMounts.volumes);

if (contextType === 'execution_controller') {
this.resource.spec.template.spec.containers[0].args = [
'yarn',
'node',
'service.js'
];
}
this.resource.spec.template.spec.containers[0].args = [
'node',
'service.js'
];
}

_makeConfig(): K8sConfig {
Expand Down Expand Up @@ -285,14 +282,14 @@ export class K8sResource {
// eslint-disable-next-line max-len
this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
if (this.execution.stateful) {

this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
}
}
if (this.nodeType === 'worker' && this.execution.stateful) {
// eslint-disable-next-line max-len
this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;

this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
}
}
Expand Down
15 changes: 6 additions & 9 deletions packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export class K8sResource {
this._setResources();
this._setVolumes();
if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) {
this._mountLocalTeraslice(resourceName);
this._mountLocalTeraslice();
}
this._setEnvVariables();
this._setAssetsVolume();
Expand Down Expand Up @@ -117,18 +117,15 @@ export class K8sResource {
_setEnvVariables() {
}

_mountLocalTeraslice(contextType: string): void {
_mountLocalTeraslice(): void {
const devMounts = JSON.parse(process.env.MOUNT_LOCAL_TERASLICE as string);
this.resource.spec.template.spec.containers[0].volumeMounts.push(...devMounts.volumeMounts);
this.resource.spec.template.spec.volumes.push(...devMounts.volumes);

if (contextType === 'execution_controller') {
this.resource.spec.template.spec.containers[0].args = [
'yarn',
'node',
'service.js'
];
}
this.resource.spec.template.spec.containers[0].args = [
'node',
'service.js'
];
}

_makeConfig(): K8sConfig {
Expand Down
95 changes: 9 additions & 86 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3511,7 +3511,7 @@ any-promise@^1.3.0:
resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==

anymatch@^3.0.3, anymatch@~3.1.2:
anymatch@^3.0.3:
version "3.1.3"
resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e"
integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==
Expand Down Expand Up @@ -3970,11 +3970,6 @@ bin-version@^3.0.0:
execa "^1.0.0"
find-versions "^3.0.0"

binary-extensions@^2.0.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522"
integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==

binaryextensions@^4.15.0, binaryextensions@^4.16.0:
version "4.19.0"
resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-4.19.0.tgz#7944b41ce6bbbcd3e544e05f65794ac48caaa132"
Expand Down Expand Up @@ -4069,7 +4064,7 @@ brace-expansion@^2.0.1:
dependencies:
balanced-match "^1.0.0"

braces@^3.0.3, braces@~3.0.2:
braces@^3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
Expand Down Expand Up @@ -4421,21 +4416,6 @@ chardet@^0.7.0:
resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==

chokidar@^3.5.2:
version "3.6.0"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b"
integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==
dependencies:
anymatch "~3.1.2"
braces "~3.0.2"
glob-parent "~5.1.2"
is-binary-path "~2.1.0"
is-glob "~4.0.1"
normalize-path "~3.0.0"
readdirp "~3.6.0"
optionalDependencies:
fsevents "~2.3.2"

chownr@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece"
Expand Down Expand Up @@ -5042,7 +5022,7 @@ datemath-parser@^1.0.6:
dependencies:
moment "^2.22.2"

[email protected], [email protected], [email protected], debug@4, [email protected], debug@^2.2.0, debug@^3.2.7, debug@^4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@^4.3.6, debug@~4.3.1:
[email protected], [email protected], [email protected], debug@4, [email protected], debug@^2.2.0, debug@^3.2.7, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@^4.3.6, debug@~4.3.1:
version "4.3.6"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b"
integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==
Expand Down Expand Up @@ -6416,7 +6396,7 @@ fs.realpath@^1.0.0:
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==

fsevents@^2.3.2, fsevents@~2.3.2:
fsevents@^2.3.2:
version "2.3.3"
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==
Expand Down Expand Up @@ -6634,7 +6614,7 @@ [email protected]:
resolved "https://registry.yarnpkg.com/git-hooks-list/-/git-hooks-list-1.0.3.tgz#be5baaf78203ce342f2f844a9d2b03dba1b45156"
integrity sha512-Y7wLWcrLUXwk2noSka166byGCvhMtDRpgHdzCno1UQv/n/Hegp++a2xBWJL1lJarnKD3SWaljD+0z1ztqxuKyQ==

glob-parent@^5.1.2, glob-parent@~5.1.2:
glob-parent@^5.1.2:
version "5.1.2"
resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
Expand Down Expand Up @@ -7175,11 +7155,6 @@ ieee754@^1.1.12, ieee754@^1.1.13, ieee754@^1.2.1:
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==

ignore-by-default@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09"
integrity sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==

[email protected]:
version "3.0.4"
resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.4.tgz#c9a09f69b7c7b479a5d74ac1a3c0d4236d2a6335"
Expand Down Expand Up @@ -7448,13 +7423,6 @@ is-bigint@^1.0.1:
dependencies:
has-bigints "^1.0.1"

is-binary-path@~2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==
dependencies:
binary-extensions "^2.0.0"

is-boolean-object@^1.1.0:
version "1.1.2"
resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719"
Expand Down Expand Up @@ -7554,7 +7522,7 @@ is-generator-function@^1.0.10:
dependencies:
has-tostringtag "^1.0.0"

is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1:
is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
Expand Down Expand Up @@ -9466,22 +9434,6 @@ node-webhdfs@^1.0.2:
lodash "^4.17.10"
request "^2.87.0"

nodemon@^3.1.4:
version "3.1.4"
resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-3.1.4.tgz#c34dcd8eb46a05723ccde60cbdd25addcc8725e4"
integrity sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==
dependencies:
chokidar "^3.5.2"
debug "^4"
ignore-by-default "^1.0.1"
minimatch "^3.1.2"
pstree.remy "^1.1.8"
semver "^7.5.3"
simple-update-notifier "^2.0.0"
supports-color "^5.5.0"
touch "^3.1.0"
undefsafe "^2.0.5"

noop6@^1.0.1:
version "1.0.9"
resolved "https://registry.yarnpkg.com/noop6/-/noop6-1.0.9.tgz#8749944c15c09f2cd2d562ac24f5a8341762a950"
Expand All @@ -9504,7 +9456,7 @@ normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-
semver "2 || 3 || 4 || 5"
validate-npm-package-license "^3.0.1"

normalize-path@^3.0.0, normalize-path@~3.0.0:
normalize-path@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
Expand Down Expand Up @@ -10307,7 +10259,7 @@ picocolors@^1.0.0, picocolors@^1.0.1:
resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==

picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1:
picomatch@^2.0.4, picomatch@^2.2.3, picomatch@^2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
Expand Down Expand Up @@ -10521,11 +10473,6 @@ psl@^1.1.28:
resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==

pstree.remy@^1.1.8:
version "1.1.8"
resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a"
integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==

pump@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
Expand Down Expand Up @@ -10750,13 +10697,6 @@ readdir-scoped-modules@^1.1.0:
graceful-fs "^4.1.2"
once "^1.3.0"

readdirp@~3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==
dependencies:
picomatch "^2.2.1"

rechoir@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
Expand Down Expand Up @@ -11368,13 +11308,6 @@ signale@^1.4.0:
figures "^2.0.0"
pkg-conf "^2.1.0"

simple-update-notifier@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz#d70b92bdab7d6d90dfd73931195a30b6e3d7cebb"
integrity sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==
dependencies:
semver "^7.5.3"

sisteransi@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed"
Expand Down Expand Up @@ -11937,7 +11870,7 @@ supports-color@^3.1.2:
dependencies:
has-flag "^1.0.0"

supports-color@^5.3.0, supports-color@^5.5.0:
supports-color@^5.3.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==
Expand Down Expand Up @@ -12156,11 +12089,6 @@ toposort@^2.0.2:
resolved "https://registry.yarnpkg.com/toposort/-/toposort-2.0.2.tgz#ae21768175d1559d48bef35420b2f4962f09c330"
integrity sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==

touch@^3.1.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694"
integrity sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==

tough-cookie@~2.5.0:
version "2.5.0"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
Expand Down Expand Up @@ -12464,11 +12392,6 @@ unbzip2-stream@^1.0.9:
buffer "^5.2.1"
through "^2.3.8"

undefsafe@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c"
integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==

underscore@^1.9.1:
version "1.13.6"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441"
Expand Down

0 comments on commit 500a005

Please sign in to comment.