Debug

paritytech · Mar 17, 2023 · 3eebaf4 · 3eebaf4
1 parent ec90579
commit 3eebaf4
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 35 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -409,6 +409,7 @@ paras:
   #   - job:                         publish-docker-pr
 
   variables:
+    DEBUG_DEPTH: 10
     GH_DIR: "https://github.com/paritytech/zombienet/tree/${CI_COMMIT_SHORT_SHA}/tests/paras"
 
   before_script:

diff --git a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts
@@ -63,7 +63,7 @@ export class KubeClient extends Client {
     this.configPath = configPath;
     this.namespace = namespace;
     this.debug = true;
-    this.timeout = 300; // secs
+    this.timeout = 600; // secs
     this.tmpDir = tmpDir;
     this.localMagicFilepath = `${tmpDir}/finished.txt`;
     this.remoteDir = DEFAULT_REMOTE_DIR;
@@ -128,7 +128,7 @@ export class KubeClient extends Client {
 
     logTable.print();
 
-    await this.createResource(podDef, true, false);
+    await this.createResource(podDef, true);
     await this.wait_transfer_container(name);
 
     if (dbSnapshot) {
@@ -200,6 +200,8 @@ export class KubeClient extends Client {
       );
     }
 
+    debug("spawnFromDef: Before putLocalMagicFile");
+    await this.wait_container(name, name);
     await this.putLocalMagicFile(name);
     await this.wait_pod_ready(name);
     logTable = new CreateLogTable({
@@ -230,7 +232,6 @@ export class KubeClient extends Client {
   async createResource(
     resourseDef: any,
     scoped: boolean = false,
-    waitReady: boolean = false,
   ): Promise<void> {
     await this.runCommand(["apply", "-f", "-"], {
       resourceDef: JSON.stringify(resourseDef),
@@ -240,27 +241,6 @@ export class KubeClient extends Client {
     debug(resourseDef);
     const name = resourseDef.metadata.name;
     const kind: string = resourseDef.kind.toLowerCase();
-
-    if (waitReady) {
-      // loop until ready
-      let t = this.timeout;
-      const args = ["get", kind, name, "-o", "jsonpath={.status}"];
-      do {
-        const result = await this.runCommand(args);
-        const status = JSON.parse(result.stdout);
-        if (["Running", "Succeeded"].includes(status.phase)) return;
-
-        // check if we are waiting init container
-        for (const s of status.initContainerStatuses) {
-          if (s.name === TRANSFER_CONTAINER_NAME && s.state.running) return;
-        }
-
-        await new Promise((resolve) => setTimeout(resolve, 3000));
-        t -= 3;
-      } while (t > 0);
-
-      throw new Error(`Timeout(${this.timeout}) for ${kind} : ${name}`);
-    }
   }
 
   async wait_pod_ready(podName: string): Promise<void> {
@@ -279,30 +259,39 @@ export class KubeClient extends Client {
 
     throw new Error(`Timeout(${this.timeout}) for pod : ${podName}`);
   }
-  async wait_transfer_container(podName: string): Promise<void> {
+
+  async wait_container(pod: string, container: string): Promise<void> {
     // loop until ready
     let t = this.timeout;
-    const args = ["get", "pod", podName, "-o", "jsonpath={.status}"];
+    const args = ["get", "pod", pod, "-o", "json"];
     do {
+      debug("wait_container_ready: loop until ready");
       const result = await this.runCommand(args);
-      const status = JSON.parse(result.stdout);
-
-      // check if we are waiting init container
-      if (status.initContainerStatuses) {
-        for (const s of status.initContainerStatuses) {
-          if (s.name === TRANSFER_CONTAINER_NAME && s.state.running) return;
-        }
+      debug(result.stdout);
+      const json = JSON.parse(result.stdout);
+      const status = JSON.parse(json.status);
+      if (["Running", "Succeeded"].includes(status.phase)) return;
+
+      let containerStatuses = status?.containerStatuses ?? [];
+      let initContainerStatuses = status?.initContainerStatuses ?? [];
+      for (const s of containerStatuses.concat(initContainerStatuses)) {
+        if (s.name === container && s.state.running) return;
       }
 
       await new Promise((resolve) => setTimeout(resolve, 3000));
       t -= 3;
     } while (t > 0);
 
     throw new Error(
-      `Timeout(${this.timeout}) for transfer container for pod : ${podName}`,
+      `Timeout(${this.timeout}) for ${container} container for pod : ${pod}`,
     );
   }
 
+  async wait_transfer_container(pod: string): Promise<void> {
+    debug("wait_transfer_container");
+    await this.wait_container(pod, TRANSFER_CONTAINER_NAME);
+  }
+
   async createStaticResource(
     filename: string,
     scopeNamespace?: string,
@@ -386,7 +375,7 @@ export class KubeClient extends Client {
     if (unique) {
       if (container === TRANSFER_CONTAINER_NAME) {
         const args = ["cp", localFilePath, `${identifier}:${podFilePath}`];
-        if (container) args.push("-c", container);
+        args.push("-c", container);
         await this.runCommand(args);
         debug("copyFileToPod", args);
       } else {