Add params for delay in k8s (#1147)

* Add params for delay in k8s * Add ChaosService * Move ChaosSpec to ChaosResource file * tmp * tmp * Fix selector * Address comments * Remove logs * minor fix * minor correctoions on chaosResource * add delay to example * add logic to set the delay config at `node` level in the networkSpec * temporary update test * refactor gen Chaos resource * fmt * Fix words * Fix words * fix example chaos delay * fmt * default correlation * add chaos test * typo * refactor: inject chaos after network spawn * fmt * remove commented code --------- Co-authored-by: Javier Viola <[email protected]> Co-authored-by: Javier Viola <[email protected]>
paritytech · Sep 3, 2023 · 84d83f3 · 84d83f3
1 parent aedf2bb
commit 84d83f3
Show file tree

Hide file tree

Showing 24 changed files with 4,274 additions and 13 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -275,3 +275,13 @@ db-snapshot:
     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
       --local-dir="${LOCAL_DIR}"
       --test="0013-db-snapshot.zndsl"
+
+chaos-delay:
+  extends:
+    - .zombienet-common
+  variables:
+    LOCAL_DIR: "/builds/parity/mirrors/zombienet/tests/chaos"
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}"
+      --test="0001-delay.zndsl"
diff --git a/examples/0005-big-network-w-delay.toml b/examples/0005-big-network-w-delay.toml
@@ -0,0 +1,32 @@
+[settings]
+timeout = 666
+
+[relaychain]
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "polkadot"
+default_args = [ "-lparachain=debug" ]
+chain = "rococo-local"
+[relaychain.default_delay_network_settings]
+  latency =  "200ms"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  [relaychain.nodes.delay_network_settings]
+    latency =  "1000ms"
+
+
+  [[relaychain.nodes]]
+  name = "bob"
+
+
+[[parachains]]
+id = 100
+
+  [parachains.collator]
+  name = "collator01"
+  image = "docker.io/parity/polkadot-parachain:latest"
+  command = "polkadot-parachain"
+    [parachains.collator.delay_network_settings]
+      latency =  "4000ms"
+      jitter = "4000ms"
+      correlation = "100"
diff --git a/examples/0005-big-network-w-delay.zndsl b/examples/0005-big-network-w-delay.zndsl
@@ -0,0 +1,7 @@
+Description: Big Network test
+Network: ./0005-big-network-w-delay.toml
+Creds: config
+
+# metrics
+alice: run ./delay_check.sh within 30 seconds
+bob: run ./delay_check.sh within 30 seconds
diff --git a/examples/delay_check.sh b/examples/delay_check.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
+ping -q -c 2 8.8.8.8 | tail -1 | awk -F "/" '{print $5}'
diff --git a/javascript/packages/orchestrator/src/configGenerator.ts b/javascript/packages/orchestrator/src/configGenerator.ts
@@ -131,6 +131,9 @@ export async function generateNetworkSpec(
       defaultPrometheusPrefix:
         config.relaychain.default_prometheus_prefix ||
         DEFAULT_PROMETHEUS_PREFIX,
+      delayNetworkSettings:
+        config.relaychain.default_delay_network_settings ||
+        config.settings?.global_delay_network_global_settings,
     },
     parachains: [],
   };
@@ -185,6 +188,7 @@ export async function generateNetworkSpec(
   }
 
   const relayChainBootnodes: string[] = [];
+
   for (const node of config.relaychain.nodes || []) {
     const nodeSetup = await getNodeFromConfig(
       networkSpec,
@@ -570,6 +574,10 @@ async function getCollatorNodeFromConfig(
     prometheusPrefix:
       parachain.prometheus_prefix ||
       networkSpec.relaychain.defaultPrometheusPrefix,
+    delayNetworkSettings:
+      collatorConfig.delay_network_settings ||
+      parachain.delayNetworkSettings ||
+      networkSpec.settings.delayNetworkSettings,
   };
 
   if (group) node.group = group;
@@ -654,6 +662,9 @@ async function getNodeFromConfig(
     p2pCertHash: node.p2p_cert_hash,
     prometheusPrefix:
       node.prometheus_prefix || networkSpec.relaychain.defaultPrometheusPrefix,
+    delayNetworkSettings:
+      node.delay_network_settings ||
+      networkSpec.relaychain.delayNetworkSettings,
   };
 
   if (group) nodeSetup.group = group;

diff --git a/javascript/packages/orchestrator/src/configTypes.ts b/javascript/packages/orchestrator/src/configTypes.ts
@@ -9,6 +9,7 @@ import {
   Resources,
   SubstrateCliArgsVersion,
   Node,
+  DelayNetworkSettings,
 } from "./sharedTypes";
 
 // network config to spawn.
@@ -44,6 +45,7 @@ export interface RelayChainConfig {
   node_groups?: NodeGroupConfig[];
   total_node_in_groups?: number;
   genesis?: JSON | ObjectJSON;
+  default_delay_network_settings?: DelayNetworkSettings;
 }
 
 export interface ComputedNetwork {
@@ -63,6 +65,7 @@ export interface ComputedNetwork {
     overrides: Override[];
     genesis?: JSON | ObjectJSON;
     defaultResources?: Resources;
+    delayNetworkSettings?: DelayNetworkSettings;
   };
   parachains: Parachain[];
   types: any;
@@ -91,6 +94,7 @@ export interface Settings {
   backchannel?: boolean; // only used in k8s at the moment, spawn a backchannel instance
   image_pull_policy?: "IfNotPresent" | "Never" | "Always";
   local_ip?: string; // ip used for expose local services (rpc/metrics/monitors)
+  global_delay_network_global_settings?: DelayNetworkSettings;
 }
 
 export interface GlobalVolume {
@@ -120,4 +124,5 @@ export interface ParachainConfig extends CommonParachainConfig {
 export interface NodeGroupConfig extends NodeCommonTypes {
   image?: string;
   count: string | number;
+  delay_network_settings?: DelayNetworkSettings;
 }
diff --git a/javascript/packages/orchestrator/src/orchestrator.ts b/javascript/packages/orchestrator/src/orchestrator.ts
@@ -103,9 +103,8 @@ export async function start(
 
     debug(JSON.stringify(networkSpec, null, 4));
 
-    const { initClient, setupChainSpec, getChainSpecRaw } = getProvider(
-      networkSpec.settings.provider,
-    );
+    const { initClient, setupChainSpec, getChainSpecRaw, genChaosDef } =
+      getProvider(networkSpec.settings.provider);
 
     // global timeout to spin the network
     const timeoutTimer = setTimeout(() => {
@@ -376,6 +375,22 @@ export async function start(
       local_ip: networkSpec.settings.local_ip,
     };
 
+    // Calculate chaos before start spawning the nodes
+    const chaosSpecs: any[] = [];
+    // network chaos is ONLY available in k8s for now
+    if (client.providerName === "kubernetes") {
+      const nodes = networkSpec.relaychain.nodes.concat(
+        networkSpec.parachains.map((para) => para.collators).flat(),
+      );
+      nodes.reduce((memo, node) => {
+        if (node.delayNetworkSettings)
+          memo.push(
+            genChaosDef(node.name, namespace, node.delayNetworkSettings),
+          );
+        return memo;
+      }, chaosSpecs);
+    }
+
     const firstNode = networkSpec.relaychain.nodes.shift();
     if (firstNode) {
       const nodeMultiAddress = await spawnNode(
@@ -508,6 +523,9 @@ export async function start(
 
     await verifyNodes(network);
 
+    // inject chaos to the running network
+    if (chaosSpecs.length > 0) await client.injectChaos(chaosSpecs);
+
     // cleanup global timeout
     network.launched = true;
     clearTimeout(timeoutTimer);

diff --git a/javascript/packages/orchestrator/src/providers/client.ts b/javascript/packages/orchestrator/src/providers/client.ts
@@ -72,7 +72,7 @@ export abstract class Client {
     filesToCopy?: fileMap[],
     keystore?: string,
     chainSpecId?: string,
-    dbSnapshot?: string,
+    dbSnapshot?: string, //delay?: DelayNetworkSettings,
   ): Promise<void>;
   abstract copyFileFromPod(
     identifier: string,
@@ -101,6 +101,7 @@ export abstract class Client {
   abstract spawnIntrospector(wsUri: string): Promise<void>;
   abstract validateAccess(): Promise<boolean>;
   abstract getLogsCommand(name: string): string;
+  abstract injectChaos(chaosSpecs: any[]): Promise<void>;
 }
 
 let client: Client;

diff --git a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts
@@ -2,9 +2,14 @@ import { getRandomPort, getSha256 } from "@zombienet/utils";
 import { getUniqueName } from "../../configGenerator";
 import { TMP_DONE, WAIT_UNTIL_SCRIPT_SUFIX } from "../../constants";
 import { Network } from "../../network";
-import { Node, ZombieRole } from "../../sharedTypes";
-import { BootNodeResource, NodeResource, ServiceResource } from "./resources";
-import { PodSpec, ServiceSpec } from "./resources/types";
+import { DelayNetworkSettings, Node, ZombieRole } from "../../sharedTypes";
+import {
+  BootNodeResource,
+  ChaosResource,
+  NodeResource,
+  ServiceResource,
+} from "./resources";
+import { ChaosSpec, PodSpec, ServiceSpec } from "./resources/types";
 
 export async function genBootnodeDef(
   namespace: string,
@@ -22,9 +27,18 @@ export async function genNodeDef(
   return nodeResource.generateSpec();
 }
 
+export function genChaosDef(
+  name: string,
+  namespace: string,
+  delay: DelayNetworkSettings,
+): ChaosSpec {
+  const resource = new ChaosResource(name, namespace, delay);
+  return resource.generateSpec();
+}
+
 export function genServiceDef(podSpec: PodSpec): ServiceSpec {
-  const serviceResource = new ServiceResource(podSpec);
-  return serviceResource.generateSpec();
+  const resource = new ServiceResource(podSpec);
+  return resource.generateSpec();
 }
 
 export function replaceNetworkRef(podDef: any, network: Network) {

diff --git a/javascript/packages/orchestrator/src/providers/k8s/index.ts b/javascript/packages/orchestrator/src/providers/k8s/index.ts
@@ -3,6 +3,7 @@ import {
   genBootnodeDef,
   genNodeDef,
   replaceNetworkRef,
+  genChaosDef,
 } from "./dynResourceDefinition";
 import { KubeClient, initClient } from "./kubeClient";
 import { getCliArgsVersion } from "./substrateCliArgsHelper";
@@ -16,4 +17,5 @@ export const provider = {
   getChainSpecRaw,
   replaceNetworkRef,
   getCliArgsVersion,
+  genChaosDef,
 };
diff --git a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts
@@ -143,6 +143,7 @@ export class KubeClient extends Client {
     await this.createResource(podDef, true);
     if (podDef.metadata.labels["zombie-role"] !== ZombieRole.Temp) {
       const serviceDef = genServiceDef(podDef);
+      writeLocalJsonFile(this.tmpDir, `${name}-service.json`, serviceDef);
       await this.createResource(serviceDef, true);
     }
 
@@ -219,6 +220,7 @@ export class KubeClient extends Client {
 
     await this.putLocalMagicFile(name);
     await this.waitPodReady(name);
+
     logTable = new CreateLogTable({
       colWidths: [20, 100],
     });
@@ -879,4 +881,15 @@ export class KubeClient extends Client {
   getLogsCommand(name: string): string {
     return `kubectl logs -f ${name} -c ${name} -n ${this.namespace}`;
   }
+
+  async injectChaos(chaosSpecs: any[]) {
+    const merged = {
+      apiVersion: "v1",
+      kind: "List",
+      items: chaosSpecs,
+    };
+
+    writeLocalJsonFile(this.tmpDir, `merged-chaos.json`, merged);
+    await this.createResource(merged, true);
+  }
 }
diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/chaosResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/chaosResource.ts
@@ -0,0 +1,58 @@
+import { ChaosSpec } from "./types";
+
+import { DelayNetworkSettings } from "../../../sharedTypes";
+
+export class ChaosResource {
+  constructor(
+    protected readonly name: string,
+    protected readonly namespace: string,
+    protected readonly delay: DelayNetworkSettings,
+  ) {}
+
+  public generateSpec() {
+    if (this.delay.latency.slice(-2) !== "ms") {
+      throw Error(
+        "Latency value should include the 'ms' indicator (e.g. '100ms')",
+      );
+    }
+
+    if (this.delay.jitter && this.delay.jitter.slice(-2) !== "ms") {
+      throw Error(
+        "Jitter value should include the 'ms' indicator (e.g. '100ms')",
+      );
+    }
+
+    if (this.delay.correlation) {
+      const correlation = parseFloat(this.delay.correlation);
+      if (Number.isNaN(correlation)) {
+        throw Error(
+          "Correlation value should be parsable as Float by k8s api (e.g. '100')",
+        );
+      } else {
+        this.delay.correlation = correlation.toString();
+      }
+    } else {
+      // set default correlation (100)
+      this.delay.correlation = "100";
+    }
+    return this.generateChaosSpec();
+  }
+
+  private generateChaosSpec(): ChaosSpec {
+    return {
+      apiVersion: "chaos-mesh.org/v1alpha1",
+      kind: "NetworkChaos",
+      metadata: { name: this.name },
+      spec: {
+        mode: "all",
+        action: "delay",
+        selector: {
+          pods: {
+            [this.namespace]: [this.name],
+          },
+        },
+        delay: this.delay,
+      },
+    };
+  }
+}
diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts
@@ -1,3 +1,4 @@
 export { BootNodeResource } from "./bootNodeResource";
+export { ChaosResource } from "./chaosResource";
 export { NodeResource } from "./nodeResource";
 export { ServiceResource } from "./serviceResource";
diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts
@@ -16,7 +16,6 @@ export class ServiceResource {
   public generateSpec() {
     const ports = this.generatePorts();
     const name = this.podSpec.metadata.name;
-
     return this.generateServiceSpec(name, ports);
   }