From d538b50e3a2bd51b7301a13bab9e16e92f25bdab Mon Sep 17 00:00:00 2001 From: Kuromesi Date: Mon, 4 Sep 2023 16:42:41 +0800 Subject: [PATCH] make some improvements Signed-off-by: Kuromesi --- lua_configuration/lua.go | 225 ++++++++++++++++++ pkg/trafficrouting/manager.go | 2 +- pkg/trafficrouting/network/custom/custom.go | 39 ++- .../network/custom/custom_test.go | 28 +-- .../VirtualService/trafficRouting.lua | 225 ++++++++++++++++++ 5 files changed, 478 insertions(+), 41 deletions(-) create mode 100644 lua_configuration/lua.go create mode 100644 pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua diff --git a/lua_configuration/lua.go b/lua_configuration/lua.go new file mode 100644 index 00000000..e33ba035 --- /dev/null +++ b/lua_configuration/lua.go @@ -0,0 +1,225 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/trafficrouting/network/custom" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + utilpointer "k8s.io/utils/pointer" +) + +type TestCase struct { + Rollout *v1alpha1.Rollout `json:"rollout,omitempty"` + TrafficRouting *v1alpha1.TrafficRouting `json:"trafficRouting,omitempty"` + Original *unstructured.Unstructured `json:"original,omitempty"` + Expected []*unstructured.Unstructured `json:"expected,omitempty"` +} + +// convert testdata to lua object for debugging +func main() { + err := PathWalk() + if err != nil { + fmt.Println(err) + } +} + +func PathWalk() error { + err := filepath.Walk("./", func(path string, f os.FileInfo, err error) error { + if !strings.Contains(path, "trafficRouting.lua") { + return nil + } + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + dir := filepath.Dir(path) + err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + fmt.Printf("--- walking path: %s ---\n", path) + err = ObjectToTable(path) + if err != nil { + return fmt.Errorf("failed to convert object to table: %s", err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err.Error()) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk path: %s", err) + } + return nil +} + +// convert a testcase object to lua table for debug +func ObjectToTable(path string) error { + dir, file := filepath.Split(path) + testCase, err := getLuaTestCase(path) + if err != nil { + return fmt.Errorf("failed to get lua testcase: %s", err) + } + uList := make(map[string]interface{}) + rollout := testCase.Rollout + trafficRouting := testCase.TrafficRouting + if rollout != nil { + steps := rollout.Spec.Strategy.Canary.Steps + for i, step := range steps { + weight := step.TrafficRoutingStrategy.Weight + if step.TrafficRoutingStrategy.Weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service + // if rollout.Spec.Strategy.Canary.TrafficRoutings[0].CreateCanaryService { + canaryService = fmt.Sprintf("%s-canary", stableService) + // } else { + // canaryService = stableService + // } + data := &custom.LuaData{ + Data: custom.Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: step.TrafficRoutingStrategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + uList[fmt.Sprintf("step_%d", i)] = data + } + } else if trafficRouting != nil { + weight := trafficRouting.Spec.Strategy.Weight + if weight == nil { + weight = utilpointer.Int32(-1) + } + var canaryService string + stableService := trafficRouting.Spec.ObjectRef[0].Service + canaryService = stableService + data := &custom.LuaData{ + Data: custom.Data{ + Labels: testCase.Original.GetLabels(), + Annotations: testCase.Original.GetAnnotations(), + Spec: testCase.Original.Object["spec"], + }, + Matches: trafficRouting.Spec.Strategy.Matches, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + CanaryService: canaryService, + StableService: stableService, + } + uList["steps_0"] = data + } else { + return fmt.Errorf("neither rollout nor trafficRouting defined in test case: %s", path) + } + + objStr, err := executeLua(uList) + if err != nil { + return fmt.Errorf("failed to execute lua: %s", err.Error()) + } + filePath := fmt.Sprintf("%s%s_obj.lua", dir, strings.Split(file, ".")[0]) + fileStream, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("failed to open file: %s", err) + } + defer fileStream.Close() + _, err = io.WriteString(fileStream, objStr) + if err != nil { + return fmt.Errorf("failed to WriteString %s", err) + } + return nil +} + +func getLuaTestCase(path string) (*TestCase, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + luaTestCase := &TestCase{} + err = yaml.Unmarshal(yamlFile, luaTestCase) + if err != nil { + return nil, err + } + return luaTestCase, nil +} + +func executeLua(steps map[string]interface{}) (string, error) { + luaManager := &luamanager.LuaManager{} + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&steps) + if err != nil { + return "", fmt.Errorf("failed to convert to unstructured: %s", err) + } + u := &unstructured.Unstructured{Object: unObj} + script := ` + function serialize(obj, isKey) + local lua = "" + local t = type(obj) + if t == "number" then + lua = lua .. obj + elseif t == "boolean" then + lua = lua .. tostring(obj) + elseif t == "string" then + if isKey then + lua = lua .. string.format("%s", obj) + else + lua = lua .. string.format("%q", obj) + end + elseif t == "table" then + lua = lua .. "{" + for k, v in pairs(obj) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + local metatable = getmetatable(obj) + if metatable ~= nil and type(metatable.__index) == "table" then + for k, v in pairs(metatable.__index) do + if type(k) == "string" then + lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. "," + else + lua = lua .. serialize(v, false) .. "," + end + end + end + lua = lua .. "}" + elseif t == "nil" then + return nil + else + error("can not serialize a " .. t .. " type.") + end + return lua + end + + function table2string(tablevalue) + local stringtable = "steps=" .. serialize(tablevalue) + print(stringtable) + return stringtable + end + return table2string(obj) + ` + l, err := luaManager.RunLuaScript(u, script) + if err != nil { + return "", fmt.Errorf("failed to run lua script: %s", err) + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTString { + return returnValue.String(), nil + } else { + return "", fmt.Errorf("unexpected lua output type") + } +} diff --git a/pkg/trafficrouting/manager.go b/pkg/trafficrouting/manager.go index 5a12f39f..ccc4ab67 100644 --- a/pkg/trafficrouting/manager.go +++ b/pkg/trafficrouting/manager.go @@ -266,7 +266,7 @@ func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, c trafficRouting := con.ObjectRef[0] if trafficRouting.CustomNetworkRefs != nil { return custom.NewCustomController(c, custom.Config{ - RolloutName: con.Key, + Key: con.Key, RolloutNs: con.Namespace, CanaryService: cService, StableService: sService, diff --git a/pkg/trafficrouting/network/custom/custom.go b/pkg/trafficrouting/network/custom/custom.go index 7ac428c2..f6495d91 100644 --- a/pkg/trafficrouting/network/custom/custom.go +++ b/pkg/trafficrouting/network/custom/custom.go @@ -26,7 +26,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" - "github.com/openkruise/rollouts/api/v1alpha1" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/trafficrouting/network" "github.com/openkruise/rollouts/pkg/util" @@ -47,6 +46,14 @@ const ( LuaConfigMap = "kruise-rollout-configuration" ) +type LuaData struct { + Data Data + CanaryWeight int32 + StableWeight int32 + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + StableService string +} type Data struct { Spec interface{} `json:"spec,omitempty"` Labels map[string]string `json:"labels,omitempty"` @@ -60,7 +67,7 @@ type customController struct { } type Config struct { - RolloutName string + Key string RolloutNs string CanaryService string StableService string @@ -151,7 +158,6 @@ func (r *customController) Finalise(ctx context.Context) error { } return err } - // when one failed how to proceed? if err := r.restoreObject(obj); err != nil { klog.Errorf("failed to restore object: %s/%s", ref.Kind, ref.Name) return err @@ -211,33 +217,18 @@ func (r *customController) restoreObject(obj *unstructured.Unstructured) error { func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) { weight := strategy.Weight matches := strategy.Matches - rollout := &v1alpha1.Rollout{} - if err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.RolloutName}, rollout); err != nil { - klog.Errorf("failed to get rollout/%s when execute custom network provider lua script", r.conf.RolloutName) - return Data{}, err - } if weight == nil { // the lua script does not have a pointer type, // so we need to pass weight=-1 to indicate the case where weight is nil. weight = utilpointer.Int32(-1) } - type LuaData struct { - Data Data - CanaryWeight int32 - StableWeight int32 - Matches []rolloutv1alpha1.HttpRouteMatch - CanaryService string - StableService string - PatchPodMetadata *rolloutv1alpha1.PatchPodTemplateMetadata - } data := &LuaData{ - Data: spec, - CanaryWeight: *weight, - StableWeight: 100 - *weight, - Matches: matches, - CanaryService: r.conf.CanaryService, - StableService: r.conf.StableService, - PatchPodMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata, + Data: spec, + CanaryWeight: *weight, + StableWeight: 100 - *weight, + Matches: matches, + CanaryService: r.conf.CanaryService, + StableService: r.conf.StableService, } unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) diff --git a/pkg/trafficrouting/network/custom/custom_test.go b/pkg/trafficrouting/network/custom/custom_test.go index 46134be2..f4f812b9 100644 --- a/pkg/trafficrouting/network/custom/custom_test.go +++ b/pkg/trafficrouting/network/custom/custom_test.go @@ -59,7 +59,7 @@ var ( "route": [ { "destination": { - "host": "echoserver", + "host": "echoserver" } } ] @@ -111,7 +111,7 @@ func TestInitialize(t *testing.T) { Namespace: util.GetRolloutNamespace(), }, Data: map[string]string{ - fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", }, } }, @@ -131,7 +131,7 @@ func TestInitialize(t *testing.T) { getUnstructured: func() *unstructured.Unstructured { u := &unstructured.Unstructured{} _ = u.UnmarshalJSON([]byte(networkDemo)) - u.SetAPIVersion("networking.test.io/v1alpha3") + u.SetAPIVersion("networking.istio.io/v1alpha3") return u }, getConfig: func() Config { @@ -140,7 +140,7 @@ func TestInitialize(t *testing.T) { CanaryService: "echoserver-canary", TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ { - APIVersion: "networking.test.io/v1alpha3", + APIVersion: "networking.istio.io/v1alpha3", Kind: "VirtualService", Name: "echoserver", }, @@ -154,14 +154,14 @@ func TestInitialize(t *testing.T) { Namespace: util.GetRolloutNamespace(), }, Data: map[string]string{ - fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript", + fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript", }, } }, expectUnstructured: func() *unstructured.Unstructured { u := &unstructured.Unstructured{} _ = u.UnmarshalJSON([]byte(networkDemo)) - u.SetAPIVersion("networking.test.io/v1alpha3") + u.SetAPIVersion("networking.istio.io/v1alpha3") annotations := map[string]string{ OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, "virtual": "test", @@ -227,22 +227,18 @@ func TestEnsureRoutes(t *testing.T) { getUnstructured: func() *unstructured.Unstructured { u := &unstructured.Unstructured{} _ = u.UnmarshalJSON([]byte(networkDemo)) - annotations := map[string]string{ - OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, - "virtual": "test", - } - u.SetAnnotations(annotations) + u.SetAPIVersion("networking.istio.io/v1alpha3") return u }, expectInfo: func() (bool, *unstructured.Unstructured) { u := &unstructured.Unstructured{} _ = u.UnmarshalJSON([]byte(networkDemo)) annotations := map[string]string{ - OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`, + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, "virtual": "test", } u.SetAnnotations(annotations) - specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}` + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}` var spec interface{} _ = json.Unmarshal([]byte(specStr), &spec) u.Object["spec"] = spec @@ -251,7 +247,7 @@ func TestEnsureRoutes(t *testing.T) { }, } config := Config{ - RolloutName: "rollout-demo", + Key: "rollout-demo", StableService: "echoserver", CanaryService: "echoserver-canary", TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{ @@ -298,11 +294,11 @@ func TestFinalise(t *testing.T) { u := &unstructured.Unstructured{} _ = u.UnmarshalJSON([]byte(networkDemo)) annotations := map[string]string{ - OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`, + OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`, "virtual": "test", } u.SetAnnotations(annotations) - specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}` + specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}}]}]}` var spec interface{} _ = json.Unmarshal([]byte(specStr), &spec) u.Object["spec"] = spec diff --git a/pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua b/pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua new file mode 100644 index 00000000..37ff292c --- /dev/null +++ b/pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua @@ -0,0 +1,225 @@ +-- obj = { canaryWeight = 20, stableWeight = 80, +-- matches = { +-- { headers = { { name = "user-agent", value = "pc", type = "Exact", }, +-- { type = "RegularExpression", name = "name", value = ".*demo", }, }, }, }, +-- canaryService = "nginx-service-canary", stableService = "nginx-service", +-- data = { +-- spec = { hosts = { "*", }, http = { { route = { { destination = { host = "nginx-service", subset = "v1"}, }, { destination = { host = "nginx-service", subset = "v2"}, } }, }, }, +-- gateways = { "nginx-gateway", }, }, }, } + +spec = obj.data.spec + +if obj.canaryWeight == -1 then + obj.canaryWeight = 100 + obj.stableWeight = 0 +end + +function FindRules(spec, protocol) + local rules = {} + if (protocol == "http") then + if (spec.http ~= nil) then + for _, http in ipairs(spec.http) do + table.insert(rules, http) + end + end + elseif (protocol == "tcp") then + if (spec.tcp ~= nil) then + for _, http in ipairs(spec.tcp) do + table.insert(rules, http) + end + end + elseif (protocol == "tls") then + if (spec.tls ~= nil) then + for _, http in ipairs(spec.tls) do + table.insert(rules, http) + end + end + end + return rules +end + +-- find matched route of VirtualService spec with stable svc +function FindMatchedRules(spec, stableService, protocol) + local matchedRoutes = {} + local rules = FindRules(spec, protocol) + -- a rule contains 'match' and 'route' + for _, rule in ipairs(rules) do + for _, route in ipairs(rule.route) do + if route.destination.host == stableService then + table.insert(matchedRoutes, rule) + break + end + end + end + return matchedRoutes +end + +function FindStableServiceSubsets(spec, stableService, protocol) + local stableSubsets = {} + local rules = FindRules(spec, protocol) + local hasRule = false + -- a rule contains 'match' and 'route' + for _, rule in ipairs(rules) do + for _, route in ipairs(rule.route) do + if route.destination.host == stableService then + hasRule = true + local contains = false + for _, v in ipairs(stableSubsets) do + if v == route.destination.subset then + contains = true + break + end + end + if not contains and route.destination.subset ~= nil then + table.insert(stableSubsets, route.destination.subset) + end + end + end + end + return hasRule, stableSubsets +end + +function DeepCopy(original) + local copy + if type(original) == 'table' then + copy = {} + for key, value in pairs(original) do + copy[key] = DeepCopy(value) + end + else + copy = original + end + return copy +end + +function CalculateWeight(route, stableWeight, n) + local weight + if (route.weight) then + weight = math.floor(route.weight * stableWeight / 100) + else + weight = math.floor(stableWeight / n) + end + return weight +end + +-- generate routes with matches, insert a rule before other rules +function GenerateMatchedRoutes(spec, matches, stableService, canaryService, stableWeight, canaryWeight, protocol) + local hasRule, stableServiceSubsets = FindStableServiceSubsets(spec, stableService, protocol) + if (not hasRule) then + return + end + for _, match in ipairs(matches) do + local route = {} + route["match"] = {} + + for key, value in pairs(match) do + local vsMatch = {} + vsMatch[key] = {} + for _, rule in ipairs(value) do + if rule["type"] == "RegularExpression" then + matchType = "regex" + elseif rule["type"] == "Exact" then + matchType = "exact" + elseif rule["type"] == "Prefix" then + matchType = "prefix" + end + if key == "headers" then + vsMatch[key][rule["name"]] = {} + vsMatch[key][rule["name"]][matchType] = rule.value + else + vsMatch[key][matchType] = rule.value + end + end + table.insert(route["match"], vsMatch) + end + route.route = { + { + destination = {} + } + } + -- if stableWeight != 0, then add stable service destinations + -- incase there are multiple subsets in stable service + if stableWeight ~= 0 then + local nRoute = {} + if #stableServiceSubsets ~= 0 then + local weight = CalculateWeight(nRoute, stableWeight, #stableServiceSubsets) + for _, r in ipairs(stableServiceSubsets) do + nRoute = { + destination = { + host = stableService, + subset = r + }, + weight = weight + } + table.insert(route.route, nRoute) + end + else + nRoute = { + destination = { + host = stableService + }, + weight = stableWeight + } + table.insert(route.route, nRoute) + end + -- update every matched route + route.route[1].weight = canaryWeight + end + -- if stableService == canaryService, then do e2e release + if stableService == canaryService then + route.route[1].destination.host = stableService + route.route[1].destination.subset = "canary" + else + route.route[1].destination.host = canaryService + end + if (protocol == "http") then + table.insert(spec.http, 1, route) + elseif (protocol == "tls") then + table.insert(spec.tls, 1, route) + elseif (protocol == "tcp") then + table.insert(spec.tcp, 1, route) + end + end +end + +-- generate routes without matches, change every rule +function GenerateRoutes(spec, stableService, canaryService, stableWeight, canaryWeight, protocol) + local matchedRules = FindMatchedRules(spec, stableService, protocol) + for _, rule in ipairs(matchedRules) do + local canary + if stableService ~= canaryService then + canary = { + destination = { + host = canaryService, + }, + weight = canaryWeight, + } + else + canary = { + destination = { + host = stableService, + subset = "canary", + }, + weight = canaryWeight, + } + end + + -- incase there are multiple versions traffic already, do a for-loop + for _, route in ipairs(rule.route) do + -- update stable service weight + route.weight = CalculateWeight(route, stableWeight, #rule.route) + end + table.insert(rule.route, canary) + end +end + +if (obj.matches) then + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http") + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp") + GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls") +else + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http") + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp") + GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls") +end +return obj.data