Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

k8splugin support manager_networkmanage_network_ns_lifecycle=true from CRI-O #1153

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

k8splugin support manager_networkmanage_network_ns_lifecycle=true from CRI-O #1153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions mgmtfn/k8splugin/cniapi/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,17 @@ type CNIPodAttr struct {

// RspAddPod contains the response to the AddPod
type RspAddPod struct {
Result uint `json:"result,omitempty"`
EndpointID string `json:"endpointid,omitempty"`
Result uint `json:"result,omitempty"`
EndpointID string `json:"endpointid,omitempty"`
ErrMsg string `json:"errmsg,omitempty"`
ErrInfo string `json:"errinfo,omitempty"`
Attr *Attr `json:"epattr,omitempty"`
}

type Attr struct {
IPAddress string `json:"ipaddress,omitempty"`
PortName string `json:"portname,omitempty"`
Gateway string `json:"gateway,omitempty"`
IPv6Address string `json:"ipv6address,omitempty"`
ErrMsg string `json:"errmsg,omitempty"`
ErrInfo string `json:"errinfo,omitempty"`
IPv6Gateway string `json:"ipv6gateway,omitempty"`
}
232 changes: 223 additions & 9 deletions mgmtfn/k8splugin/contivk8s/k8s_cni.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,16 @@ import (
"fmt"
"net"
"os"
"os/exec"
"strconv"
"strings"
"time"

"github.com/contiv/netplugin/mgmtfn/k8splugin/cniapi"
"github.com/contiv/netplugin/mgmtfn/k8splugin/contivk8s/clients"
"github.com/contiv/netplugin/version"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"

logger "github.com/Sirupsen/logrus"
ip "github.com/containernetworking/cni/pkg/types"
Expand Down Expand Up @@ -67,7 +72,185 @@ func getPodInfo(ppInfo *cniapi.CNIPodAttr) error {
return nil
}

func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
// nsToPID is a utility that extracts the PID from the netns
func nsToPID(ns string) (int, error) {
elements := strings.Split(ns, "/")
return strconv.Atoi(elements[2])
}

func nsToFD(nspath string) (netns.NsHandle, error) {
log.Infof(">> Get fd from ns: %v", nspath)

ns, err := os.Readlink(nspath)
if err != nil {
log.Errorf("invalid netns path. Error: %s", err)
return netns.None(), err
}

fd, err := netns.GetFromPath(ns)
if err != nil {
log.Errorf("fd not found. Error: %s", err)
return netns.None(), err
}

return fd, nil
}

// getLink is a wrapper that fetches the netlink corresponding to the ifname
func getLink(ifname string) (netlink.Link, error) {
// find the link
link, err := netlink.LinkByName(ifname)
if err != nil {
if !strings.Contains(err.Error(), "Link not found") {
log.Errorf("unable to find link %q. Error: %q", ifname, err)
return link, err
}
// try once more as sometimes (somehow) link creation is taking
// sometime, causing link not found error
time.Sleep(1 * time.Second)
link, err = netlink.LinkByName(ifname)
if err != nil {
log.Errorf("unable to find link %q. Error %q", ifname, err)
}
return link, err
}
return link, err
}

func moveToNS(ifname, nspath string) error {
//find the link
link, err := getLink(ifname)
if err != nil {
log.Errorf("unable to find link %q. Error %q", ifname, err)
return err
}
log.Infof("> Netns Path %s", nspath)

// convert netns to pid that netlink needs
var pid int

if ok := strings.HasPrefix(nspath, "/proc/"); ok {
pid, err = nsToPID(nspath)
} else {
log.Info("Is not a process")
pid = -1
}

log.Infof(">> Move to netns pid %d", pid)
if pid != -1 {
err = netlink.LinkSetNsPid(link, pid)
if err != nil {
log.Errorf("unable to move interface %s to pid %d. Error: %s",
ifname, pid, err)
return err
}
return nil
}

fd, err := nsToFD(nspath)
if err != nil {
return err
}

err = netlink.LinkSetNsFd(link, int(fd))
if err != nil {
log.Errorf("unable to move interface %s to fd %d. Error: %s",
ifname, nspath, err)
return err
}

return nil
}

// setIfAttrs sets the required attributes for the container interface
func setIfAttrs(nspath, ifname, cidr, cidr6, newname string) error {
nsenterPath, err := exec.LookPath("nsenter")
if err != nil {
return err
}
ipPath, err := exec.LookPath("ip")
if err != nil {
return err
}

// rename to the desired ifname
net := "--net=" + nspath
rename, err := exec.Command(nsenterPath, net, "--", ipPath, "link",
"set", "dev", ifname, "name", newname).CombinedOutput()
if err != nil {
log.Errorf("unable to rename interface %s to %s. Error: %s",
ifname, newname, err)
return nil
}
log.Infof("Output from rename: %v", rename)

// set the ip address
assignIP, err := exec.Command(nsenterPath, net, "--", ipPath,
"address", "add", cidr, "dev", newname).CombinedOutput()

if err != nil {
log.Errorf("unable to assign ip %s to %s. Error: %s",
cidr, newname, err)
return nil
}
log.Infof("Output from ip assign: %v", assignIP)

if cidr6 != "" {
out, err := exec.Command(nsenterPath, net, "--", ipPath,
"-6", "address", "add", cidr6, "dev", newname).CombinedOutput()
if err != nil {
log.Errorf("unable to assign IPv6 %s to %s. Error: %s",
cidr6, newname, err)
return nil
}
log.Infof("Output of IPv6 assign: %v", out)
}

// Finally, mark the link up
bringUp, err := exec.Command(nsenterPath, net, "--", ipPath,
"link", "set", "dev", newname, "up").CombinedOutput()

if err != nil {
log.Errorf("unable to assign ip %s to %s. Error: %s",
cidr, newname, err)
return nil
}
log.Debugf("Output from ip assign: %v", bringUp)
return nil

}

// setDefGw sets the default gateway for the container namespace
func setDefGw(nspath, gw, gw6, intfName string) error {
nsenterPath, err := exec.LookPath("nsenter")
if err != nil {
return err
}
routePath, err := exec.LookPath("route")
if err != nil {
return err
}
// set default gw
net := "--net=" + nspath
out, err := exec.Command(nsenterPath, net, "--", routePath, "add",
"default", "gw", gw, intfName).CombinedOutput()
if err != nil {
log.Errorf("unable to set default gw %s. Error: %s - %s", gw, err, out)
return nil
}

if gw6 != "" {
out, err := exec.Command(nsenterPath, net, "--", routePath,
"-6", "add", "default", "gw", gw6, intfName).CombinedOutput()
if err != nil {
log.Errorf("unable to set default IPv6 gateway %s. Error: %s - %s", gw6, err, out)
return nil
}
}

return nil
}
func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) error {

// Add to contiv network
result, err := nc.AddPod(pInfo)
Expand Down Expand Up @@ -95,15 +278,40 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
}
os.Exit(1)
}
log.Infof("EP created IP: %s\n", result.Attr.IPAddress)

// move to the desired netns
err = moveToNS(result.Attr.PortName, pInfo.NwNameSpace)
if err != nil {
log.Errorf("Error moving to netns. Err: %v", err)
return err
}

err = setIfAttrs(pInfo.NwNameSpace, result.Attr.PortName, result.Attr.IPAddress,
result.Attr.IPv6Address, pInfo.IntfName)

if err != nil {
log.Errorf("Error setting interface attributes. Err: %v", err)
return err
}

if result.Attr.Gateway != "" {
// Set default gateway
err = setDefGw(pInfo.NwNameSpace, result.Attr.Gateway, result.Attr.IPv6Gateway,
pInfo.IntfName)
if err != nil {
log.Errorf("Error setting default gateway. Err: %v", err)
return err
}
}

log.Infof("EP created IP: %s\n", result.IPAddress)
// Write the ip address of the created endpoint to stdout

// ParseCIDR returns a reference to IPNet
ip4Net, err := ip.ParseCIDR(result.IPAddress)
ip4Net, err := ip.ParseCIDR(result.Attr.IPAddress)
if err != nil {
log.Errorf("Failed to parse IPv4 CIDR: %v", err)
return
return err
}

out := CNIResponse{
Expand All @@ -115,11 +323,11 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
Address: net.IPNet{IP: ip4Net.IP, Mask: ip4Net.Mask},
})

if result.IPv6Address != "" {
ip6Net, err := ip.ParseCIDR(result.IPv6Address)
if result.Attr.IPv6Address != "" {
ip6Net, err := ip.ParseCIDR(result.Attr.IPv6Address)
if err != nil {
log.Errorf("Failed to parse IPv6 CIDR: %v", err)
return
return err
}

out.IPs = append(out.IPs, &cni.IPConfig{
Expand All @@ -131,11 +339,13 @@ func addPodToContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
log.Errorf("Failed to marshal json: %v", err)
return
return err
}

log.Infof("Response from CNI executable: \n%s", fmt.Sprintf("%s", data))
fmt.Printf(fmt.Sprintf("%s", data))

return nil
}

func deletePodFromContiv(nc *clients.NWClient, pInfo *cniapi.CNIPodAttr) {
Expand Down Expand Up @@ -214,7 +424,11 @@ func mainfunc() {

nc := clients.NewNWClient()
if cniCmd == "ADD" {
addPodToContiv(nc, &pInfo)
err = addPodToContiv(nc, &pInfo)
if err != nil {
deletePodFromContiv(nc, &pInfo)
log.Infof("Error on add pod. Err %v", err)
}
} else if cniCmd == "DEL" {
deletePodFromContiv(nc, &pInfo)
}
Expand Down
Loading