From fcebe27df9b81bd64138af0b7d94451531ca5d11 Mon Sep 17 00:00:00 2001 From: Lee Briggs Date: Sun, 28 Jul 2019 16:05:43 -0700 Subject: [PATCH] Attempt at cordon but no drain - Change flag name to drain (automatically adds `--no-drain` flag) - Add drain flag to NewAPICordonDrainer - Improve drain function to remove function wrap - gofmt + README nit. Additional logging when drain is disabled Provide some visibility as to why nodes are not being drained when the WithDrain option is set. Some additional drive-by changes to comments to walk them out a bit. --- README.md | 18 +++++++++++++++-- cmd/draino/draino.go | 6 +++++- internal/kubernetes/drainer.go | 28 ++++++++++++++++++++++++++ internal/kubernetes/podfilters_test.go | 4 ++-- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 23b2bd66..b36d66a5 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,14 @@ Flags: --drain-buffer=10m0s Minimum time between starting each drain. Nodes are always cordoned immediately. --node-label=KEY=VALUE ... Only nodes with this label will be eligible for cordoning and draining. May be specified multiple times. + --namespace="kube-system" Namespace used to create leader election lock object. + --leader-election-lease-duration=15s + Lease duration for leader election. + --leader-election-renew-deadline=10s + Leader election renew deadline. + --leader-election-retry-period=2s + Leader election retry period. + --skip-drain Whether to skip draining nodes after cordoning. --evict-daemonset-pods Evict pods that were created by an extant DaemonSet. --evict-emptydir-pods Evict pods with local storage, i.e. with emptyDir volumes. --evict-unreplicated-pods Evict pods that were not created by a replication controller. @@ -46,7 +54,6 @@ Flags: Args: Nodes for which any of these conditions are true will be cordoned and drained. - ``` ## Considerations @@ -149,4 +156,11 @@ is marked as `Failed`. If you want to reschedule a drain tentative on that node, ``` kubectl annotate node {node-name} draino/drain-retry=true -``` \ No newline at end of file +``` +## Modes + +### Dry Run +Draino can be run in dry run mode using the `--dry-run` flag. + +### Cordon Only +Draino can also optionally be run in a mode where the nodes are only cordoned, and not drained. This can be achieved by using the `--skip-drain` flag. diff --git a/cmd/draino/draino.go b/cmd/draino/draino.go index 5e0987c1..690ee050 100644 --- a/cmd/draino/draino.go +++ b/cmd/draino/draino.go @@ -64,6 +64,7 @@ func main() { leaderElectionRenewDeadline = app.Flag("leader-election-renew-deadline", "Leader election renew deadline.").Default(DefaultLeaderElectionRenewDeadline.String()).Duration() leaderElectionRetryPeriod = app.Flag("leader-election-retry-period", "Leader election retry period.").Default(DefaultLeaderElectionRetryPeriod.String()).Duration() + skipDrain = app.Flag("skip-drain", "Whether to skip draining nodes after cordoning.").Default("false").Bool() evictDaemonSetPods = app.Flag("evict-daemonset-pods", "Evict pods that were created by an extant DaemonSet.").Bool() evictLocalStoragePods = app.Flag("evict-emptydir-pods", "Evict pods with local storage, i.e. with emptyDir volumes.").Bool() evictUnreplicatedPods = app.Flag("evict-unreplicated-pods", "Evict pods that were not created by a replication controller.").Bool() @@ -142,7 +143,10 @@ func main() { kubernetes.NewAPICordonDrainer(cs, kubernetes.MaxGracePeriod(*maxGracePeriod), kubernetes.EvictionHeadroom(*evictionHeadroom), - kubernetes.WithPodFilter(kubernetes.NewPodFilters(pf...))), + kubernetes.WithSkipDrain(*skipDrain), + kubernetes.WithPodFilter(kubernetes.NewPodFilters(pf...)), + kubernetes.WithAPICordonDrainerLogger(log), + ), kubernetes.NewEventRecorder(cs), kubernetes.WithLogger(log), kubernetes.WithDrainBuffer(*drainBuffer)) diff --git a/internal/kubernetes/drainer.go b/internal/kubernetes/drainer.go index a8cdb27a..7ffce9b8 100644 --- a/internal/kubernetes/drainer.go +++ b/internal/kubernetes/drainer.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pkg/errors" + "go.uber.org/zap" core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -38,6 +39,7 @@ const ( kindDaemonSet = "DaemonSet" ConditionDrainedScheduled = "DrainScheduled" + DefaultSkipDrain = false ) type errTimeout struct{} @@ -93,11 +95,13 @@ func (d *NoopCordonDrainer) MarkDrain(n *core.Node, when, finish time.Time, fail // APICordonDrainer drains Kubernetes nodes via the Kubernetes API. type APICordonDrainer struct { c kubernetes.Interface + l *zap.Logger filter PodFilterFunc maxGracePeriod time.Duration evictionHeadroom time.Duration + skipDrain bool } // SuppliedCondition defines the condition will be watched. @@ -135,14 +139,31 @@ func WithPodFilter(f PodFilterFunc) APICordonDrainerOption { } } +// WithDrain determines if we're actually going to drain nodes +func WithSkipDrain(b bool) APICordonDrainerOption { + return func(d *APICordonDrainer) { + d.skipDrain = b + } +} + +// WithAPICordonDrainerLogger configures a APICordonDrainer to use the supplied +// logger. +func WithAPICordonDrainerLogger(l *zap.Logger) APICordonDrainerOption { + return func(d *APICordonDrainer) { + d.l = l + } +} + // NewAPICordonDrainer returns a CordonDrainer that cordons and drains nodes via // the Kubernetes API. func NewAPICordonDrainer(c kubernetes.Interface, ao ...APICordonDrainerOption) *APICordonDrainer { d := &APICordonDrainer{ c: c, + l: zap.NewNop(), filter: NewPodFilters(), maxGracePeriod: DefaultMaxGracePeriod, evictionHeadroom: DefaultEvictionOverhead, + skipDrain: DefaultSkipDrain, } for _, o := range ao { o(d) @@ -234,6 +255,13 @@ func IsMarkedForDrain(n *core.Node) bool { // Drain the supplied node. Evicts the node of all but mirror and DaemonSet pods. func (d *APICordonDrainer) Drain(n *core.Node) error { + + // Do nothing if draining is not enabled. + if d.skipDrain { + d.l.Debug("Skipping drain because draining is disabled") + return nil + } + pods, err := d.getPods(n.GetName()) if err != nil { return errors.Wrapf(err, "cannot get pods for node %s", n.GetName()) diff --git a/internal/kubernetes/podfilters_test.go b/internal/kubernetes/podfilters_test.go index 25e123df..983d2785 100644 --- a/internal/kubernetes/podfilters_test.go +++ b/internal/kubernetes/podfilters_test.go @@ -198,7 +198,7 @@ func TestPodFilters(t *testing.T) { name: "NoPodAnnotations", pod: core.Pod{ ObjectMeta: meta.ObjectMeta{ - Name: podName, + Name: podName, }, }, filter: UnprotectedPodFilter("Protect"), @@ -208,7 +208,7 @@ func TestPodFilters(t *testing.T) { name: "NoPodAnnotationsWithEmptyUserValue", pod: core.Pod{ ObjectMeta: meta.ObjectMeta{ - Name: podName, + Name: podName, }, }, filter: UnprotectedPodFilter("Protect="),