Merge branch 'kubernetes:master' into master

README.md

@@ -76,6 +76,7 @@ As of the 1.26 release, enhancements from this repo are visualized in the Enhanc
 
 Links:
 
+- [1.32 Milestone](https://bit.ly/k8s132-enhancements)
 - [1.31 Milestone](https://bit.ly/k8s131-enhancements)
 - [1.30 Milestone](https://bit.ly/k8s130-enhancements)
 - [1.29 Milestone](https://bit.ly/k8s129-enhancements)

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.1.1
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/oauth2 v0.0.0-20210112200429-01de73cf58bd
+	golang.org/x/oauth2 v0.21.0
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	k8s.io/release v0.7.1-0.20210218090651-d71805402dab
 	k8s.io/test-infra v0.0.0-20200813194141-e9678d500461
@@ -62,7 +62,6 @@ require (
 	golang.org/x/sys v0.0.0-20210112080510-489259a85091 // indirect
 	golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -34,6 +34,7 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
@@ -606,8 +607,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE=
 github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs=
 github.com/google/go-containerregistry v0.0.0-20200123184029-53ce695e4179/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs=
@@ -1474,8 +1476,9 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210112200429-01de73cf58bd h1:0n2rzLq6xLtV9OFaT0BF2syUkjOwRrJ1zvXY5hH7Kkc=
 golang.org/x/oauth2 v0.0.0-20210112200429-01de73cf58bd/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1745,7 +1748,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
 google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
 google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

keps/prod-readiness/sig-auth/4633.yaml

@@ -1,3 +1,5 @@
 kep-number: 4633
 alpha:
   approver: "@jpbetz"
+beta:
+  approver: "@jpbetz"

keps/sig-api-machinery/3157-watch-list/README.md

@@ -92,6 +92,7 @@ tags, and then generate with `hack/update-toc.sh`.
     - [Results with WATCH-LIST](#results-with-watch-list)
   - [Required changes for a WATCH request with the RV set to the last observed value (RV > 0)](#required-changes-for-a-watch-request-with-the-rv-set-to-the-last-observed-value-rv--0)
   - [Provide a fix for the long-standing issue <a href="https://github.com/kubernetes/kubernetes/issues/59848">https://github.com/kubernetes/kubernetes/issues/59848</a>](#provide-a-fix-for-the-long-standing-issue-httpsgithubcomkuberneteskubernetesissues59848)
+  - [Replacing standard List request with WatchList mechanism for client-go's List method.](#replacing-standard-list-request-with-watchlist-mechanism-for-client-gos-list-method)
   - [Test Plan](#test-plan)
       - [Prerequisite testing updates](#prerequisite-testing-updates)
       - [Unit tests](#unit-tests)
@@ -581,6 +582,73 @@ Then on the server side we:
 3. reject the request if waitUntilFreshAndBlock times out, thus forcing informers to retry.
 4. otherwise, construct the final list and send back to a client.
 
+### Replacing standard List request with WatchList mechanism for client-go's List method.
+
+Replacing the underlying implementation of the List method for client-go based clients (like typed or dynamic client) 
+with the WatchList mechanism requires ensuring that the data returned by both the standard List request and 
+the new WatchList mechanism remains identical. The challenge is that WatchList no longer retrieves the entire 
+list from the server at once but only receives individual items, which forces us to "manually" reconstruct 
+the list object on the client side.
+
+To correctly construct the list object on the client side, we need ListKind information.
+However, simply reconstructing the list object based on these data is not enough. 
+In the case of a standard List request, the server's response (a versioned list) is processed through a chain of decoders, 
+which can potentially modify the resulting list object. 
+A good example is the WithoutVersionDecoder, which removes the GVK information from the list object.
+Thus the "manually" constructed list object may not be consistent 
+with the transformations applied by the decoders, leading to differences.
+
+To ensure full compatibility, the server must provide a versioned empty list in the format requested by the client (e.g., protobuf representation). 
+We don't know how the client's decoder behaves for different encodings, i.e., whether the decoder actually supports 
+the encoding we intend to use for reconstruction. Therefore, to ensure maximal compatibility, we will ensure that 
+the encoding used for the reconstruction of the list matches the format that the client originally requested. 
+This guarantees that the returned list object can be correctly decoded by the client, 
+preserving the actual encoding format as intended.
+
+The proposed solution is to add a new annotation (`k8s.io/initial-events-list-blueprint`) to the object returned 
+in the bookmark event (The bookmark event is sent when the state is synced and marks the end of WatchList stream). 
+This annotation will store an empty, versioned list encoded as a Base64 string. 
+This annotation will be added to the same object/place the `k8s.io/initial-events-end` annotation is added.
+
+When the client receives such a bookmark, it will base64 decode the empty list and pass it to the decoder chain. 
+Only after a successful response from the decoders the list will be populated with data received from subsequent 
+watch events and returned.
+
+For example:
+```
+GET /api/v1/namespaces/test/pods?watch=1&sendInitialEvents=true&allowWatchBookmarks=true&resourceVersion=&resourceVersionMatch=NotOlderThan
+---
+200 OK
+Transfer-Encoding: chunked
+Content-Type: application/json
+
+{
+  "type": "ADDED",
+  "object": {"kind": "Pod", "apiVersion": "v1", "metadata": {"resourceVersion": "8467", "name": "foo"}, ...}
+}
+{
+  "type": "ADDED",
+  "object": {"kind": "Pod", "apiVersion": "v1", "metadata":     {"resourceVersion": "5726", "name": "bar"}, ...}
+}
+{
+"type":"BOOKMARK",
+"object":{"kind":"Pod","apiVersion":"v1","metadata":{"resourceVersion":"13519","annotations":{"k8s.io/initial-events-end":"true","k8s.io/initial-events-embedded-list":"eyJraW5kIjoiUG9kTGlzdCIsImFwaVZlcnNpb24iOiJ2MSIsIm1ldGFkYXRhIjp7fSwiaXRlbXMiOm51bGx9Cg=="}} ...}
+}
+...
+<followed by regular watch stream starting>
+```
+
+**Alternatives**
+
+We could modify the type of the object passed in the last bookmark event to include the list. 
+This approach would require changes to the reflector, as it would need to recognize the new object type in the bookmark event. 
+However, this could potentially break other clients that are not expecting a different object in the bookmark event.
+
+Another option would be to issue an empty list request to the API server to receive a list response from the client. 
+This approach would involve modifying client-go and implementing some form of caching mechanism, 
+possibly with invalidation policies. 
+Non-client-go clients that want to use this new feature would need to rebuild this mechanism as well.
+
 ### Test Plan
 <!--
 **Note:** *Not required until targeted at a release.*
@@ -675,13 +743,18 @@ We expect no non-infra related flakes in the last month as a GA graduation crite
   with data obtained through a standard list request. The detector will be added to the reflector 
   and activated when an environment variable is set. The environment variable will be set for all jobs run in the Kube CI. 
 - Update the client-go generated List function to watchList data when the feature gate has been enabled and the ListOptions are satisfied.
+  This change must be applied to the typed, dynamic and metadata clients.
 - Implement a mechanism for automatically detecting etcd configuration
   Whether it is safe to use the RequestWatchProgress API call 
   or if the experimental-watch-progress-notify-interval flag has been set.
   Knowing etcd configuration will be used to automatically disable the streaming feature.
 - Use WatchProgressRequester to request progress notifications directly from etcd.
   This mechanism was developed in [Consistent Reads from Cache KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/2340-Consistent-reads-from-cache#use-requestprogress-to-enable-automatic-watch-updates)
   and will reduce the overall latency for watchlist requests.
+- The watchlist call, which serves as a drop-in replacement for list calls in client libraries, 
+  must properly set the kind and apiVersion fields. 
+  These fields are important for the correct decoding of the objects.
+  See also: https://github.com/kubernetes/kubernetes/pull/126191
 
 #### GA
 - [Switch](https://github.com/kubernetes/kubernetes/blob/a07b1aaa5b39b351ec8586de800baa5715304a3f/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go#L416) 

keps/sig-api-machinery/3157-watch-list/kep.yaml

@@ -20,12 +20,12 @@ stage: beta
 # The most recent milestone for which work toward delivery of this KEP has been
 # done. This can be the current (upcoming) milestone, if it is being actively
 # worked on.
-latest-milestone: "v1.31"
+latest-milestone: "v1.32"
 
 # The milestone at which this feature was, or is targeted to be, at each stage.
 milestone:
   alpha: "v1.27"
-  beta: "v1.31"
+  beta: "v1.32"
 
 # The following PRR answers are required at alpha release
 # List the feature gate name and the components for which it must be enabled