Add seed to discovery service protocol (#3479)

discovery/api/server/api.go

@@ -71,11 +71,12 @@ func (w *Wrapper) GetPresentations(ctx context.Context, request GetPresentations
 		timestamp = *request.Params.Timestamp
 	}
 
-	presentations, newTimestamp, err := w.Server.Get(contextWithForwardedHost(ctx), request.ServiceID, timestamp)
+	presentations, seed, newTimestamp, err := w.Server.Get(contextWithForwardedHost(ctx), request.ServiceID, timestamp)
 	if err != nil {
 		return nil, err
 	}
 	return GetPresentations200JSONResponse{
+		Seed:      seed,
 		Entries:   presentations,
 		Timestamp: newTimestamp,
 	}, nil

discovery/api/server/api_test.go

@@ -35,22 +35,24 @@ const serviceID = "wonderland"
 func TestWrapper_GetPresentations(t *testing.T) {
 	lastTimestamp := 1
 	presentations := map[string]vc.VerifiablePresentation{}
+	seed := "seed"
 	ctx := context.Background()
 	t.Run("no timestamp", func(t *testing.T) {
 		test := newMockContext(t)
-		test.server.EXPECT().Get(gomock.Any(), serviceID, 0).Return(presentations, lastTimestamp, nil)
+		test.server.EXPECT().Get(gomock.Any(), serviceID, 0).Return(presentations, seed, lastTimestamp, nil)
 
 		response, err := test.wrapper.GetPresentations(ctx, GetPresentationsRequestObject{ServiceID: serviceID})
 
 		require.NoError(t, err)
 		require.IsType(t, GetPresentations200JSONResponse{}, response)
 		assert.Equal(t, lastTimestamp, response.(GetPresentations200JSONResponse).Timestamp)
 		assert.Equal(t, presentations, response.(GetPresentations200JSONResponse).Entries)
+		assert.Equal(t, seed, response.(GetPresentations200JSONResponse).Seed)
 	})
 	t.Run("with timestamp", func(t *testing.T) {
 		givenTimestamp := 1
 		test := newMockContext(t)
-		test.server.EXPECT().Get(gomock.Any(), serviceID, 1).Return(presentations, lastTimestamp, nil)
+		test.server.EXPECT().Get(gomock.Any(), serviceID, 1).Return(presentations, seed, lastTimestamp, nil)
 
 		response, err := test.wrapper.GetPresentations(ctx, GetPresentationsRequestObject{
 			ServiceID: serviceID,
@@ -66,7 +68,7 @@ func TestWrapper_GetPresentations(t *testing.T) {
 	})
 	t.Run("error", func(t *testing.T) {
 		test := newMockContext(t)
-		test.server.EXPECT().Get(gomock.Any(), serviceID, 0).Return(nil, 0, errors.New("foo"))
+		test.server.EXPECT().Get(gomock.Any(), serviceID, 0).Return(nil, "", 0, errors.New("foo"))
 
 		_, err := test.wrapper.GetPresentations(ctx, GetPresentationsRequestObject{ServiceID: serviceID})
 

discovery/api/server/client/http.go

@@ -71,31 +71,31 @@ func (h DefaultHTTPClient) Register(ctx context.Context, serviceEndpointURL stri
 	return nil
 }
 
-func (h DefaultHTTPClient) Get(ctx context.Context, serviceEndpointURL string, timestamp int) (map[string]vc.VerifiablePresentation, int, error) {
+func (h DefaultHTTPClient) Get(ctx context.Context, serviceEndpointURL string, timestamp int) (map[string]vc.VerifiablePresentation, string, int, error) {
 	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, serviceEndpointURL, nil)
 	httpRequest.URL.RawQuery = url.Values{"timestamp": []string{fmt.Sprintf("%d", timestamp)}}.Encode()
 	if err != nil {
-		return nil, 0, err
+		return nil, "", 0, err
 	}
 	httpRequest.Header.Set("X-Forwarded-Host", httpRequest.Host) // prevent cycles
 	httpResponse, err := h.client.Do(httpRequest)
 	if err != nil {
-		return nil, 0, fmt.Errorf("failed to invoke remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", 0, fmt.Errorf("failed to invoke remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
 	defer httpResponse.Body.Close()
 	if err := core.TestResponseCode(200, httpResponse); err != nil {
 		httpErr := err.(core.HttpError) // TestResponseCodeWithLog always returns an HttpError
-		return nil, 0, fmt.Errorf("non-OK response from remote Discovery Service (url=%s): %s", serviceEndpointURL, problemResponseToError(httpErr))
+		return nil, "", 0, fmt.Errorf("non-OK response from remote Discovery Service (url=%s): %s", serviceEndpointURL, problemResponseToError(httpErr))
 	}
 	responseData, err := io.ReadAll(httpResponse.Body)
 	if err != nil {
-		return nil, 0, fmt.Errorf("failed to read response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", 0, fmt.Errorf("failed to read response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
 	var result PresentationsResponse
 	if err := json.Unmarshal(responseData, &result); err != nil {
-		return nil, 0, fmt.Errorf("failed to unmarshal response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", 0, fmt.Errorf("failed to unmarshal response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
-	return result.Entries, result.Timestamp, nil
+	return result.Entries, result.Seed, result.Timestamp, nil
 }
 
 // problemResponseToError converts a Problem Details response to an error.

discovery/api/server/client/http_test.go

@@ -79,29 +79,32 @@ func TestHTTPInvoker_Get(t *testing.T) {
 	t.Run("no timestamp from client", func(t *testing.T) {
 		handler := &testHTTP.Handler{StatusCode: http.StatusOK}
 		handler.ResponseData = map[string]interface{}{
+			"seed":      "seed",
 			"entries":   map[string]interface{}{"1": vp},
 			"timestamp": 1,
 		}
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		presentations, timestamp, err := client.Get(context.Background(), server.URL, 0)
+		presentations, seed, timestamp, err := client.Get(context.Background(), server.URL, 0)
 
 		assert.NoError(t, err)
 		assert.Len(t, presentations, 1)
 		assert.Equal(t, "0", handler.RequestQuery.Get("timestamp"))
 		assert.Equal(t, 1, timestamp)
+		assert.Equal(t, "seed", seed)
 	})
 	t.Run("timestamp provided by client", func(t *testing.T) {
 		handler := &testHTTP.Handler{StatusCode: http.StatusOK}
 		handler.ResponseData = map[string]interface{}{
+			"seed":      "seed",
 			"entries":   map[string]interface{}{"1": vp},
 			"timestamp": 1,
 		}
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		presentations, timestamp, err := client.Get(context.Background(), server.URL, 1)
+		presentations, _, timestamp, err := client.Get(context.Background(), server.URL, 1)
 
 		assert.NoError(t, err)
 		assert.Len(t, presentations, 1)
@@ -119,7 +122,7 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(http.HandlerFunc(handler))
 		client := New(false, time.Minute, server.TLS)
 
-		_, _, err := client.Get(context.Background(), server.URL, 0)
+		_, _, _, err := client.Get(context.Background(), server.URL, 0)
 
 		require.NoError(t, err)
 		assert.True(t, strings.HasPrefix(capturedRequest.Header.Get("X-Forwarded-Host"), "127.0.0.1"))
@@ -129,7 +132,7 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		_, _, err := client.Get(context.Background(), server.URL, 0)
+		_, _, _, err := client.Get(context.Background(), server.URL, 0)
 
 		assert.ErrorContains(t, err, "non-OK response from remote Discovery Service")
 		assert.ErrorContains(t, err, "server returned HTTP status code 500")
@@ -141,7 +144,7 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		_, _, err := client.Get(context.Background(), server.URL, 0)
+		_, _, _, err := client.Get(context.Background(), server.URL, 0)
 
 		assert.ErrorContains(t, err, "failed to unmarshal response from remote Discovery Service")
 	})

discovery/api/server/client/interface.go

@@ -31,5 +31,5 @@ type HTTPClient interface {
 	// Get retrieves Verifiable Presentations from the remote Discovery Service, that were added since the given timestamp.
 	// If the call succeeds it returns the Verifiable Presentations and the timestamp that was returned by the server.
 	// If the given timestamp is 0, all Verifiable Presentations are retrieved.
-	Get(ctx context.Context, serviceEndpointURL string, timestamp int) (map[string]vc.VerifiablePresentation, int, error)
+	Get(ctx context.Context, serviceEndpointURL string, timestamp int) (map[string]vc.VerifiablePresentation, string, int, error)
 }

discovery/api/server/client/types.go

@@ -24,6 +24,8 @@ import "github.com/nuts-foundation/go-did/vc"
 type PresentationsResponse struct {
 	// Entries contains mappings from timestamp (as string) to a VerifiablePresentation.
 	Entries map[string]vc.VerifiablePresentation `json:"entries"`
+	// Seed is a unique value for the combination of serviceID and a server instance.
+	Seed string `json:"seed"`
 	// Timestamp is the timestamp of the latest entry. It's not a unix timestamp but a Lamport Clock.
 	Timestamp int `json:"timestamp"`
 }

discovery/client.go

@@ -367,16 +367,21 @@ func (u *clientUpdater) updateService(ctx context.Context, service ServiceDefini
 	log.Logger().
 		WithField("discoveryService", service.ID).
 		Tracef("Checking for new Verifiable Presentations from Discovery Service (timestamp: %d)", currentTimestamp)
-	presentations, serverTimestamp, err := u.client.Get(ctx, service.Endpoint, currentTimestamp)
+	presentations, seed, serverTimestamp, err := u.client.Get(ctx, service.Endpoint, currentTimestamp)
 	if err != nil {
 		return fmt.Errorf("failed to get presentations from discovery service (id=%s): %w", service.ID, err)
 	}
+	// check testSeed in store, wipe if it's different. Done by the store for transaction safety.
+	err = u.store.wipeOnSeedChange(service.ID, seed)
+	if err != nil {
+		return fmt.Errorf("failed to wipe on testSeed change (service=%s, testSeed=%s): %w", service.ID, seed, err)
+	}
 	for _, presentation := range presentations {
 		if err := u.verifier(service, presentation); err != nil {
 			log.Logger().WithError(err).Warnf("Presentation verification failed, not adding it (service=%s, id=%s)", service.ID, presentation.ID)
 			continue
 		}
-		if err := u.store.add(service.ID, presentation, serverTimestamp); err != nil {
+		if err := u.store.add(service.ID, presentation, seed, serverTimestamp); err != nil {
 			return fmt.Errorf("failed to store presentation (service=%s, id=%s): %w", service.ID, presentation.ID, err)
 		}
 		log.Logger().

discovery/client_test.go

@@ -221,7 +221,7 @@ func Test_defaultClientRegistrationManager_deactivate(t *testing.T) {
 		ctx.invoker.EXPECT().Register(gomock.Any(), gomock.Any(), gomock.Any())
 		ctx.wallet.EXPECT().BuildPresentation(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(&vpAlice, nil)
 		ctx.subjectManager.EXPECT().ListDIDs(gomock.Any(), aliceSubject).Return([]did.DID{aliceDID}, nil)
-		require.NoError(t, ctx.store.add(testServiceID, vpAlice, 1))
+		require.NoError(t, ctx.store.add(testServiceID, vpAlice, testSeed, 1))
 
 		err := ctx.manager.deactivate(audit.TestContext(), testServiceID, aliceSubject)
 
@@ -236,7 +236,7 @@ func Test_defaultClientRegistrationManager_deactivate(t *testing.T) {
 			claims["retract_jti"] = vpAlice.ID.String()
 			vp.Type = append(vp.Type, retractionPresentationType)
 		}, vcAlice)
-		require.NoError(t, ctx.store.add(testServiceID, vpAliceDeactivated, 1))
+		require.NoError(t, ctx.store.add(testServiceID, vpAliceDeactivated, testSeed, 1))
 
 		err := ctx.manager.deactivate(audit.TestContext(), testServiceID, aliceSubject)
 
@@ -255,7 +255,7 @@ func Test_defaultClientRegistrationManager_deactivate(t *testing.T) {
 		ctx.invoker.EXPECT().Register(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("remote error"))
 		ctx.wallet.EXPECT().BuildPresentation(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(&vpAlice, nil)
 		ctx.subjectManager.EXPECT().ListDIDs(gomock.Any(), aliceSubject).Return([]did.DID{aliceDID}, nil)
-		require.NoError(t, ctx.store.add(testServiceID, vpAlice, 1))
+		require.NoError(t, ctx.store.add(testServiceID, vpAlice, testSeed, 1))
 
 		err := ctx.manager.deactivate(audit.TestContext(), testServiceID, aliceSubject)
 
@@ -266,7 +266,7 @@ func Test_defaultClientRegistrationManager_deactivate(t *testing.T) {
 		ctx := newTestContext(t)
 		ctx.wallet.EXPECT().BuildPresentation(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil, assert.AnError)
 		ctx.subjectManager.EXPECT().ListDIDs(gomock.Any(), aliceSubject).Return([]did.DID{aliceDID}, nil)
-		require.NoError(t, ctx.store.add(testServiceID, vpAlice, 1))
+		require.NoError(t, ctx.store.add(testServiceID, vpAlice, testSeed, 1))
 
 		err := ctx.manager.deactivate(audit.TestContext(), testServiceID, aliceSubject)
 
@@ -394,7 +394,7 @@ func Test_clientUpdater_updateService(t *testing.T) {
 		httpClient := client.NewMockHTTPClient(ctrl)
 		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
 
-		httpClient.EXPECT().Get(ctx, testDefinitions()[testServiceID].Endpoint, 0).Return(map[string]vc.VerifiablePresentation{}, 0, nil)
+		httpClient.EXPECT().Get(ctx, testDefinitions()[testServiceID].Endpoint, 0).Return(map[string]vc.VerifiablePresentation{}, testSeed, 0, nil)
 
 		err := updater.updateService(ctx, testDefinitions()[testServiceID])
 
@@ -406,7 +406,7 @@ func Test_clientUpdater_updateService(t *testing.T) {
 		httpClient := client.NewMockHTTPClient(ctrl)
 		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
 
-		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 0).Return(map[string]vc.VerifiablePresentation{"1": vpAlice}, 1, nil)
+		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 0).Return(map[string]vc.VerifiablePresentation{"1": vpAlice}, testSeed, 1, nil)
 
 		err := updater.updateService(ctx, testDefinitions()[testServiceID])
 
@@ -423,7 +423,7 @@ func Test_clientUpdater_updateService(t *testing.T) {
 			return nil
 		}, httpClient)
 
-		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 0).Return(map[string]vc.VerifiablePresentation{"1": vpAlice, "2": vpBob}, 2, nil)
+		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 0).Return(map[string]vc.VerifiablePresentation{"1": vpAlice, "2": vpBob}, testSeed, 2, nil)
 
 		err := updater.updateService(ctx, testDefinitions()[testServiceID])
 
@@ -440,28 +440,49 @@ func Test_clientUpdater_updateService(t *testing.T) {
 		resetStore(t, storageEngine.GetSQLDatabase())
 		ctrl := gomock.NewController(t)
 		httpClient := client.NewMockHTTPClient(ctrl)
-		err := store.setTimestamp(store.db, testServiceID, 1)
+		err := store.setTimestamp(store.db, testServiceID, testSeed, 1)
 		require.NoError(t, err)
 		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
 
-		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 1).Return(map[string]vc.VerifiablePresentation{"1": vpAlice}, 1, nil)
+		httpClient.EXPECT().Get(ctx, serviceDefinition.Endpoint, 1).Return(map[string]vc.VerifiablePresentation{"1": vpAlice}, testSeed, 1, nil)
 
 		err = updater.updateService(ctx, testDefinitions()[testServiceID])
 
 		require.NoError(t, err)
 	})
+	t.Run("seed change wipes entries", func(t *testing.T) {
+		resetStore(t, storageEngine.GetSQLDatabase())
+		ctrl := gomock.NewController(t)
+		httpClient := client.NewMockHTTPClient(ctrl)
+		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
+		store.add(testServiceID, vpAlice, testSeed, 0)
+
+		exists, err := store.exists(testServiceID, aliceDID.String(), vpAlice.ID.String())
+		require.NoError(t, err)
+		require.True(t, exists)
+
+		httpClient.EXPECT().Get(ctx, testDefinitions()[testServiceID].Endpoint, 1).Return(map[string]vc.VerifiablePresentation{}, "other", 0, nil)
+
+		err = updater.updateService(ctx, testDefinitions()[testServiceID])
+
+		require.NoError(t, err)
+		exists, err = store.exists(testServiceID, aliceDID.String(), vpAlice.ID.String())
+		require.NoError(t, err)
+		require.False(t, exists)
+	})
 }
 
 func Test_clientUpdater_update(t *testing.T) {
+	seed := "seed"
 	t.Run("proceeds when service update fails", func(t *testing.T) {
 		storageEngine := storage.NewTestStorageEngine(t)
 		require.NoError(t, storageEngine.Start())
 		store := setupStore(t, storageEngine.GetSQLDatabase())
 		ctrl := gomock.NewController(t)
 		httpClient := client.NewMockHTTPClient(ctrl)
-		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/usecase", gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, 0, nil)
-		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/other", gomock.Any()).Return(nil, 0, errors.New("test"))
-		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/unsupported", gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, 0, nil)
+		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/usecase", gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, seed, 0, nil)
+		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/other", gomock.Any()).Return(nil, "", 0, errors.New("test"))
+		httpClient.EXPECT().Get(gomock.Any(), "http://example.com/unsupported", gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, seed, 0, nil)
 		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
 
 		err := updater.update(context.Background())
@@ -474,7 +495,7 @@ func Test_clientUpdater_update(t *testing.T) {
 		store := setupStore(t, storageEngine.GetSQLDatabase())
 		ctrl := gomock.NewController(t)
 		httpClient := client.NewMockHTTPClient(ctrl)
-		httpClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, 0, nil).MinTimes(2)
+		httpClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(map[string]vc.VerifiablePresentation{}, seed, 0, nil).MinTimes(2)
 		updater := newClientUpdater(testDefinitions(), store, alwaysOkVerifier, httpClient)
 
 		err := updater.update(context.Background())

discovery/interface.go

@@ -48,7 +48,7 @@ type Server interface {
 	Register(context context.Context, serviceID string, presentation vc.VerifiablePresentation) error
 	// Get retrieves the presentations for the given service, starting from the given timestamp.
 	// If the node is not configured as server for the given serviceID, the call will be forwarded to the configured server.
-	Get(context context.Context, serviceID string, startAfter int) (map[string]vc.VerifiablePresentation, int, error)
+	Get(context context.Context, serviceID string, startAfter int) (map[string]vc.VerifiablePresentation, string, int, error)
 }
 
 // Client defines the API for Discovery Clients.