chore: QLDB fixes

Query for the table existence rather than relying on error codes from
CREATE TABLE. Those are not stable and change between QLDB client versions.

docker-compose.payments.yml

@@ -17,6 +17,7 @@ services:
   localstack:
     container_name: localstack
     image: localstack/localstack
+    stop_grace_period: 1s
     ports:
       - "127.0.0.1:4566:4566"            # LocalStack Gateway
       - "127.0.0.1:4510-4559:4510-4559"  # external services port range
@@ -44,11 +45,13 @@ services:
     command: bat-go serve nitro inside-enclave --egress-address none --log-address none --upstream-url http://0.0.0.0:8080
     environment:
       - ENCLAVE_MOCKING=1
-      - AWS_REGION="us-west-2"
       - DEBUG=1
+      - ADDR=0.0.0.0:18080
+      - ADDR2=0.0.0.0:18443
+      - AWS_REGION="us-west-2"
       - QLDB_LEDGER_ARN=arn:aws:qldb:us-west-2:239563960694:ledger/testing-igor-payment
       - QLDB_LEDGER_NAME=testing-igor-payment
-      - QLDB_ROLE_ARN=arn:aws:iam::239563960694:role/aws-reserved/sso.amazonaws.com/us-west-2/AWSReservedSSO_BraveStlMetaSbxPaytSrvcsAccess_db468d5f79a737ee
+      - QLDB_ROLE_ARN=arn:aws:iam::239563960694:role/settlements-dev-sso-qldb-20240628161845683100000001
       - AWS_REGION=${AWS_REGION-}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-}

libs/nitro/vsock.go

@@ -247,14 +247,22 @@ func syncCopy(wg *sync.WaitGroup, dst io.WriteCloser, src io.ReadCloser) {
 
 func Listen(ctx context.Context, address string) (net.Listener, error) {
 	if enclaveMocking {
-		return net.Listen("tcp", "address")
+		l, err := net.Listen("tcp", address)
+		if err != nil {
+			return nil, fmt.Errorf("failed to listen to tcp address %v - %w", address, err)
+		}
+		return l, nil
 	}
 
 	// TODO: share with parseVsockAddr
 	port, err := strconv.ParseUint(strings.Split(address, ":")[1], 10, 32)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse vsock address - %w", err)
+		return nil, fmt.Errorf("failed to parse vsock address - %w", err)
 	}
 
-	return vsock.Listen(uint32(port), &vsock.Config{})
+	l, err := vsock.Listen(uint32(port), &vsock.Config{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to listen to vsock %v - %w", address, err)
+	}
+	return l, nil
 }

services/nitro/nitro.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
+	"os"
 	"runtime/debug"
 	"strconv"
 	"strings"
@@ -147,6 +148,7 @@ func RunNitroServerInEnclave(cmd *cobra.Command, args []string) error {
 				Str("panic", fmt.Sprintf("%+v", rec)).
 				Str("stacktrace", string(debug.Stack())).
 				Msg("panic recovered")
+			os.Exit(2)
 		}
 	}()
 
@@ -170,24 +172,25 @@ func RunNitroServerInEnclave(cmd *cobra.Command, args []string) error {
 	// setup vsock listener
 	httpListener, err := nitro.Listen(ctx, httpListenAddress)
 	if err != nil {
-		logger.Panic().Err(err).Msg("listening on vsock port failed")
+		logger.Fatal().Err(err).Msg("failed to listen to HTTP port")
 	}
 	httpsListener, err := nitro.Listen(ctx, httpsListenAddress)
 	if err != nil {
-		logger.Panic().Err(err).Msg("listening on vsock port failed")
+		logger.Fatal().Err(err).Msg("failed to listen to HTTPS port")
 	}
 	logger.Info().Msg("vsock listener setup")
 
 	tlsCertificate, err := createSelfSignedCertificate()
 	if err != nil {
-		logger.Panic().Err(err).Msg("failed to create a self-signed certoificate")
+		logger.Fatal().Err(err).Msg("failed to create a self-signed certificate")
 	}
 
 	// setup server
 	srv := http.Server{
 		Handler:      chi.ServerBaseContext(ctx, r),
 		ReadTimeout:  3 * time.Second,
 		WriteTimeout: 20 * time.Second,
+		TLSConfig:    &tls.Config{},
 	}
 	srv.TLSConfig.Certificates = []tls.Certificate{tlsCertificate}
 
@@ -230,7 +233,7 @@ func createSelfSignedCertificate() (tls.Certificate, error) {
 		BasicConstraintsValid: true,
 	}
 	template.DNSNames = []string{"nitro.localdomain"}
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, privateKey.PublicKey, privateKey)
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, privateKey.Public(), privateKey)
 	if err != nil {
 		return tls.Certificate{}, err
 	}

services/payments/datastore.go

@@ -4,11 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 
-	smithy "github.com/aws/smithy-go"
 	"github.com/brave-intl/bat-go/libs/logging"
 	appaws "github.com/brave-intl/bat-go/libs/nitro/aws"
+	"github.com/rs/zerolog"
 
 	"github.com/amazon-ion/ion-go/ion"
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -33,10 +32,6 @@ type QLDBDatastore struct {
 // ErrNotConfiguredYet - service not fully configured.
 var ErrNotConfiguredYet = errors.New("not yet configured")
 
-const (
-	tableAlreadyCreatedCode = "412"
-)
-
 type Datastore interface {
 	InsertPaymentState(ctx context.Context, state *paymentLib.PaymentState) (string, error)
 	InsertChainAddress(ctx context.Context, address ChainAddress) (string, error)
@@ -415,88 +410,77 @@ func (s *Service) configureDatastore(ctx context.Context) error {
 	return driver.setupLedger(ctx)
 }
 
+func ensureQLDBTable(
+	txn qldbdriver.Transaction,
+	tableName string,
+	logger *zerolog.Logger,
+) (bool, error) {
+	// Check for table existence. Note that in QLDB DROP TABLE does not delete
+	// the table but merely marks it as inactive and it is still reachable by
+	// its id.
+	//
+	// NOTE: previously the code relied on CREATE TABLE reporting an error when
+	// table did not exist and then checked for a particular error code to
+	// distinguish between non-existing table and other errors. But error codes
+	// are not stable with QLDB, so explicit query against the
+	// information_schema should be more robust.
+	r, err := txn.Execute("SELECT tableId FROM information_schema.user_tables WHERE name = ? AND status = 'ACTIVE'", tableName)
+	if err != nil {
+		return false, fmt.Errorf(
+			"failed to query user_tables for %s: %w", tableName, err)
+	}
+	if r.Next(txn) {
+		// We have an entry for the table so it exists.
+		return false, nil
+	}
+
+	logger.Warn().Err(err).Msgf("Failed to locate %s table, creating one", tableName)
+	_, err = txn.Execute(fmt.Sprintf("CREATE TABLE %s", tableName))
+	if err != nil {
+		return false, fmt.Errorf(
+			"failed to create %s table due to: %w", tableName, err)
+	}
+	return true, nil
+}
+
 func (q *QLDBDatastore) setupLedger(ctx context.Context) error {
 	logger := logging.Logger(ctx, "payments.setupLedger")
 
-	fmt.Printf("# X Y %s %s", os.Getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN"), os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI"))
-
 	// create the tables needed in the ledger
 	_, err := q.Execute(context.Background(), func(txn qldbdriver.Transaction) (interface{}, error) {
-		logger.Debug().Msg("ensuring QLDB ledger tables exists")
-		var (
-			ae smithy.APIError
-			ok bool
-		)
-		_, err := txn.Execute("CREATE TABLE transactions")
+		logger.Debug().Msg("ensuring QLDB ledger tables exist")
+		created, err := ensureQLDBTable(txn, "transactions", logger)
 		if err != nil {
-			logger.Warn().Err(err).Msg("error creating transactions table")
-			if errors.As(err, &ae) {
-				logger.Warn().Err(err).Str("code", ae.ErrorCode()).Msg("api error creating transactions table")
-				if ae.ErrorCode() == tableAlreadyCreatedCode {
-					// table has already been created
-					ok = true
-				}
-			}
-			if !ok {
-				return nil, fmt.Errorf("failed to create transactions table due to: %w", err)
-			}
-		} else {
+			return nil, err
+		}
+		if created {
 			_, err = txn.Execute("CREATE INDEX ON transactions (idempotencyKey)")
 			if err != nil {
 				return nil, err
 			}
 		}
 
-		ok = false
-		_, err = txn.Execute("CREATE TABLE authorizations")
+		_, err = ensureQLDBTable(txn, "authorizations", logger)
 		if err != nil {
-			logger.Warn().Err(err).Msg("error creating authorizationss table")
-			if errors.As(err, &ae) {
-				logger.Warn().Err(err).Str("code", ae.ErrorCode()).Msg("api error creating authorizations table")
-				if ae.ErrorCode() == tableAlreadyCreatedCode {
-					// table has already been created
-					ok = true
-				}
-			}
-			if !ok {
-				return nil, fmt.Errorf("failed to create authorizations table due to: %w", err)
-			}
+			return nil, err
 		}
-		ok = false
-		_, err = txn.Execute("CREATE TABLE chainaddresses")
+
+		created, err = ensureQLDBTable(txn, "chainaddresses", logger)
 		if err != nil {
-			logger.Warn().Err(err).Msg("error creating chainaddresses table")
-			if errors.As(err, &ae) {
-				logger.Warn().Err(err).Str("code", ae.ErrorCode()).Msg("api error creating chainaddresses table")
-				if ae.ErrorCode() == tableAlreadyCreatedCode {
-					// table has already been created
-					ok = true
-				}
-			}
-			if !ok {
-				return nil, fmt.Errorf("failed to create chainaddresses table due to: %w", err)
-			}
-		} else {
+			return nil, err
+		}
+		if created {
 			_, err = txn.Execute("CREATE INDEX ON chainaddresses (publicKey)")
 			if err != nil {
 				return nil, err
 			}
 		}
-		ok = false
-		_, err = txn.Execute("CREATE TABLE vaults")
+
+		created, err = ensureQLDBTable(txn, "vaults", logger)
 		if err != nil {
-			logger.Warn().Err(err).Msg("error creating vaults table")
-			if errors.As(err, &ae) {
-				logger.Warn().Err(err).Str("code", ae.ErrorCode()).Msg("api error creating vaults table")
-				if ae.ErrorCode() == tableAlreadyCreatedCode {
-					// table has already been created
-					ok = true
-				}
-			}
-			if !ok {
-				return nil, fmt.Errorf("failed to create vaults table due to: %w", err)
-			}
-		} else {
+			return nil, err
+		}
+		if created {
 			_, err = txn.Execute("CREATE INDEX ON vaults (publicKey)")
 			if err != nil {
 				return nil, err
@@ -509,7 +493,7 @@ func (q *QLDBDatastore) setupLedger(ctx context.Context) error {
 		return nil, nil
 	})
 	if err != nil {
-		return fmt.Errorf("failed to create tables: %w", err)
+		return fmt.Errorf("failed to create QLDB ledge tables: %w", err)
 	}
 	return nil
 }