From 478a8ee695dc1a30ddfc18f796393769d2a8fc24 Mon Sep 17 00:00:00 2001 From: reinkrul Date: Sun, 12 Nov 2023 13:41:07 +0100 Subject: [PATCH] SQL: introduce SQLite storage (#2595) --- Dockerfile | 5 +- README.rst | 1 + docs/pages/deployment/cli-reference.rst | 3 +- docs/pages/deployment/server_options.rst | 1 + .../deployment/storage-configuration.rst | 16 ++++ go.mod | 11 ++- go.sum | 21 +++-- storage/cmd/cmd.go | 1 + storage/config.go | 7 ++ storage/engine.go | 87 +++++++++++++++++-- storage/engine_test.go | 55 +++++++++++- storage/interface.go | 4 + storage/mock.go | 15 ++++ storage/sql_migrations/1_initial.down.sql | 1 + storage/sql_migrations/1_initial.up.sql | 1 + storage/sql_migrations/README.md | 9 ++ storage/test.go | 6 +- 17 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 storage/sql_migrations/1_initial.down.sql create mode 100644 storage/sql_migrations/1_initial.up.sql create mode 100644 storage/sql_migrations/README.md diff --git a/Dockerfile b/Dockerfile index d97ebc5c7f..78c0645711 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,9 @@ ARG GIT_VERSION=undefined LABEL maintainer="wout.slakhorst@nuts.nl" RUN apk update \ + && apk add --no-cache \ + gcc \ + musl-dev \ && update-ca-certificates ENV GO111MODULE on @@ -22,7 +25,7 @@ COPY go.sum . RUN go mod download && go mod verify COPY . . -RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-w -s -X 'github.com/nuts-foundation/nuts-node/core.GitCommit=${GIT_COMMIT}' -X 'github.com/nuts-foundation/nuts-node/core.GitBranch=${GIT_BRANCH}' -X 'github.com/nuts-foundation/nuts-node/core.GitVersion=${GIT_VERSION}'" -o /opt/nuts/nuts +RUN CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags="-w -s -X 'github.com/nuts-foundation/nuts-node/core.GitCommit=${GIT_COMMIT}' -X 'github.com/nuts-foundation/nuts-node/core.GitBranch=${GIT_BRANCH}' -X 'github.com/nuts-foundation/nuts-node/core.GitVersion=${GIT_VERSION}'" -o /opt/nuts/nuts # alpine FROM alpine:3.18.4 diff --git a/README.rst b/README.rst index ddee309491..f6316b814b 100644 --- a/README.rst +++ b/README.rst @@ -252,6 +252,7 @@ The following options can be configured on the server: storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index 83c30879d8..0bf3c272ea 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -44,7 +44,7 @@ The following options apply to the server commands below: --http.default.log string What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). (default "metadata") --http.default.tls string Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', --internalratelimiter When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. (default true) - --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson]) + --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson]) --jsonld.contexts.remoteallowlist strings In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. (default [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json]) --loggerformat string Log format (text, json) (default "text") --network.bootstrapnodes strings List of bootstrap nodes (':') which the node initially connect to. @@ -70,6 +70,7 @@ The following options apply to the server commands below: --storage.redis.sentinel.username string Username for authenticating to Redis Sentinels. --storage.redis.tls.truststorefile string PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). --storage.redis.username string Redis database username. If set, it overrides the username in the connection URL. + --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory --strictmode When set, insecure settings are forbidden. (default true) --tls.certfile string PEM file containing the certificate for the server (also used as client certificate). --tls.certheader string Name of the HTTP header that will contain the client certificate when TLS is offloaded. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index f389b0d124..03b274bb46 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -78,6 +78,7 @@ storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. diff --git a/docs/pages/deployment/storage-configuration.rst b/docs/pages/deployment/storage-configuration.rst index 22dfece03b..8bec008f61 100644 --- a/docs/pages/deployment/storage-configuration.rst +++ b/docs/pages/deployment/storage-configuration.rst @@ -51,6 +51,22 @@ The server's certificate will be verified against the OS' CA bundle. Make sure to `configure persistence for your Redis server `_. +SQL +=== + +.. note:: + + SQL storage is still in development, for now you'll still need the other storage options described by this document. + +As we're transitioning to protocols with less shared state, we foresee Nuts' data models to become more relational. +To simplify things, we intent to move towards SQL based storage in the future. +The first database to be supported in SQLite, to aid development and demo/workshop setups. Other, supported SQL databases might be: +- MySQL family (MariaDB, Percona) +- PostgreSQL + +By default, storage SQLite will be used in a file called ``sqlite.db`` in the configured data directory. +This can be overridden by configuring a connection string in ``storage.sql.connection`` (only SQLite for now). + Redis Sentinel ^^^^^^^^^^^^^^ diff --git a/go.mod b/go.mod index 3aa6ee78ba..ee60992699 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,11 @@ require ( github.com/chromedp/chromedp v0.9.3 github.com/dlclark/regexp2 v1.10.0 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang-migrate/migrate/v4 v4.16.2 github.com/goodsign/monday v1.0.1 github.com/google/uuid v1.4.0 github.com/hashicorp/vault/api v1.10.0 + github.com/jinzhu/now v1.1.5 // indirect github.com/knadh/koanf v1.5.0 github.com/labstack/echo/v4 v4.11.3 github.com/lestrrat-go/jwx/v2 v2.0.16 @@ -33,6 +35,7 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 github.com/redis/go-redis/v9 v9.3.0 + github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -49,9 +52,8 @@ require ( gopkg.in/Regis24GmbH/go-phonetics.v2 v2.0.3 gopkg.in/yaml.v3 v3.0.1 schneider.vip/problem v1.8.1 -) -require github.com/santhosh-tekuri/jsonschema v1.2.4 +) require ( github.com/PaesslerAG/gval v1.2.2 // indirect @@ -90,7 +92,7 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -117,6 +119,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect @@ -165,5 +168,7 @@ require ( golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/Regis24GmbH/go-diacritics.v2 v2.0.3 // indirect + gorm.io/driver/sqlite v1.5.4 + gorm.io/gorm v1.25.5 rsc.io/qr v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index f300ffbf24..e6d0c82292 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= -github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA= +github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -218,8 +220,8 @@ github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUz github.com/goodsign/monday v1.0.1 h1:yJogH0uQNn4blHjoC3ESbdV0P1OhDtGYdd6x0w7QZBo= github.com/goodsign/monday v1.0.1/go.mod h1:r4T4breXpoFwspQNM+u2sLxJb2zyTaxVGqUfTBjWOu8= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -300,8 +302,9 @@ github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -612,8 +615,8 @@ go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -854,6 +857,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= +gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= diff --git a/storage/cmd/cmd.go b/storage/cmd/cmd.go index 911aa3396e..9cf8d59626 100644 --- a/storage/cmd/cmd.go +++ b/storage/cmd/cmd.go @@ -38,5 +38,6 @@ func FlagSet() *pflag.FlagSet { flagSet.StringSlice("storage.redis.sentinel.nodes", defs.Redis.Sentinel.Nodes, "Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel.") flagSet.String("storage.redis.sentinel.username", defs.Redis.Sentinel.Username, "Username for authenticating to Redis Sentinels.") flagSet.String("storage.redis.sentinel.password", defs.Redis.Sentinel.Password, "Password for authenticating to Redis Sentinels.") + flagSet.String("storage.sql.connection", defs.SQL.ConnectionString, "Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory") return flagSet } diff --git a/storage/config.go b/storage/config.go index f62ff4c35a..d703ffa9cf 100644 --- a/storage/config.go +++ b/storage/config.go @@ -22,9 +22,16 @@ package storage type Config struct { BBolt BBoltConfig `koanf:"bbolt"` Redis RedisConfig `koanf:"redis"` + SQL SQLConfig `koanf:"sql"` } // DefaultConfig returns the default configuration for the module. func DefaultConfig() Config { return Config{} } + +// SQLConfig specifies config for the SQL storage engine. +type SQLConfig struct { + // ConnectionString is the connection string for the SQL database. + ConnectionString string `koanf:"connection"` +} diff --git a/storage/engine.go b/storage/engine.go index 3a00578de3..fecd2db7d6 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -20,12 +20,20 @@ package storage import ( "context" + "embed" "errors" "fmt" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/sqlite3" + "github.com/golang-migrate/migrate/v4/source/iofs" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage/log" "github.com/redis/go-redis/v9" + "github.com/sirupsen/logrus" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "path" "strings" "sync" "time" @@ -33,6 +41,9 @@ import ( const storeShutdownTimeout = 5 * time.Second +//go:embed sql_migrations/*.sql +var sqlMigrationsFS embed.FS + // New creates a new instance of the storage engine. func New() Engine { return &engine{ @@ -48,6 +59,7 @@ type engine struct { stores map[string]stoabs.Store databases []database sessionDatabase SessionDatabase + sqlDB *gorm.DB config Config } @@ -56,18 +68,22 @@ func (e *engine) Config() interface{} { } // Name returns the name of the engine. -func (e engine) Name() string { +func (e *engine) Name() string { return "Storage" } -func (e engine) Start() error { +func (e *engine) Start() error { + if err := e.initSQLDatabase(); err != nil { + return fmt.Errorf("failed to initialize SQL database: %w", err) + } return nil } -func (e engine) Shutdown() error { +func (e *engine) Shutdown() error { e.storesMux.Lock() defer e.storesMux.Unlock() + // Close KV stores shutdown := func(store stoabs.Store) error { // Refactored to separate function, otherwise defer would be in for loop which leaks resources. ctx, cancel := context.WithTimeout(context.Background(), storeShutdownTimeout) @@ -91,8 +107,16 @@ func (e engine) Shutdown() error { return errors.New("one or more stores failed to close") } + // Close session database e.sessionDatabase.close() - + // Close SQL db + if e.sqlDB != nil { + underlyingDB, err := e.sqlDB.DB() + if err != nil { + return err + } + return underlyingDB.Close() + } return nil } @@ -114,7 +138,6 @@ func (e *engine) Configure(config core.ServerConfig) error { return fmt.Errorf("unable to configure BBolt database: %w", err) } e.databases = append(e.databases, bboltDB) - return nil } @@ -129,6 +152,49 @@ func (e *engine) GetSessionDatabase() SessionDatabase { return e.sessionDatabase } +func (e *engine) GetSQLDatabase() *gorm.DB { + return e.sqlDB +} + +// initSQLDatabase initializes the SQL database connection. +// If the connection string is not configured, it defaults to a SQLite database, stored in the node's data directory. +// Note: only SQLite is supported for now +func (e *engine) initSQLDatabase() error { + connectionString := e.config.SQL.ConnectionString + if len(connectionString) == 0 { + connectionString = "file:" + path.Join(e.datadir, "sqlite.db") + } + var err error + e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{}) + if err != nil { + return err + } + log.Logger().Debug("Running database migrations...") + underlyingDB, err := e.sqlDB.DB() + if err != nil { + return err + } + sourceDriver, err := iofs.New(sqlMigrationsFS, "sql_migrations") + if err != nil { + return err + } + databaseDriver, err := sqlite3.WithInstance(underlyingDB, &sqlite3.Config{}) + if err != nil { + return err + } + migrations, err := migrate.NewWithInstance("iofs", sourceDriver, e.sqlDB.Name(), databaseDriver) + if err != nil { + return err + } + migrations.Log = sqlMigrationLogger{} + err = migrations.Up() + if errors.Is(err, migrate.ErrNoChange) { + // There was nothing to migrate + return nil + } + return err +} + type provider struct { moduleName string engine *engine @@ -183,3 +249,14 @@ func (p *provider) getStore(moduleName string, name string, adapter database) (s } return store, err } + +type sqlMigrationLogger struct { +} + +func (m sqlMigrationLogger) Printf(format string, v ...interface{}) { + log.Logger().Infof(format, v...) +} + +func (m sqlMigrationLogger) Verbose() bool { + return log.Logger().Level >= logrus.DebugLevel +} diff --git a/storage/engine_test.go b/storage/engine_test.go index 6b010bea1b..82f1af7524 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -22,9 +22,12 @@ import ( "errors" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" + "github.com/nuts-foundation/nuts-node/test/io" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "os" + "path" "testing" ) @@ -33,7 +36,7 @@ func Test_New(t *testing.T) { } func Test_engine_Name(t *testing.T) { - assert.Equal(t, "Storage", engine{}.Name()) + assert.Equal(t, "Storage", (&engine{}).Name()) } func Test_engine_lifecycle(t *testing.T) { @@ -96,3 +99,53 @@ func Test_engine_Shutdown(t *testing.T) { assert.EqualError(t, err, "one or more stores failed to close") }) } + +func Test_engine_sqlDatabase(t *testing.T) { + t.Run("defaults to SQLite in data directory", func(t *testing.T) { + e := New() + dataDir := io.TestDirectory(t) + require.NoError(t, e.Configure(core.ServerConfig{Datadir: dataDir})) + require.NoError(t, e.Start()) + t.Cleanup(func() { + _ = e.Shutdown() + }) + assert.FileExists(t, path.Join(dataDir, "sqlite.db")) + }) + t.Run("unable to open SQLite database", func(t *testing.T) { + dataDir := io.TestDirectory(t) + require.NoError(t, os.Remove(dataDir)) + e := New() + require.NoError(t, e.Configure(core.ServerConfig{Datadir: dataDir})) + err := e.Start() + assert.EqualError(t, err, "failed to initialize SQL database: unable to open database file") + }) + t.Run("nothing to migrate (already migrated)", func(t *testing.T) { + dataDir := io.TestDirectory(t) + e := New() + require.NoError(t, e.Configure(core.ServerConfig{Datadir: dataDir})) + require.NoError(t, e.Start()) + require.NoError(t, e.Shutdown()) + e = New() + require.NoError(t, e.Configure(core.ServerConfig{Datadir: dataDir})) + require.NoError(t, e.Start()) + require.NoError(t, e.Shutdown()) + }) + t.Run("runs migrations", func(t *testing.T) { + e := New().(*engine) + e.config.SQL.ConnectionString = SQLiteInMemoryConnectionString + require.NoError(t, e.Configure(*core.NewServerConfig())) + require.NoError(t, e.Start()) + t.Cleanup(func() { + _ = e.Shutdown() + }) + + underlyingDB, err := e.GetSQLDatabase().DB() + require.NoError(t, err) + row := underlyingDB.QueryRow("SELECT count(*) FROM schema_migrations") + require.NoError(t, row.Err()) + var count int + assert.NoError(t, row.Scan(&count)) + assert.Equal(t, 1, count) + }) + +} diff --git a/storage/interface.go b/storage/interface.go index e23888542f..83ab900ad0 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/nuts-foundation/go-stoabs" "github.com/nuts-foundation/nuts-node/core" + "gorm.io/gorm" "time" ) @@ -37,6 +38,9 @@ type Engine interface { GetProvider(moduleName string) Provider // GetSessionDatabase returns the SessionDatabase GetSessionDatabase() SessionDatabase + + // GetSQLDatabase returns the SQL database. + GetSQLDatabase() *gorm.DB } // Provider lets callers get access to stores. diff --git a/storage/mock.go b/storage/mock.go index 559941d7bd..a4aa1c0bfa 100644 --- a/storage/mock.go +++ b/storage/mock.go @@ -15,6 +15,7 @@ import ( stoabs "github.com/nuts-foundation/go-stoabs" core "github.com/nuts-foundation/nuts-node/core" gomock "go.uber.org/mock/gomock" + gorm "gorm.io/gorm" ) // MockEngine is a mock of Engine interface. @@ -82,6 +83,20 @@ func (mr *MockEngineMockRecorder) GetSessionDatabase() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionDatabase", reflect.TypeOf((*MockEngine)(nil).GetSessionDatabase)) } +// SQLDatabase mocks base method. +func (m *MockEngine) GetSQLDatabase() *gorm.DB { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSQLDatabase") + ret0, _ := ret[0].(*gorm.DB) + return ret0 +} + +// SQLDatabase indicates an expected call of SQLDatabase. +func (mr *MockEngineMockRecorder) SQLDatabase() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSQLDatabase", reflect.TypeOf((*MockEngine)(nil).GetSQLDatabase)) +} + // Shutdown mocks base method. func (m *MockEngine) Shutdown() error { m.ctrl.T.Helper() diff --git a/storage/sql_migrations/1_initial.down.sql b/storage/sql_migrations/1_initial.down.sql new file mode 100644 index 0000000000..6bb32b4a81 --- /dev/null +++ b/storage/sql_migrations/1_initial.down.sql @@ -0,0 +1 @@ +select 1 \ No newline at end of file diff --git a/storage/sql_migrations/1_initial.up.sql b/storage/sql_migrations/1_initial.up.sql new file mode 100644 index 0000000000..6bb32b4a81 --- /dev/null +++ b/storage/sql_migrations/1_initial.up.sql @@ -0,0 +1 @@ +select 1 \ No newline at end of file diff --git a/storage/sql_migrations/README.md b/storage/sql_migrations/README.md new file mode 100644 index 0000000000..535333d49a --- /dev/null +++ b/storage/sql_migrations/README.md @@ -0,0 +1,9 @@ +This directory contains SQL schema migrations, run at startup of the node. + +Refer to https://github.com/golang-migrate/migrate/blob/master/MIGRATIONS.md on how to write migrations. + +Files should be named according to the following: `__..sql`. +For instance: `2_usecase_list.up.sql`. + +AVOID changing migrations in master (unless the migration breaks the node horribly) for those running a `master` version. +DO NOT alter migrations in a released version: it might break vendor deployments or cause data corruption. \ No newline at end of file diff --git a/storage/test.go b/storage/test.go index 95fb052a18..34cc9a0ef7 100644 --- a/storage/test.go +++ b/storage/test.go @@ -28,8 +28,12 @@ import ( "testing" ) +// SQLiteInMemoryConnectionString is a connection string for an in-memory SQLite database +const SQLiteInMemoryConnectionString = "file::memory:?cache=shared" + func NewTestStorageEngineInDir(dir string) Engine { - result := New() + result := New().(*engine) + result.config.SQL = SQLConfig{ConnectionString: SQLiteInMemoryConnectionString} _ = result.Configure(core.TestServerConfig(core.ServerConfig{Datadir: dir + "/data"})) return result }