feat: Refactor CLI code to handle force sync option

- Modified `String` method of `arrayForce` to join strings with a comma - Added import statements for packages "os" and "github.com/rs/zerolog" - Initialized logger with ConsoleWriter for standard error output - Added command line flag for "force" option - Logged message with tables to force sync if length of "forceSync" array is greater than 0
teamniteo · Aug 12, 2023 · 1c4fdac · 1c4fdac
1 parent 8ae0c5e
commit 1c4fdac
Show file tree

Hide file tree

Showing 17 changed files with 411 additions and 133 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -10,8 +10,6 @@ on:
     branches: ["main"]
 
 jobs:
-  vuln:
-    uses: cristalhq/.github/.github/workflows/[email protected]
 
   build:
     runs-on: ubuntu-latest
@@ -48,7 +46,7 @@ jobs:
         run: make build
 
       - name: Test
-        run: go test -v ./...
+        run: go test -timeout 30s -v ./...
 
       - name: Upload assets
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/vuln.yml b/.github/workflows/vuln.yml
@@ -0,0 +1,11 @@
+name: vuln
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  vuln:
+    uses: cristalhq/.github/.github/workflows/[email protected]
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 # pg-subsetter
 
-[![lint](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml) [![build](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml)
+[![lint](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml) [![build](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml) [![vuln](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml)
 
 
-`pg-subsetter` is a powerful and efficient tool designed to synchronize a fraction of a PostgreSQL database to another PostgreSQL database on the fly, it does not copy the SCHEMA, this means that your target database has to have schema populated in some other way.
+`pg-subsetter` is a tool designed to synchronize a fraction of a PostgreSQL database to another PostgreSQL database on the fly, it does not copy the SCHEMA. This means that your target database has to have schema populated in some other way.
 
 ### Database Fraction Synchronization
 `pg-subsetter` allows you to select and sync a specific subset of your database. Whether it's a fraction of a table or a particular dataset, you can have it replicated in another database without synchronizing the entire DB.
@@ -12,7 +12,7 @@
 Foreign keys play a vital role in maintaining the relationships between tables. `pg-subsetter` ensures that all foreign keys are handled correctly during the synchronization process, maintaining the integrity and relationships of the data.
 
 ### Efficient COPY Method
-Utilizing the native PostgreSQL COPY command, pg-subsetter performs data transfer with high efficiency. This method significantly speeds up the synchronization process, minimizing downtime and resource consumption.
+Utilizing the native PostgreSQL COPY command, `pg-subsetter` performs data transfer with high efficiency. This method significantly speeds up the synchronization process, minimizing downtime and resource consumption.
 
 ### Stateless Operation
 `pg-subsetter` is built to be stateless, meaning it does not maintain any internal state between runs. This ensures that each synchronization process is independent, enhancing reliability and making it easier to manage and scale.
@@ -23,17 +23,26 @@ Utilizing the native PostgreSQL COPY command, pg-subsetter performs data transfe
 ```
 Usage of subsetter:
   -dst string
-        Destination DSN
+        Destination database DSN
   -f float
         Fraction of rows to copy (default 0.05)
+  -force value
+        Query to copy required tables (users: id = 1)
   -src string
-        Source DSN
+        Source database DSN
 ```
 
 
-Example:
+### Example
 
-```pg-subsetter -src postgresql://:@/bigdb -dst postgresql://:@/littledb -f 0.05```
+Copy a fraction of the database and force certain rows to be also copied over.
+
+```
+pg-subsetter \
+      -src "postgres://test_source@localhost:5432/test_source?sslmode=disable" \
+      -dst "postgres://test_target@localhost:5432/test_target?sslmode=disable" \
+      -f 0.05
+```
 
 # Installing
 

diff --git a/cli/force.go b/cli/force.go
@@ -0,0 +1,24 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"niteo.co/subsetter/subsetter"
+)
+
+type arrayForce []subsetter.Force
+
+func (i *arrayForce) String() string {
+	return fmt.Sprintf("%v", *i)
+}
+
+func (i *arrayForce) Set(value string) error {
+	q := strings.SplitAfter(strings.TrimSpace(value), ":")
+
+	*i = append(*i, subsetter.Force{
+		Table: q[0],
+		Where: q[1],
+	})
+	return nil
+}
diff --git a/cli/main.go b/cli/main.go
@@ -2,17 +2,26 @@ package main
 
 import (
 	"flag"
+	"os"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/pkgerrors"
+	"niteo.co/subsetter/subsetter"
 )
 
 var src = flag.String("src", "", "Source database DSN")
 var dst = flag.String("dst", "", "Destination database DSN")
 var fraction = flag.Float64("f", 0.05, "Fraction of rows to copy")
+var verbose = flag.Bool("verbose", true, "Show more information during sync")
+var forceSync arrayForce
 
 func main() {
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
+
+	flag.Var(&forceSync, "force", "Query to copy required tables (users: id = 1)")
 	flag.Parse()
-	log.Info().Msg("Starting")
 
 	if *src == "" || *dst == "" {
 		log.Fatal().Msg("Source and destination DSNs are required")
@@ -22,4 +31,20 @@ func main() {
 		log.Fatal().Msg("Fraction must be between 0 and 1")
 	}
 
+	if len(forceSync) > 0 {
+		log.Info().Str("forced", forceSync.String()).Msg("Forcing sync for tables")
+	}
+
+	s, err := subsetter.NewSync(*src, *dst, *fraction, forceSync, *verbose)
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("Failed to configure sync")
+	}
+
+	defer s.Close()
+
+	err = s.Sync()
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("Failed to sync")
+	}
+
 }
diff --git a/flake.nix b/flake.nix
@@ -37,7 +37,6 @@
               golangci-lint
               postgresql
               process-compose
-              shellcheck
               nixpkgs-fmt
               pgweb
             ];

diff --git a/go.mod b/go.mod
@@ -7,7 +7,10 @@ require github.com/rs/zerolog v1.30.0
 require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
+	golang.org/x/sync v0.1.0 // indirect
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -8,10 +8,13 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
 github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -28,6 +31,8 @@ golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
 golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=

diff --git a/subsetter/db_test.go b/subsetter/db_test.go
@@ -4,24 +4,65 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 
-	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
 )
 
-func getTestConnection() *pgx.Conn {
+var testConnSrc *pgxpool.Pool
+var onceTestSrc sync.Once
+var testConnTrg *pgxpool.Pool
+var onceTestTrg sync.Once
+
+func getTestConnection() *pgxpool.Pool {
 	DATABASE_URL := os.Getenv("DATABASE_URL")
 	if DATABASE_URL == "" {
 		DATABASE_URL = "postgres://test_source@localhost:5432/test_source?sslmode=disable"
 	}
 
-	conn, err := pgx.Connect(context.Background(), DATABASE_URL)
-	if err != nil {
-		panic(err)
+	onceTestSrc.Do(func() {
+		c, err := pgxpool.New(context.Background(), DATABASE_URL)
+		if err != nil {
+			panic(err)
+		}
+		testConnSrc = c
+	})
+
+	return testConnSrc
+}
+
+func getRealConnection(url string) *pgxpool.Pool {
+
+	onceTestSrc.Do(func() {
+		c, err := pgxpool.New(context.Background(), url)
+		if err != nil {
+			panic(err)
+		}
+		testConnSrc = c
+	})
+
+	return testConnSrc
+}
+
+func getTestConnectionDst() *pgxpool.Pool {
+	DATABASE_URL := os.Getenv("DATABASE_URL")
+	if DATABASE_URL == "" {
+		DATABASE_URL = "postgres://test_target@localhost:5432/test_target?sslmode=disable"
 	}
-	return conn
+
+	onceTestTrg.Do(func() {
+		c, err := pgxpool.New(context.Background(), DATABASE_URL)
+		if err != nil {
+			panic(err)
+		}
+		testConnTrg = c
+	})
+
+	return testConnTrg
 }
 
-func populateTests(conn *pgx.Conn) {
+func initSchema(conn *pgxpool.Pool) {
+
 	_, err := conn.Exec(context.Background(), `
 		CREATE TABLE simple (
 			id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -41,7 +82,7 @@ func populateTests(conn *pgx.Conn) {
 	}
 }
 
-func populateTestsWithData(conn *pgx.Conn, table string, size int) {
+func populateTestsWithData(conn *pgxpool.Pool, table string, size int) {
 	for i := 0; i < size; i++ {
 		query := fmt.Sprintf("INSERT INTO %s (text) VALUES ('test%d') RETURNING id", table, i)
 		var row string
@@ -58,7 +99,7 @@ func populateTestsWithData(conn *pgx.Conn, table string, size int) {
 	}
 }
 
-func clearPopulateTests(conn *pgx.Conn) {
+func clearSchema(conn *pgxpool.Pool) {
 	_, err := conn.Exec(context.Background(), `
 		ALTER TABLE relation DROP CONSTRAINT relation_simple_fk;
 		DROP TABLE simple;

diff --git a/subsetter/info.go b/subsetter/info.go
@@ -8,8 +8,9 @@ func GetTargetSet(fraction float64, tables []Table) []Table {
 
 	for _, table := range tables {
 		subset = append(subset, Table{
-			Name: table.Name,
-			Rows: int(math.Pow(10, math.Log10(float64(table.Rows))*fraction)),
+			Name:      table.Name,
+			Rows:      int(math.Pow(10, math.Log10(float64(table.Rows))*fraction)),
+			Relations: table.Relations,
 		})
 	}
 

diff --git a/subsetter/info_test.go b/subsetter/info_test.go
@@ -1,24 +1,22 @@
 package subsetter
 
 import (
-	"context"
 	"testing"
 )
 
 func TestGetTargetSet(t *testing.T) {
 	conn := getTestConnection()
-	populateTests(conn)
-	defer conn.Close(context.Background())
-	defer clearPopulateTests(conn)
+	initSchema(conn)
+	defer clearSchema(conn)
 
 	tests := []struct {
 		name     string
 		fraction float64
 		tables   []Table
 		want     []Table
 	}{
-		{"simple", 0.5, []Table{{"simple", 1000, []string{}}}, []Table{{"simple", 31, []string{}}}},
-		{"simple", 0.5, []Table{{"simple", 10, []string{}}}, []Table{{"simple", 3, []string{}}}},
+		{"simple", 0.5, []Table{{"simple", 1000, []Relation{}}}, []Table{{"simple", 31, []Relation{}}}},
+		{"simple", 0.5, []Table{{"simple", 10, []Relation{}}}, []Table{{"simple", 3, []Relation{}}}},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {