From 107249f7a0f35793cb4dc8c7f146687c36ab2dc6 Mon Sep 17 00:00:00 2001 From: Ayu Date: Wed, 31 Jul 2024 11:57:40 +0300 Subject: [PATCH] feat(db): support custom properties metadata (#108) * feat: add events table * refactor: prepared statements are called from a map * refactor: lazy load prepared statements * fix(db): do not use reserved keyword group * test: add event test db call * fix: restrict custom prop types to string number bool * perf: commit multiple properties in a single transaction * lint: remove cyclop * fix: add batch id for each unique event for better grouping * style: remove newline --- core/.golangci.yml | 9 +- core/api/oas_json_gen.go | 266 ++++++++++++++++++++++++++ core/api/oas_schemas_gen.go | 169 ++++++++++++++++ core/api/oas_validators_gen.go | 2 + core/db/db.go | 1 + core/db/duckdb/client.go | 49 +++-- core/db/duckdb/event.go | 105 ++++++---- core/db/duckdb/event_test.go | 29 +++ core/go.mod | 1 + core/go.sum | 4 +- core/migrations/0004_duckdb_events.go | 72 +++++++ core/migrations/migrations.go | 1 + core/model/event.go | 11 ++ core/openapi.yaml | 25 +++ core/services/event.go | 58 ++++++ dashboard/app/api/types.d.ts | 21 +- 16 files changed, 760 insertions(+), 63 deletions(-) create mode 100644 core/migrations/0004_duckdb_events.go diff --git a/core/.golangci.yml b/core/.golangci.yml index 398dc456..d2b51a07 100644 --- a/core/.golangci.yml +++ b/core/.golangci.yml @@ -6,11 +6,6 @@ run: # This file contains only configs which differ from defaults. # All possible options can be found here https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml linters-settings: - cyclop: - # The maximal code complexity to report. - # Default: 10 - max-complexity: 40 - errcheck: # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. # Such cases aren't reported by default. @@ -117,7 +112,7 @@ linters: - bodyclose # checks whether HTTP response body is closed successfully - containedctx # checks for context.Context contained in a struct - contextcheck # checks for common mistakes using context - - cyclop # checks function and package cyclomatic complexity + # - cyclop # checks function and package cyclomatic complexity # - dupl # tool for code clone detection - durationcheck # checks for two durations multiplied together - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error @@ -133,7 +128,7 @@ linters: # - gocognit # computes and checks the cognitive complexity of functions - goconst # finds repeated strings that could be replaced by a constant - gocritic # provides diagnostics that check for bugs, performance and style issues - - gocyclo # computes and checks the cyclomatic complexity of functions + # - gocyclo # computes and checks the cyclomatic complexity of functions - gofumpt # checks if the code is formatted with gofumpt - godot # checks if comments end in a period - goimports # in addition to fixing imports, goimports also formats your code in the same style as gofmt diff --git a/core/api/oas_json_gen.go b/core/api/oas_json_gen.go index 60bef0cc..d3eb0026 100644 --- a/core/api/oas_json_gen.go +++ b/core/api/oas_json_gen.go @@ -542,6 +542,247 @@ func (s *ConflictErrorError) UnmarshalJSON(data []byte) error { return s.Decode(d) } +// Encode implements json.Marshaler. +func (s *EventCustom) Encode(e *jx.Encoder) { + e.ObjStart() + s.encodeFields(e) + e.ObjEnd() +} + +// encodeFields encodes fields. +func (s *EventCustom) encodeFields(e *jx.Encoder) { + { + e.FieldStart("g") + e.Str(s.G) + } + { + e.FieldStart("n") + e.Str(s.N) + } + { + e.FieldStart("p") + s.P.Encode(e) + } +} + +var jsonFieldsNameOfEventCustom = [3]string{ + 0: "g", + 1: "n", + 2: "p", +} + +// Decode decodes EventCustom from json. +func (s *EventCustom) Decode(d *jx.Decoder) error { + if s == nil { + return errors.New("invalid: unable to decode EventCustom to nil") + } + var requiredBitSet [1]uint8 + + if err := d.ObjBytes(func(d *jx.Decoder, k []byte) error { + switch string(k) { + case "g": + requiredBitSet[0] |= 1 << 0 + if err := func() error { + v, err := d.Str() + s.G = string(v) + if err != nil { + return err + } + return nil + }(); err != nil { + return errors.Wrap(err, "decode field \"g\"") + } + case "n": + requiredBitSet[0] |= 1 << 1 + if err := func() error { + v, err := d.Str() + s.N = string(v) + if err != nil { + return err + } + return nil + }(); err != nil { + return errors.Wrap(err, "decode field \"n\"") + } + case "p": + requiredBitSet[0] |= 1 << 2 + if err := func() error { + if err := s.P.Decode(d); err != nil { + return err + } + return nil + }(); err != nil { + return errors.Wrap(err, "decode field \"p\"") + } + default: + return d.Skip() + } + return nil + }); err != nil { + return errors.Wrap(err, "decode EventCustom") + } + // Validate required fields. + var failures []validate.FieldError + for i, mask := range [1]uint8{ + 0b00000111, + } { + if result := (requiredBitSet[i] & mask) ^ mask; result != 0 { + // Mask only required fields and check equality to mask using XOR. + // + // If XOR result is not zero, result is not equal to expected, so some fields are missed. + // Bits of fields which would be set are actually bits of missed fields. + missed := bits.OnesCount8(result) + for bitN := 0; bitN < missed; bitN++ { + bitIdx := bits.TrailingZeros8(result) + fieldIdx := i*8 + bitIdx + var name string + if fieldIdx < len(jsonFieldsNameOfEventCustom) { + name = jsonFieldsNameOfEventCustom[fieldIdx] + } else { + name = strconv.Itoa(fieldIdx) + } + failures = append(failures, validate.FieldError{ + Name: name, + Error: validate.ErrFieldRequired, + }) + // Reset bit. + result &^= 1 << bitIdx + } + } + } + if len(failures) > 0 { + return &validate.Error{Fields: failures} + } + + return nil +} + +// MarshalJSON implements stdjson.Marshaler. +func (s *EventCustom) MarshalJSON() ([]byte, error) { + e := jx.Encoder{} + s.Encode(&e) + return e.Bytes(), nil +} + +// UnmarshalJSON implements stdjson.Unmarshaler. +func (s *EventCustom) UnmarshalJSON(data []byte) error { + d := jx.DecodeBytes(data) + return s.Decode(d) +} + +// Encode implements json.Marshaler. +func (s EventCustomP) Encode(e *jx.Encoder) { + e.ObjStart() + s.encodeFields(e) + e.ObjEnd() +} + +// encodeFields implements json.Marshaler. +func (s EventCustomP) encodeFields(e *jx.Encoder) { + for k, elem := range s { + e.FieldStart(k) + + elem.Encode(e) + } +} + +// Decode decodes EventCustomP from json. +func (s *EventCustomP) Decode(d *jx.Decoder) error { + if s == nil { + return errors.New("invalid: unable to decode EventCustomP to nil") + } + m := s.init() + if err := d.ObjBytes(func(d *jx.Decoder, k []byte) error { + var elem EventCustomPItem + if err := func() error { + if err := elem.Decode(d); err != nil { + return err + } + return nil + }(); err != nil { + return errors.Wrapf(err, "decode field %q", k) + } + m[string(k)] = elem + return nil + }); err != nil { + return errors.Wrap(err, "decode EventCustomP") + } + + return nil +} + +// MarshalJSON implements stdjson.Marshaler. +func (s EventCustomP) MarshalJSON() ([]byte, error) { + e := jx.Encoder{} + s.Encode(&e) + return e.Bytes(), nil +} + +// UnmarshalJSON implements stdjson.Unmarshaler. +func (s *EventCustomP) UnmarshalJSON(data []byte) error { + d := jx.DecodeBytes(data) + return s.Decode(d) +} + +// Encode encodes EventCustomPItem as json. +func (s EventCustomPItem) Encode(e *jx.Encoder) { + switch s.Type { + case StringEventCustomPItem: + e.Str(s.String) + case IntEventCustomPItem: + e.Int(s.Int) + case BoolEventCustomPItem: + e.Bool(s.Bool) + } +} + +// Decode decodes EventCustomPItem from json. +func (s *EventCustomPItem) Decode(d *jx.Decoder) error { + if s == nil { + return errors.New("invalid: unable to decode EventCustomPItem to nil") + } + // Sum type type_discriminator. + switch t := d.Next(); t { + case jx.Bool: + v, err := d.Bool() + s.Bool = bool(v) + if err != nil { + return err + } + s.Type = BoolEventCustomPItem + case jx.Number: + v, err := d.Int() + s.Int = int(v) + if err != nil { + return err + } + s.Type = IntEventCustomPItem + case jx.String: + v, err := d.Str() + s.String = string(v) + if err != nil { + return err + } + s.Type = StringEventCustomPItem + default: + return errors.Errorf("unexpected json type %q", t) + } + return nil +} + +// MarshalJSON implements stdjson.Marshaler. +func (s EventCustomPItem) MarshalJSON() ([]byte, error) { + e := jx.Encoder{} + s.Encode(&e) + return e.Bytes(), nil +} + +// UnmarshalJSON implements stdjson.Unmarshaler. +func (s *EventCustomPItem) UnmarshalJSON(data []byte) error { + d := jx.DecodeBytes(data) + return s.Decode(d) +} + // Encode encodes EventHit as json. func (s EventHit) Encode(e *jx.Encoder) { e.ObjStart() @@ -551,6 +792,24 @@ func (s EventHit) Encode(e *jx.Encoder) { func (s EventHit) encodeFields(e *jx.Encoder) { switch s.Type { + case EventCustomEventHit: + e.FieldStart("e") + e.Str("custom") + { + s := s.EventCustom + { + e.FieldStart("g") + e.Str(s.G) + } + { + e.FieldStart("n") + e.Str(s.N) + } + { + e.FieldStart("p") + s.P.Encode(e) + } + } case EventLoadEventHit: e.FieldStart("e") e.Str("load") @@ -625,6 +884,9 @@ func (s *EventHit) Decode(d *jx.Decoder) error { return err } switch typ { + case "custom": + s.Type = EventCustomEventHit + found = true case "load": s.Type = EventLoadEventHit found = true @@ -653,6 +915,10 @@ func (s *EventHit) Decode(d *jx.Decoder) error { if err := s.EventUnload.Decode(d); err != nil { return err } + case EventCustomEventHit: + if err := s.EventCustom.Decode(d); err != nil { + return err + } default: return errors.Errorf("inferred invalid type: %s", s.Type) } diff --git a/core/api/oas_schemas_gen.go b/core/api/oas_schemas_gen.go index 4eda7ddb..cfc86096 100644 --- a/core/api/oas_schemas_gen.go +++ b/core/api/oas_schemas_gen.go @@ -167,6 +167,149 @@ type DeleteWebsitesIDNoContent struct{} func (*DeleteWebsitesIDNoContent) deleteWebsitesIDRes() {} +// Event with custom properties. +// Ref: #/components/schemas/EventCustom +type EventCustom struct { + // Group name of events. Currently, only the hostname is supported. + G string `json:"g"` + // Event name or key. + N string `json:"n"` + // Event properties. + P EventCustomP `json:"p"` +} + +// GetG returns the value of G. +func (s *EventCustom) GetG() string { + return s.G +} + +// GetN returns the value of N. +func (s *EventCustom) GetN() string { + return s.N +} + +// GetP returns the value of P. +func (s *EventCustom) GetP() EventCustomP { + return s.P +} + +// SetG sets the value of G. +func (s *EventCustom) SetG(val string) { + s.G = val +} + +// SetN sets the value of N. +func (s *EventCustom) SetN(val string) { + s.N = val +} + +// SetP sets the value of P. +func (s *EventCustom) SetP(val EventCustomP) { + s.P = val +} + +// Event properties. +type EventCustomP map[string]EventCustomPItem + +func (s *EventCustomP) init() EventCustomP { + m := *s + if m == nil { + m = map[string]EventCustomPItem{} + *s = m + } + return m +} + +// EventCustomPItem represents sum type. +type EventCustomPItem struct { + Type EventCustomPItemType // switch on this field + String string + Int int + Bool bool +} + +// EventCustomPItemType is oneOf type of EventCustomPItem. +type EventCustomPItemType string + +// Possible values for EventCustomPItemType. +const ( + StringEventCustomPItem EventCustomPItemType = "string" + IntEventCustomPItem EventCustomPItemType = "int" + BoolEventCustomPItem EventCustomPItemType = "bool" +) + +// IsString reports whether EventCustomPItem is string. +func (s EventCustomPItem) IsString() bool { return s.Type == StringEventCustomPItem } + +// IsInt reports whether EventCustomPItem is int. +func (s EventCustomPItem) IsInt() bool { return s.Type == IntEventCustomPItem } + +// IsBool reports whether EventCustomPItem is bool. +func (s EventCustomPItem) IsBool() bool { return s.Type == BoolEventCustomPItem } + +// SetString sets EventCustomPItem to string. +func (s *EventCustomPItem) SetString(v string) { + s.Type = StringEventCustomPItem + s.String = v +} + +// GetString returns string and true boolean if EventCustomPItem is string. +func (s EventCustomPItem) GetString() (v string, ok bool) { + if !s.IsString() { + return v, false + } + return s.String, true +} + +// NewStringEventCustomPItem returns new EventCustomPItem from string. +func NewStringEventCustomPItem(v string) EventCustomPItem { + var s EventCustomPItem + s.SetString(v) + return s +} + +// SetInt sets EventCustomPItem to int. +func (s *EventCustomPItem) SetInt(v int) { + s.Type = IntEventCustomPItem + s.Int = v +} + +// GetInt returns int and true boolean if EventCustomPItem is int. +func (s EventCustomPItem) GetInt() (v int, ok bool) { + if !s.IsInt() { + return v, false + } + return s.Int, true +} + +// NewIntEventCustomPItem returns new EventCustomPItem from int. +func NewIntEventCustomPItem(v int) EventCustomPItem { + var s EventCustomPItem + s.SetInt(v) + return s +} + +// SetBool sets EventCustomPItem to bool. +func (s *EventCustomPItem) SetBool(v bool) { + s.Type = BoolEventCustomPItem + s.Bool = v +} + +// GetBool returns bool and true boolean if EventCustomPItem is bool. +func (s EventCustomPItem) GetBool() (v bool, ok bool) { + if !s.IsBool() { + return v, false + } + return s.Bool, true +} + +// NewBoolEventCustomPItem returns new EventCustomPItem from bool. +func NewBoolEventCustomPItem(v bool) EventCustomPItem { + var s EventCustomPItem + s.SetBool(v) + return s +} + // Website hit event. // Ref: #/components/schemas/EventHit // EventHit represents sum type. @@ -174,6 +317,7 @@ type EventHit struct { Type EventHitType // switch on this field EventLoad EventLoad EventUnload EventUnload + EventCustom EventCustom } // EventHitType is oneOf type of EventHit. @@ -183,6 +327,7 @@ type EventHitType string const ( EventLoadEventHit EventHitType = "load" EventUnloadEventHit EventHitType = "unload" + EventCustomEventHit EventHitType = "custom" ) // IsEventLoad reports whether EventHit is EventLoad. @@ -191,6 +336,9 @@ func (s EventHit) IsEventLoad() bool { return s.Type == EventLoadEventHit } // IsEventUnload reports whether EventHit is EventUnload. func (s EventHit) IsEventUnload() bool { return s.Type == EventUnloadEventHit } +// IsEventCustom reports whether EventHit is EventCustom. +func (s EventHit) IsEventCustom() bool { return s.Type == EventCustomEventHit } + // SetEventLoad sets EventHit to EventLoad. func (s *EventHit) SetEventLoad(v EventLoad) { s.Type = EventLoadEventHit @@ -233,6 +381,27 @@ func NewEventUnloadEventHit(v EventUnload) EventHit { return s } +// SetEventCustom sets EventHit to EventCustom. +func (s *EventHit) SetEventCustom(v EventCustom) { + s.Type = EventCustomEventHit + s.EventCustom = v +} + +// GetEventCustom returns EventCustom and true boolean if EventHit is EventCustom. +func (s EventHit) GetEventCustom() (v EventCustom, ok bool) { + if !s.IsEventCustom() { + return v, false + } + return s.EventCustom, true +} + +// NewEventCustomEventHit returns new EventHit from EventCustom. +func NewEventCustomEventHit(v EventCustom) EventHit { + var s EventHit + s.SetEventCustom(v) + return s +} + // Page view load event. // Ref: #/components/schemas/EventLoad type EventLoad struct { diff --git a/core/api/oas_validators_gen.go b/core/api/oas_validators_gen.go index 251dcb69..960f6c4e 100644 --- a/core/api/oas_validators_gen.go +++ b/core/api/oas_validators_gen.go @@ -69,6 +69,8 @@ func (s EventHit) Validate() error { return err } return nil + case EventCustomEventHit: + return nil // no validation needed default: return errors.Errorf("invalid type %q", s.Type) } diff --git a/core/db/db.go b/core/db/db.go index 88d3008b..73eb4983 100644 --- a/core/db/db.go +++ b/core/db/db.go @@ -46,6 +46,7 @@ type AnalyticsClient interface { GetSettingsUsage(ctx context.Context) (*model.GetSettingsUsage, error) PatchSettingsUsage(ctx context.Context, settings *model.GetSettingsUsage) error // Events + AddEvents(ctx context.Context, event *[]model.EventHit) error AddPageView(ctx context.Context, event *model.PageViewHit) error UpdatePageView(ctx context.Context, event *model.PageViewDuration) error // Pages diff --git a/core/db/duckdb/client.go b/core/db/duckdb/client.go index 348054d6..6f3b4f35 100644 --- a/core/db/duckdb/client.go +++ b/core/db/duckdb/client.go @@ -1,6 +1,9 @@ package duckdb import ( + "context" + + "github.com/alphadose/haxmap" "github.com/go-faster/errors" "github.com/jmoiron/sqlx" "github.com/medama-io/medama/db" @@ -8,6 +11,8 @@ import ( type Client struct { *sqlx.DB + // Map of prepared statements. + statements *haxmap.Map[string, *sqlx.NamedStmt] } // Compile time check for Client. @@ -36,30 +41,40 @@ func NewClient(host string) (*Client, error) { } return &Client{ - DB: db, + DB: db, + statements: haxmap.New[string, *sqlx.NamedStmt](), }, nil } // Close closes the database connection and any prepared statements. func (c *Client) Close() error { - // Helper function to close a statement and wrap any error. - closeStmt := func(stmt *sqlx.NamedStmt) error { - if stmt != nil { - if err := stmt.Close(); err != nil { - return errors.Wrap(err, "duckdb") - } - } - return nil - } - // Close the statements. - if err := closeStmt(addStmt); err != nil { - return err - } - if err := closeStmt(updateStmt); err != nil { - return err - } + c.closeStatements() // Close the database connection. return c.DB.Close() } + +// GetPreparedStmt returns a prepared statement by name. This is lazy loaded and cached after +// the first call. +func (c *Client) GetPreparedStmt(ctx context.Context, name string, query string) (*sqlx.NamedStmt, error) { + stmt, ok := c.statements.Get(name) + if ok { + return stmt, nil + } + + stmt, err := c.DB.PrepareNamedContext(ctx, query) + if err != nil { + return nil, errors.Wrap(err, "unable to create prepared statement") + } + + c.statements.Set(name, stmt) + return stmt, nil +} + +func (c *Client) closeStatements() { + c.statements.ForEach(func(_ string, stmt *sqlx.NamedStmt) bool { + stmt.Close() + return true + }) +} diff --git a/core/db/duckdb/event.go b/core/db/duckdb/event.go index b5be371c..9488573e 100644 --- a/core/db/duckdb/event.go +++ b/core/db/duckdb/event.go @@ -2,31 +2,73 @@ package duckdb import ( "context" - "sync" "github.com/go-faster/errors" - "github.com/jmoiron/sqlx" "github.com/medama-io/medama/model" - "github.com/medama-io/medama/util/logger" ) -var ( - //nolint:gochecknoglobals // Reason: Singleton patterns are typically written like this. - addOnce sync.Once - //nolint:gochecknoglobals // Reason: Prepared statements are meant to be global. - addStmt *sqlx.NamedStmt - //nolint:gochecknoglobals // Reason: Singleton patterns are typically written like this. - updateOnce sync.Once - //nolint:gochecknoglobals // Reason: Prepared statements are meant to be global. - updateStmt *sqlx.NamedStmt +const ( + addEventName = "addEvent" + addPageViewName = "addPageView" + updatePageViewName = "updatePageView" ) +// AddEvent adds an event with a custom property to the database. +func (c *Client) AddEvents(ctx context.Context, events *[]model.EventHit) error { + exec := `--sql + INSERT INTO events ( + batch_id, + group_name, + name, + value, + date_created + ) VALUES ( + :batch_id, + :group_name, + :name, + :value, + NOW() + )` + + stmt, err := c.GetPreparedStmt(ctx, addEventName, exec) + if err != nil { + return errors.Wrap(err, "duckdb") + } + + // Start a transaction for batch insert + tx, err := c.DB.BeginTxx(ctx, nil) + if err != nil { + return errors.Wrap(err, "duckdb: begin event hit transaction") + } + defer tx.Rollback() //nolint: errcheck // Called on defer + + txStmt := tx.NamedStmtContext(ctx, stmt) + + for _, event := range *events { + paramMap := map[string]interface{}{ + "batch_id": event.BatchID, + "group_name": event.Group, + "name": event.Name, + "value": event.Value, + } + + _, err = txStmt.ExecContext(ctx, paramMap) + if err != nil { + return errors.Wrap(err, "duckdb") + } + } + + err = tx.Commit() + if err != nil { + return errors.Wrap(err, "duckdb: commit event hit transaction") + } + + return nil +} + // AddPageView adds a page view to the database. func (c *Client) AddPageView(ctx context.Context, event *model.PageViewHit) error { - var err error - // Prepare named exec once. - addOnce.Do(func() { - exec := `--sql + exec := `--sql INSERT INTO views ( bid, hostname, @@ -65,13 +107,10 @@ func (c *Client) AddPageView(ctx context.Context, event *model.PageViewHit) erro NOW() )` - addStmt, err = c.DB.PrepareNamedContext(ctx, exec) - if err != nil { - log := logger.Get() - log.Error().Err(err).Msg("failed to create prepared statement for add page view") - panic("failed to create prepared statement for add page view") - } - }) + stmt, err := c.GetPreparedStmt(ctx, addPageViewName, exec) + if err != nil { + return errors.Wrap(err, "duckdb") + } paramMap := map[string]interface{}{ "bid": event.BID, @@ -92,7 +131,7 @@ func (c *Client) AddPageView(ctx context.Context, event *model.PageViewHit) erro "utm_campaign": event.UTMCampaign, } - _, err = addStmt.ExecContext(ctx, paramMap) + _, err = stmt.ExecContext(ctx, paramMap) if err != nil { return errors.Wrap(err, "db") } @@ -102,26 +141,20 @@ func (c *Client) AddPageView(ctx context.Context, event *model.PageViewHit) erro // UpdatePageView updates a page view in the database. func (c *Client) UpdatePageView(ctx context.Context, event *model.PageViewDuration) error { - var err error - // Prepare named exec once. - updateOnce.Do(func() { - exec := `--sql + exec := `--sql UPDATE views SET duration_ms = :duration_ms WHERE bid = :bid` - updateStmt, err = c.DB.PrepareNamedContext(ctx, exec) - if err != nil { - log := logger.Get() - log.Error().Err(err).Msg("failed to create prepared statement for update page view") - panic(err) - } - }) + stmt, err := c.GetPreparedStmt(ctx, updatePageViewName, exec) + if err != nil { + return errors.Wrap(err, "db") + } paramMap := map[string]interface{}{ "bid": event.BID, "duration_ms": event.DurationMs, } - _, err = updateStmt.ExecContext(ctx, paramMap) + _, err = stmt.ExecContext(ctx, paramMap) if err != nil { return errors.Wrap(err, "db") } diff --git a/core/db/duckdb/event_test.go b/core/db/duckdb/event_test.go index 36f6c902..a9491c15 100644 --- a/core/db/duckdb/event_test.go +++ b/core/db/duckdb/event_test.go @@ -6,6 +6,35 @@ import ( "github.com/medama-io/medama/model" ) +func TestAddEvents(t *testing.T) { + assert, _, ctx, client := SetupDatabase(t) + rows := client.DB.QueryRow("SELECT COUNT(*) FROM events WHERE group_name = 'add-event-test.io'") + var count int + err := rows.Scan(&count) + assert.NoError(err) + assert.Equal(0, count) + + event1 := model.EventHit{ + Group: "add-event-test.io", + Name: "test_event", + Value: "test_value", + } + + event2 := model.EventHit{ + Group: "add-event-test.io", + Name: "test_event2", + Value: "test_value2", + } + + err = client.AddEvents(ctx, &[]model.EventHit{event1, event2}) + assert.NoError(err) + + rows = client.DB.QueryRow("SELECT COUNT(*) FROM events WHERE group_name = 'add-event-test.io'") + err = rows.Scan(&count) + assert.NoError(err) + assert.Equal(2, count) +} + func TestAddPageView(t *testing.T) { assert, _, ctx, client := SetupDatabase(t) rows := client.DB.QueryRow("SELECT COUNT(*) FROM views WHERE hostname = 'add-page-view-test.io'") diff --git a/core/go.mod b/core/go.mod index 6cd37b85..791d7689 100644 --- a/core/go.mod +++ b/core/go.mod @@ -4,6 +4,7 @@ go 1.22.2 require ( github.com/alexedwards/argon2id v1.0.0 + github.com/alphadose/haxmap v1.4.0 github.com/caarlos0/env/v11 v11.1.0 github.com/gkampitakis/go-snaps v0.5.4 github.com/go-faster/errors v0.7.1 diff --git a/core/go.sum b/core/go.sum index 02f37200..a1ecca6d 100644 --- a/core/go.sum +++ b/core/go.sum @@ -2,6 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w= github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw= +github.com/alphadose/haxmap v1.4.0 h1:1yn+oGzy2THJj1DMuJBzRanE3sMnDAjJVbU0L31Jp3w= +github.com/alphadose/haxmap v1.4.0/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= github.com/boyter/go-string v1.0.5 h1:/xcOlWdgelLYLVkUU0xBLfioGjZ9KIMUMI/RXG138YY= @@ -76,8 +78,6 @@ github.com/medama-io/go-referrer-parser v0.0.0-20240706151617-0106555291e7 h1:r/ github.com/medama-io/go-referrer-parser v0.0.0-20240706151617-0106555291e7/go.mod h1:y/Y+TQijcFNVXWiZ7YhiThXVRbORFdhcY0osQZXQw8Q= github.com/medama-io/go-timezone-country v0.0.0-20240125021558-8a6127efd8f7 h1:mydNOo0Zm10bC/RX4h9iwe18hGS0KB5cTqo/y/WpbLQ= github.com/medama-io/go-timezone-country v0.0.0-20240125021558-8a6127efd8f7/go.mod h1:Wq7lg5D0ZdQ3bHnzOTKsb1YGlxm/l82OVA4aIbAA5w4= -github.com/medama-io/go-useragent v0.0.0-20240705132343-7fdad75704b9 h1:WJcl6hA8S7/F5MmiPaFdUduMyj40IhIAuh14zuLCZyk= -github.com/medama-io/go-useragent v0.0.0-20240705132343-7fdad75704b9/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw= github.com/medama-io/go-useragent v0.0.0-20240707203018-4bd80a87eb23 h1:myjtzE9EGr2zS0d9jguGbZGCgj2117X82L9ZAK1AeYo= github.com/medama-io/go-useragent v0.0.0-20240707203018-4bd80a87eb23/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= diff --git a/core/migrations/0004_duckdb_events.go b/core/migrations/0004_duckdb_events.go new file mode 100644 index 00000000..100ed9f0 --- /dev/null +++ b/core/migrations/0004_duckdb_events.go @@ -0,0 +1,72 @@ +package migrations + +import ( + "github.com/medama-io/medama/db/duckdb" +) + +func Up0004(c *duckdb.Client) error { + // Begin transaction + tx, err := c.Beginx() + if err != nil { + return err + } + + // Create events table. + // + // batch_id is used to group together multiple properties of the same event. + // + // group_name is the group name of the event, typically the hostname. + // + // name is the name of the event. + // + // value is the value of the event. + // + // date_created is the date the event was created. + _, err = tx.Exec(`--sql + CREATE TABLE IF NOT EXISTS events ( + batch_id TEXT NOT NULL, + group_name TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT NOT NULL, + date_created TIMESTAMPTZ NOT NULL + )`) + if err != nil { + rollbackErr := tx.Rollback() + if rollbackErr != nil { + return rollbackErr + } + + return err + } + + // Commit transaction + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + +func Down0004(c *duckdb.Client) error { + // Begin transaction + tx, err := c.Beginx() + if err != nil { + return err + } + + // Drop views table + _, err = tx.Exec(`--sql + DROP TABLE IF EXISTS events`) + if err != nil { + return err + } + + // Commit transaction + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/core/migrations/migrations.go b/core/migrations/migrations.go index 7094871c..153e5ad3 100644 --- a/core/migrations/migrations.go +++ b/core/migrations/migrations.go @@ -54,6 +54,7 @@ func NewMigrationsService(ctx context.Context, sqliteC *sqlite.Client, duckdbC * duckdbMigrations := []*Migration[duckdb.Client]{ {ID: 2, Name: "0002_duckdb_schema.go", Type: DuckDB, Up: Up0002, Down: Down0002}, {ID: 3, Name: "0003_duckdb_referrer.go", Type: DuckDB, Up: Up0003, Down: Down0003}, + {ID: 4, Name: "0004_duckdb_events.go", Type: DuckDB, Up: Up0004, Down: Down0004}, } log := logger.Get() diff --git a/core/model/event.go b/core/model/event.go index df363212..5893f395 100644 --- a/core/model/event.go +++ b/core/model/event.go @@ -7,6 +7,17 @@ const ( RequestKeyBody RequestKey = "request" ) +type EventHit struct { + // BatchID - Used to group together multiple properties of the same event. + BatchID string `db:"batch_id"` + // Group - The group name of the event, typically the hostname. + Group string `db:"group_name"` + // Name - The name of the event. + Name string `db:"name"` + // Value - The value of the event. + Value string `db:"value"` +} + type PageViewHit struct { // Beacon ID - Used to determine if multiple event types are // associated with a single page view. diff --git a/core/openapi.yaml b/core/openapi.yaml index caaed77e..b7c7f8c2 100644 --- a/core/openapi.yaml +++ b/core/openapi.yaml @@ -1408,17 +1408,42 @@ components: required: - b - m + EventCustom: + title: EventCustom + description: Event with custom properties. + type: object + properties: + g: + type: string + description: Group name of events. Currently, only the hostname is supported. + n: + type: string + description: Event name or key. + p: + type: object + description: Event properties. + additionalProperties: + oneOf: + - type: string + - type: integer + - type: boolean + required: + - g + - n + - p EventHit: title: EventHit description: Website hit event. oneOf: - $ref: "#/components/schemas/EventLoad" - $ref: "#/components/schemas/EventUnload" + - $ref: "#/components/schemas/EventCustom" discriminator: propertyName: e mapping: load: "#/components/schemas/EventLoad" unload: "#/components/schemas/EventUnload" + custom: "#/components/schemas/EventCustom" FilterString: type: object properties: diff --git a/core/services/event.go b/core/services/event.go index 6d5787ae..87b49e07 100644 --- a/core/services/event.go +++ b/core/services/event.go @@ -4,12 +4,15 @@ import ( "context" "net/http" "net/url" + "strconv" "strings" "time" + "github.com/go-faster/errors" "github.com/medama-io/medama/api" "github.com/medama-io/medama/model" "github.com/medama-io/medama/util/logger" + "go.jetify.com/typeid" "golang.org/x/text/language" "golang.org/x/text/language/display" ) @@ -283,6 +286,61 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, params api // Log success log.Debug().Msg("hit: updated page view") + case api.EventCustomEventHit: + group := req.EventCustom.G + log = log.With().Str("hostname", group).Logger() + + // Verify hostname exists + if !h.hostnames.Has(group) { + log.Warn().Msg("hit: website not found") + return ErrNotFound(model.ErrWebsiteNotFound), nil + } + + events := []model.EventHit{} + + // Generate batch ID to group all the properties of the same event. + batchIDType, err := typeid.WithPrefix("event") + if err != nil { + return ErrInternalServerError(errors.Wrap(err, "services: typeid custom event")), nil + } + batchID := batchIDType.String() + + for name, item := range req.EventCustom.P { + var value string + + switch item.Type { + case api.StringEventCustomPItem: + value = item.String + case api.IntEventCustomPItem: + value = strconv.Itoa(item.Int) + case api.BoolEventCustomPItem: + value = strconv.FormatBool(item.Bool) + default: + log.Error().Str("type", string(item.Type)).Msg("hit: invalid custom event property type") + return ErrBadRequest(model.ErrInvalidTrackerEvent), nil + } + + events = append(events, model.EventHit{ + BatchID: batchID, + Group: group, + Name: name, + Value: value, + }) + + log = log.With(). + Str("group", group). + Str("name", name). + Str("value", value). + Logger() + + err := h.analyticsDB.AddEvents(ctx, &events) + if err != nil { + log.Error().Err(err).Msg("hit: failed to add event") + return ErrInternalServerError(err), nil + } + + log.Debug().Msg("hit: added custom event") + } default: log.Error().Str("type", string(req.Type)).Msg("hit: invalid event hit type") return ErrBadRequest(model.ErrInvalidTrackerEvent), nil diff --git a/dashboard/app/api/types.d.ts b/dashboard/app/api/types.d.ts index 14550951..940c5ec2 100644 --- a/dashboard/app/api/types.d.ts +++ b/dashboard/app/api/types.d.ts @@ -482,11 +482,30 @@ export interface components { */ e: "unload"; }; + /** + * EventCustom + * @description Event with custom properties. + */ + EventCustom: { + /** @description Group name of events. Currently, only the hostname is supported. */ + g: string; + /** @description Event name or key. */ + n: string; + /** @description Event properties. */ + p: { + [key: string]: (string | number | boolean) | undefined; + }; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + e: "custom"; + }; /** * EventHit * @description Website hit event. */ - EventHit: components["schemas"]["EventLoad"] | components["schemas"]["EventUnload"]; + EventHit: components["schemas"]["EventLoad"] | components["schemas"]["EventUnload"] | components["schemas"]["EventCustom"]; FilterString: { /** @description Equal to. */ eq?: string;