mirror of
https://github.com/dstotijn/hetty.git
synced 2025-07-01 18:47:29 -04:00
Replace SQLite with BadgerDB
This commit is contained in:
David Stotijn
53
pkg/db/badger/badger.go
Normal file
53
pkg/db/badger/badger.go
Normal file
@ -0,0 +1,53 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/dgraph-io/badger/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
// Key prefixes. Each prefix value should be unique.
|
||||
projectPrefix = 0x00
|
||||
reqLogPrefix = 0x01
|
||||
resLogPrefix = 0x02
|
||||
|
||||
// Request log indices.
|
||||
reqLogProjectIDIndex = 0x00
|
||||
)
|
||||
|
||||
// Database is used to store and retrieve data from an underlying Badger database.
|
||||
type Database struct {
|
||||
badger *badger.DB
|
||||
}
|
||||
|
||||
// OpenDatabase opens a new Badger database.
|
||||
func OpenDatabase(opts badger.Options) (*Database, error) {
|
||||
db, err := badger.Open(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("badger: failed to open database: %w", err)
|
||||
}
|
||||
|
||||
return &Database{badger: db}, nil
|
||||
}
|
||||
|
||||
// Close closes the underlying Badger database.
|
||||
func (db *Database) Close() error {
|
||||
return db.badger.Close()
|
||||
}
|
||||
|
||||
// DatabaseFromBadgerDB returns a Database with `db` set as the underlying
|
||||
// Badger database.
|
||||
func DatabaseFromBadgerDB(db *badger.DB) *Database {
|
||||
return &Database{badger: db}
|
||||
}
|
||||
|
||||
func entryKey(prefix, index byte, value []byte) []byte {
|
||||
// Key consists of: | prefix (byte) | index (byte) | value
|
||||
key := make([]byte, 2+len(value))
|
||||
key[0] = prefix
|
||||
key[1] = index
|
||||
copy(key[2:len(value)+2], value)
|
||||
|
||||
return key
|
||||
}
|
||||
110
pkg/db/badger/proj.go
Normal file
110
pkg/db/badger/proj.go
Normal file
@ -0,0 +1,110 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/dgraph-io/badger/v3"
|
||||
"github.com/oklog/ulid"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/proj"
|
||||
)
|
||||
|
||||
func (db *Database) UpsertProject(ctx context.Context, project proj.Project) error {
|
||||
buf := bytes.Buffer{}
|
||||
|
||||
err := gob.NewEncoder(&buf).Encode(project)
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to encode project: %w", err)
|
||||
}
|
||||
|
||||
err = db.badger.Update(func(txn *badger.Txn) error {
|
||||
return txn.Set(entryKey(projectPrefix, 0, project.ID[:]), buf.Bytes())
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *Database) FindProjectByID(ctx context.Context, projectID ulid.ULID) (project proj.Project, err error) {
|
||||
err = db.badger.View(func(txn *badger.Txn) error {
|
||||
item, err := txn.Get(entryKey(projectPrefix, 0, projectID[:]))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = item.Value(func(rawProject []byte) error {
|
||||
return gob.NewDecoder(bytes.NewReader(rawProject)).Decode(&project)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to retrieve or parse project: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if errors.Is(err, badger.ErrKeyNotFound) {
|
||||
return proj.Project{}, proj.ErrProjectNotFound
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return proj.Project{}, fmt.Errorf("badger: failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return project, nil
|
||||
}
|
||||
|
||||
func (db *Database) DeleteProject(ctx context.Context, projectID ulid.ULID) error {
|
||||
err := db.ClearRequestLogs(ctx, projectID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to delete project request logs: %w", err)
|
||||
}
|
||||
|
||||
err = db.badger.Update(func(txn *badger.Txn) error {
|
||||
return txn.Delete(entryKey(projectPrefix, 0, projectID[:]))
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to delete project item: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *Database) Projects(ctx context.Context) ([]proj.Project, error) {
|
||||
projects := make([]proj.Project, 0)
|
||||
|
||||
err := db.badger.View(func(txn *badger.Txn) error {
|
||||
var rawProject []byte
|
||||
prefix := entryKey(projectPrefix, 0, nil)
|
||||
|
||||
iterator := txn.NewIterator(badger.DefaultIteratorOptions)
|
||||
defer iterator.Close()
|
||||
|
||||
for iterator.Seek(prefix); iterator.ValidForPrefix(prefix); iterator.Next() {
|
||||
rawProject, err := iterator.Item().ValueCopy(rawProject)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy value: %w", err)
|
||||
}
|
||||
|
||||
var project proj.Project
|
||||
err = gob.NewDecoder(bytes.NewReader(rawProject)).Decode(&project)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode project: %w", err)
|
||||
}
|
||||
|
||||
projects = append(projects, project)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("badger: failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return projects, nil
|
||||
}
|
||||
284
pkg/db/badger/proj_test.go
Normal file
284
pkg/db/badger/proj_test.go
Normal file
@ -0,0 +1,284 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
badgerdb "github.com/dgraph-io/badger/v3"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/oklog/ulid"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/proj"
|
||||
"github.com/dstotijn/hetty/pkg/scope"
|
||||
"github.com/dstotijn/hetty/pkg/search"
|
||||
)
|
||||
|
||||
//nolint:gosec
|
||||
var ulidEntropy = rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
var regexpCompareOpt = cmp.Comparer(func(x, y *regexp.Regexp) bool {
|
||||
switch {
|
||||
case x == nil && y == nil:
|
||||
return true
|
||||
case x == nil || y == nil:
|
||||
return false
|
||||
default:
|
||||
return x.String() == y.String()
|
||||
}
|
||||
})
|
||||
|
||||
func TestUpsertProject(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
badgerDB, err := badgerdb.Open(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
|
||||
database := DatabaseFromBadgerDB(badgerDB)
|
||||
defer database.Close()
|
||||
|
||||
searchExpr, err := search.ParseQuery("foo AND bar OR NOT baz")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (expected: nil, got: %v)", err)
|
||||
}
|
||||
|
||||
exp := proj.Project{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy),
|
||||
Name: "foobar",
|
||||
Settings: proj.Settings{
|
||||
ReqLogBypassOutOfScope: true,
|
||||
ReqLogOnlyFindInScope: true,
|
||||
ScopeRules: []scope.Rule{
|
||||
{
|
||||
URL: regexp.MustCompile("^https://(.*)example.com(.*)$"),
|
||||
Header: scope.Header{
|
||||
Key: regexp.MustCompile("^X-Foo(.*)$"),
|
||||
Value: regexp.MustCompile("^foo(.*)$"),
|
||||
},
|
||||
Body: regexp.MustCompile("^foo(.*)"),
|
||||
},
|
||||
},
|
||||
SearchExpr: searchExpr,
|
||||
},
|
||||
}
|
||||
|
||||
err = database.UpsertProject(context.Background(), exp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error storing project: %v", err)
|
||||
}
|
||||
|
||||
var rawProject []byte
|
||||
|
||||
err = badgerDB.View(func(txn *badgerdb.Txn) error {
|
||||
item, err := txn.Get(entryKey(projectPrefix, 0, exp.ID[:]))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rawProject, err = item.ValueCopy(nil)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error retrieving project from database: %v", err)
|
||||
}
|
||||
|
||||
got := proj.Project{}
|
||||
|
||||
err = gob.NewDecoder(bytes.NewReader(rawProject)).Decode(&got)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error decoding project: %v", err)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(exp, got, regexpCompareOpt, cmpopts.IgnoreUnexported(proj.Project{})); diff != "" {
|
||||
t.Fatalf("project not equal (-exp, +got):\n%v", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindProjectByID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("existing project", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
badgerDB, err := badgerdb.Open(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
|
||||
database := DatabaseFromBadgerDB(badgerDB)
|
||||
defer database.Close()
|
||||
|
||||
exp := proj.Project{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy),
|
||||
Name: "foobar",
|
||||
Settings: proj.Settings{},
|
||||
}
|
||||
|
||||
buf := bytes.Buffer{}
|
||||
|
||||
err = gob.NewEncoder(&buf).Encode(exp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error encoding project: %v", err)
|
||||
}
|
||||
|
||||
err = badgerDB.Update(func(txn *badgerdb.Txn) error {
|
||||
return txn.Set(entryKey(projectPrefix, 0, exp.ID[:]), buf.Bytes())
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error setting project: %v", err)
|
||||
}
|
||||
|
||||
got, err := database.FindProjectByID(context.Background(), exp.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error finding project: %v", err)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(exp, got, cmpopts.IgnoreUnexported(proj.Project{})); diff != "" {
|
||||
t.Fatalf("project not equal (-exp, +got):\n%v", diff)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("project not found", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
database, err := OpenDatabase(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
projectID := ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy)
|
||||
|
||||
_, err = database.FindProjectByID(context.Background(), projectID)
|
||||
if !errors.Is(err, proj.ErrProjectNotFound) {
|
||||
t.Fatalf("expected `proj.ErrProjectNotFound`, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteProject(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
badgerDB, err := badgerdb.Open(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
|
||||
database := DatabaseFromBadgerDB(badgerDB)
|
||||
defer database.Close()
|
||||
|
||||
// Store fixtures.
|
||||
projectID := ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy)
|
||||
reqLogID := ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy)
|
||||
|
||||
err = badgerDB.Update(func(txn *badgerdb.Txn) error {
|
||||
if err := txn.Set(entryKey(projectPrefix, 0, projectID[:]), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := txn.Set(entryKey(reqLogPrefix, 0, reqLogID[:]), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := txn.Set(entryKey(resLogPrefix, 0, reqLogID[:]), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
err := txn.Set(entryKey(reqLogPrefix, reqLogProjectIDIndex, append(projectID[:], reqLogID[:]...)), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating fixtures: %v", err)
|
||||
}
|
||||
|
||||
err = database.DeleteProject(context.Background(), projectID)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error deleting project: %v", err)
|
||||
}
|
||||
|
||||
// Assert project key was deleted.
|
||||
err = badgerDB.View(func(txn *badgerdb.Txn) error {
|
||||
_, err := txn.Get(entryKey(projectPrefix, 0, projectID[:]))
|
||||
return err
|
||||
})
|
||||
if !errors.Is(err, badgerdb.ErrKeyNotFound) {
|
||||
t.Fatalf("expected `badger.ErrKeyNotFound`, got: %v", err)
|
||||
}
|
||||
|
||||
// Assert request log item was deleted.
|
||||
err = badgerDB.View(func(txn *badgerdb.Txn) error {
|
||||
_, err := txn.Get(entryKey(reqLogPrefix, 0, reqLogID[:]))
|
||||
return err
|
||||
})
|
||||
if !errors.Is(err, badgerdb.ErrKeyNotFound) {
|
||||
t.Fatalf("expected `badger.ErrKeyNotFound`, got: %v", err)
|
||||
}
|
||||
|
||||
// Assert response log item was deleted.
|
||||
err = badgerDB.View(func(txn *badgerdb.Txn) error {
|
||||
_, err := txn.Get(entryKey(resLogPrefix, 0, reqLogID[:]))
|
||||
return err
|
||||
})
|
||||
if !errors.Is(err, badgerdb.ErrKeyNotFound) {
|
||||
t.Fatalf("expected `badger.ErrKeyNotFound`, got: %v", err)
|
||||
}
|
||||
|
||||
// Assert request log project ID index key was deleted.
|
||||
err = badgerDB.View(func(txn *badgerdb.Txn) error {
|
||||
_, err := txn.Get(entryKey(reqLogPrefix, reqLogProjectIDIndex, append(projectID[:], reqLogID[:]...)))
|
||||
return err
|
||||
})
|
||||
if !errors.Is(err, badgerdb.ErrKeyNotFound) {
|
||||
t.Fatalf("expected `badger.ErrKeyNotFound`, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjects(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
database, err := OpenDatabase(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
exp := []proj.Project{
|
||||
{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy),
|
||||
Name: "one",
|
||||
},
|
||||
{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now())+100, ulidEntropy),
|
||||
Name: "two",
|
||||
},
|
||||
}
|
||||
|
||||
// Store fixtures.
|
||||
for _, project := range exp {
|
||||
err = database.UpsertProject(context.Background(), project)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating project fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := database.Projects(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error finding projects: %v", err)
|
||||
}
|
||||
|
||||
if len(exp) != len(got) {
|
||||
t.Fatalf("expected %v projects, got: %v", len(exp), len(got))
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(exp, got, cmpopts.IgnoreUnexported(proj.Project{})); diff != "" {
|
||||
t.Fatalf("projects not equal (-exp, +got):\n%v", diff)
|
||||
}
|
||||
}
|
||||
251
pkg/db/badger/reqlog.go
Normal file
251
pkg/db/badger/reqlog.go
Normal file
@ -0,0 +1,251 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/dgraph-io/badger/v3"
|
||||
"github.com/oklog/ulid"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/reqlog"
|
||||
"github.com/dstotijn/hetty/pkg/scope"
|
||||
)
|
||||
|
||||
func (db *Database) FindRequestLogs(ctx context.Context, filter reqlog.FindRequestsFilter, scope *scope.Scope) ([]reqlog.RequestLog, error) {
|
||||
if filter.ProjectID.Compare(ulid.ULID{}) == 0 {
|
||||
return nil, reqlog.ErrProjectIDMustBeSet
|
||||
}
|
||||
|
||||
txn := db.badger.NewTransaction(false)
|
||||
defer txn.Discard()
|
||||
|
||||
reqLogIDs, err := findRequestLogIDsByProjectID(txn, filter.ProjectID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("badger: failed to find request log IDs: %w", err)
|
||||
}
|
||||
|
||||
reqLogs := make([]reqlog.RequestLog, 0, len(reqLogIDs))
|
||||
|
||||
for _, reqLogID := range reqLogIDs {
|
||||
reqLog, err := getRequestLogWithResponse(txn, reqLogID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("badger: failed to get request log (id: %v): %w", reqLogID.String(), err)
|
||||
}
|
||||
|
||||
if filter.OnlyInScope {
|
||||
if !reqLog.MatchScope(scope) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Filter by search expression.
|
||||
// TODO: Once pagination is introduced, this filter logic should be done
|
||||
// as items are retrieved (e.g. when using a `badger.Iterator`).
|
||||
if filter.SearchExpr != nil {
|
||||
match, err := reqLog.Matches(filter.SearchExpr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"badger: failed to match search expression for request log (id: %v): %w",
|
||||
reqLogID.String(), err,
|
||||
)
|
||||
}
|
||||
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
reqLogs = append(reqLogs, reqLog)
|
||||
}
|
||||
|
||||
return reqLogs, nil
|
||||
}
|
||||
|
||||
func getRequestLogWithResponse(txn *badger.Txn, reqLogID ulid.ULID) (reqlog.RequestLog, error) {
|
||||
item, err := txn.Get(entryKey(reqLogPrefix, 0, reqLogID[:]))
|
||||
if err != nil {
|
||||
return reqlog.RequestLog{}, fmt.Errorf("failed to lookup request log item: %w", err)
|
||||
}
|
||||
|
||||
reqLog := reqlog.RequestLog{
|
||||
ID: reqLogID,
|
||||
}
|
||||
|
||||
err = item.Value(func(rawReqLog []byte) error {
|
||||
err = gob.NewDecoder(bytes.NewReader(rawReqLog)).Decode(&reqLog)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode request log: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return reqlog.RequestLog{}, fmt.Errorf("failed to retrieve or parse request log value: %w", err)
|
||||
}
|
||||
|
||||
item, err = txn.Get(entryKey(resLogPrefix, 0, reqLogID[:]))
|
||||
|
||||
if errors.Is(err, badger.ErrKeyNotFound) {
|
||||
return reqLog, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return reqlog.RequestLog{}, fmt.Errorf("failed to get response log: %w", err)
|
||||
}
|
||||
|
||||
err = item.Value(func(rawReslog []byte) error {
|
||||
var resLog reqlog.ResponseLog
|
||||
err = gob.NewDecoder(bytes.NewReader(rawReslog)).Decode(&resLog)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode response log: %w", err)
|
||||
}
|
||||
|
||||
reqLog.Response = &resLog
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return reqlog.RequestLog{}, fmt.Errorf("failed to retrieve or parse response log value: %w", err)
|
||||
}
|
||||
|
||||
return reqLog, nil
|
||||
}
|
||||
|
||||
func (db *Database) FindRequestLogByID(ctx context.Context, reqLogID ulid.ULID) (reqLog reqlog.RequestLog, err error) {
|
||||
txn := db.badger.NewTransaction(false)
|
||||
defer txn.Discard()
|
||||
|
||||
reqLog, err = getRequestLogWithResponse(txn, reqLogID)
|
||||
if err != nil {
|
||||
return reqlog.RequestLog{}, fmt.Errorf("badger: failed to get request log: %w", err)
|
||||
}
|
||||
|
||||
return reqLog, nil
|
||||
}
|
||||
|
||||
func (db *Database) StoreRequestLog(ctx context.Context, reqLog reqlog.RequestLog) error {
|
||||
buf := bytes.Buffer{}
|
||||
|
||||
err := gob.NewEncoder(&buf).Encode(reqLog)
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to encode request log: %w", err)
|
||||
}
|
||||
|
||||
entries := []*badger.Entry{
|
||||
// Request log itself.
|
||||
{
|
||||
Key: entryKey(reqLogPrefix, 0, reqLog.ID[:]),
|
||||
Value: buf.Bytes(),
|
||||
},
|
||||
// Index by project ID.
|
||||
{
|
||||
Key: entryKey(reqLogPrefix, reqLogProjectIDIndex, append(reqLog.ProjectID[:], reqLog.ID[:]...)),
|
||||
},
|
||||
}
|
||||
|
||||
err = db.badger.Update(func(txn *badger.Txn) error {
|
||||
for i := range entries {
|
||||
err := txn.SetEntry(entries[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *Database) StoreResponseLog(ctx context.Context, reqLogID ulid.ULID, resLog reqlog.ResponseLog) error {
|
||||
buf := bytes.Buffer{}
|
||||
|
||||
err := gob.NewEncoder(&buf).Encode(resLog)
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to encode response log: %w", err)
|
||||
}
|
||||
|
||||
err = db.badger.Update(func(txn *badger.Txn) error {
|
||||
return txn.SetEntry(&badger.Entry{
|
||||
Key: entryKey(resLogPrefix, 0, reqLogID[:]),
|
||||
Value: buf.Bytes(),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *Database) ClearRequestLogs(ctx context.Context, projectID ulid.ULID) error {
|
||||
// Note: this transaction is used just for reading; we use the `badger.WriteBatch`
|
||||
// API to bulk delete items.
|
||||
txn := db.badger.NewTransaction(false)
|
||||
defer txn.Discard()
|
||||
|
||||
reqLogIDs, err := findRequestLogIDsByProjectID(txn, projectID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to find request log IDs: %w", err)
|
||||
}
|
||||
|
||||
writeBatch := db.badger.NewWriteBatch()
|
||||
defer writeBatch.Cancel()
|
||||
|
||||
for _, reqLogID := range reqLogIDs {
|
||||
// Delete request logs.
|
||||
err := writeBatch.Delete(entryKey(reqLogPrefix, 0, reqLogID[:]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to delete request log: %w", err)
|
||||
}
|
||||
|
||||
// Delete related response log.
|
||||
err = writeBatch.Delete(entryKey(resLogPrefix, 0, reqLogID[:]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to delete request log: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeBatch.Flush(); err != nil {
|
||||
return fmt.Errorf("badger: failed to commit batch write: %w", err)
|
||||
}
|
||||
|
||||
err = db.badger.DropPrefix(entryKey(reqLogPrefix, reqLogProjectIDIndex, projectID[:]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("badger: failed to drop request log project ID index items: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findRequestLogIDsByProjectID(txn *badger.Txn, projectID ulid.ULID) ([]ulid.ULID, error) {
|
||||
reqLogIDs := make([]ulid.ULID, 0)
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchValues = false
|
||||
iterator := txn.NewIterator(opts)
|
||||
defer iterator.Close()
|
||||
|
||||
var projectIndexKey []byte
|
||||
|
||||
prefix := entryKey(reqLogPrefix, reqLogProjectIDIndex, projectID[:])
|
||||
|
||||
for iterator.Seek(prefix); iterator.ValidForPrefix(prefix); iterator.Next() {
|
||||
projectIndexKey = iterator.Item().KeyCopy(projectIndexKey)
|
||||
|
||||
var id ulid.ULID
|
||||
// The request log ID starts *after* the first 2 prefix and index bytes
|
||||
// and the 16 byte project ID.
|
||||
if err := id.UnmarshalBinary(projectIndexKey[18:]); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse request log ID: %w", err)
|
||||
}
|
||||
|
||||
reqLogIDs = append(reqLogIDs, id)
|
||||
}
|
||||
|
||||
return reqLogIDs, nil
|
||||
}
|
||||
121
pkg/db/badger/reqlog_test.go
Normal file
121
pkg/db/badger/reqlog_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
package badger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
badgerdb "github.com/dgraph-io/badger/v3"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/oklog/ulid"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/reqlog"
|
||||
)
|
||||
|
||||
func TestFindRequestLogs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("without project ID in filter", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
database, err := OpenDatabase(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
filter := reqlog.FindRequestsFilter{}
|
||||
|
||||
_, err = database.FindRequestLogs(context.Background(), filter, nil)
|
||||
if !errors.Is(err, reqlog.ErrProjectIDMustBeSet) {
|
||||
t.Fatalf("expected `reqlog.ErrProjectIDMustBeSet`, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("returns request logs and related response logs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
database, err := OpenDatabase(badgerdb.DefaultOptions("").WithInMemory(true))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open badger database: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
projectID := ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy)
|
||||
|
||||
exp := []reqlog.RequestLog{
|
||||
{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now()), ulidEntropy),
|
||||
ProjectID: projectID,
|
||||
URL: mustParseURL(t, "https://example.com/foobar"),
|
||||
Method: http.MethodPost,
|
||||
Proto: "HTTP/1.1",
|
||||
Header: http.Header{
|
||||
"X-Foo": []string{"baz"},
|
||||
},
|
||||
Body: []byte("foo"),
|
||||
Response: &reqlog.ResponseLog{
|
||||
Proto: "HTTP/1.1",
|
||||
Status: "200 OK",
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
"X-Yolo": []string{"swag"},
|
||||
},
|
||||
Body: []byte("bar"),
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: ulid.MustNew(ulid.Timestamp(time.Now())+100, ulidEntropy),
|
||||
ProjectID: projectID,
|
||||
URL: mustParseURL(t, "https://example.com/foo?bar=baz"),
|
||||
Method: http.MethodGet,
|
||||
Proto: "HTTP/1.1",
|
||||
Header: http.Header{
|
||||
"X-Foo": []string{"baz"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Store fixtures.
|
||||
for _, reqLog := range exp {
|
||||
err = database.StoreRequestLog(context.Background(), reqLog)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating request log fixture: %v", err)
|
||||
}
|
||||
|
||||
if reqLog.Response != nil {
|
||||
err = database.StoreResponseLog(context.Background(), reqLog.ID, *reqLog.Response)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating response log fixture: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
filter := reqlog.FindRequestsFilter{
|
||||
ProjectID: projectID,
|
||||
}
|
||||
|
||||
got, err := database.FindRequestLogs(context.Background(), filter, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error finding request logs: %v", err)
|
||||
}
|
||||
|
||||
if diff := cmp.Diff(exp, got); diff != "" {
|
||||
t.Fatalf("request logs not equal (-exp, +got):\n%v", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func mustParseURL(t *testing.T, s string) *url.URL {
|
||||
t.Helper()
|
||||
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return u
|
||||
}
|
||||
@ -1,82 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/reqlog"
|
||||
)
|
||||
|
||||
type reqURL url.URL
|
||||
|
||||
type httpRequest struct {
|
||||
ID int64 `db:"req_id"`
|
||||
Proto string `db:"req_proto"`
|
||||
URL reqURL `db:"url"`
|
||||
Method string `db:"method"`
|
||||
Body []byte `db:"req_body"`
|
||||
Timestamp time.Time `db:"req_timestamp"`
|
||||
httpResponse
|
||||
}
|
||||
|
||||
type httpResponse struct {
|
||||
ID sql.NullInt64 `db:"res_id"`
|
||||
RequestID sql.NullInt64 `db:"res_req_id"`
|
||||
Proto sql.NullString `db:"res_proto"`
|
||||
StatusCode sql.NullInt64 `db:"status_code"`
|
||||
StatusReason sql.NullString `db:"status_reason"`
|
||||
Body []byte `db:"res_body"`
|
||||
Timestamp sql.NullTime `db:"res_timestamp"`
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer.
|
||||
func (u *reqURL) Scan(value interface{}) error {
|
||||
rawURL, ok := value.(string)
|
||||
if !ok {
|
||||
return errors.New("sqlite: cannot scan non-string value")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite: could not parse URL: %w", err)
|
||||
}
|
||||
|
||||
*u = reqURL(*parsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dto httpRequest) toRequestLog() reqlog.Request {
|
||||
u := url.URL(dto.URL)
|
||||
reqLog := reqlog.Request{
|
||||
ID: dto.ID,
|
||||
Request: http.Request{
|
||||
Proto: dto.Proto,
|
||||
Method: dto.Method,
|
||||
URL: &u,
|
||||
},
|
||||
Body: dto.Body,
|
||||
Timestamp: dto.Timestamp,
|
||||
}
|
||||
|
||||
if dto.httpResponse.ID.Valid {
|
||||
reqLog.Response = &reqlog.Response{
|
||||
ID: dto.httpResponse.ID.Int64,
|
||||
RequestID: dto.httpResponse.RequestID.Int64,
|
||||
Response: http.Response{
|
||||
Status: strconv.FormatInt(dto.StatusCode.Int64, 10) + " " + dto.StatusReason.String,
|
||||
StatusCode: int(dto.StatusCode.Int64),
|
||||
Proto: dto.httpResponse.Proto.String,
|
||||
},
|
||||
Body: dto.httpResponse.Body,
|
||||
Timestamp: dto.httpResponse.Timestamp.Time,
|
||||
}
|
||||
}
|
||||
|
||||
return reqLog
|
||||
}
|
||||
@ -1,135 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
sq "github.com/Masterminds/squirrel"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/search"
|
||||
)
|
||||
|
||||
var stringLiteralMap = map[string]string{
|
||||
// http_requests
|
||||
"req.id": "req.id",
|
||||
"req.proto": "req.proto",
|
||||
"req.url": "req.url",
|
||||
"req.method": "req.method",
|
||||
"req.body": "req.body",
|
||||
"req.timestamp": "req.timestamp",
|
||||
// http_responses
|
||||
"res.id": "res.id",
|
||||
"res.proto": "res.proto",
|
||||
"res.statusCode": "res.status_code",
|
||||
"res.statusReason": "res.status_reason",
|
||||
"res.body": "res.body",
|
||||
"res.timestamp": "res.timestamp",
|
||||
// TODO: http_headers
|
||||
}
|
||||
|
||||
func parseSearchExpr(expr search.Expression) (sq.Sqlizer, error) {
|
||||
switch e := expr.(type) {
|
||||
case *search.PrefixExpression:
|
||||
return parsePrefixExpr(e)
|
||||
case *search.InfixExpression:
|
||||
return parseInfixExpr(e)
|
||||
case *search.StringLiteral:
|
||||
return parseStringLiteral(e)
|
||||
default:
|
||||
return nil, fmt.Errorf("expression type (%v) not supported", expr)
|
||||
}
|
||||
}
|
||||
|
||||
func parsePrefixExpr(expr *search.PrefixExpression) (sq.Sqlizer, error) {
|
||||
switch expr.Operator {
|
||||
case search.TokOpNot:
|
||||
// TODO: Find a way to prefix an `sq.Sqlizer` with "NOT".
|
||||
return nil, errors.New("not implemented")
|
||||
default:
|
||||
return nil, errors.New("operator is not supported")
|
||||
}
|
||||
}
|
||||
|
||||
func parseInfixExpr(expr *search.InfixExpression) (sq.Sqlizer, error) {
|
||||
switch expr.Operator {
|
||||
case search.TokOpAnd:
|
||||
left, err := parseSearchExpr(expr.Left)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
right, err := parseSearchExpr(expr.Right)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sq.And{left, right}, nil
|
||||
case search.TokOpOr:
|
||||
left, err := parseSearchExpr(expr.Left)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
right, err := parseSearchExpr(expr.Right)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sq.Or{left, right}, nil
|
||||
}
|
||||
|
||||
left, ok := expr.Left.(*search.StringLiteral)
|
||||
if !ok {
|
||||
return nil, errors.New("left operand must be a string literal")
|
||||
}
|
||||
|
||||
right, ok := expr.Right.(*search.StringLiteral)
|
||||
if !ok {
|
||||
return nil, errors.New("right operand must be a string literal")
|
||||
}
|
||||
|
||||
mappedLeft, ok := stringLiteralMap[left.Value]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid string literal: %v", left)
|
||||
}
|
||||
|
||||
switch expr.Operator {
|
||||
case search.TokOpEq:
|
||||
return sq.Eq{mappedLeft: right.Value}, nil
|
||||
case search.TokOpNotEq:
|
||||
return sq.NotEq{mappedLeft: right.Value}, nil
|
||||
case search.TokOpGt:
|
||||
return sq.Gt{mappedLeft: right.Value}, nil
|
||||
case search.TokOpLt:
|
||||
return sq.Lt{mappedLeft: right.Value}, nil
|
||||
case search.TokOpGtEq:
|
||||
return sq.GtOrEq{mappedLeft: right.Value}, nil
|
||||
case search.TokOpLtEq:
|
||||
return sq.LtOrEq{mappedLeft: right.Value}, nil
|
||||
case search.TokOpRe:
|
||||
return sq.Expr(fmt.Sprintf("regexp(?, %v)", mappedLeft), right.Value), nil
|
||||
case search.TokOpNotRe:
|
||||
return sq.Expr(fmt.Sprintf("NOT regexp(?, %v)", mappedLeft), right.Value), nil
|
||||
default:
|
||||
return nil, errors.New("unsupported operator")
|
||||
}
|
||||
}
|
||||
|
||||
func parseStringLiteral(strLiteral *search.StringLiteral) (sq.Sqlizer, error) {
|
||||
// Sorting is not necessary, but makes it easier to do assertions in tests.
|
||||
sortedKeys := make([]string, 0, len(stringLiteralMap))
|
||||
|
||||
for _, v := range stringLiteralMap {
|
||||
sortedKeys = append(sortedKeys, v)
|
||||
}
|
||||
|
||||
sort.Strings(sortedKeys)
|
||||
|
||||
or := make(sq.Or, len(stringLiteralMap))
|
||||
for i, value := range sortedKeys {
|
||||
or[i] = sq.Like{value: "%" + strLiteral.Value + "%"}
|
||||
}
|
||||
|
||||
return or, nil
|
||||
}
|
||||
@ -1,221 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
sq "github.com/Masterminds/squirrel"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/search"
|
||||
)
|
||||
|
||||
func TestParseSearchExpr(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
searchExpr search.Expression
|
||||
expectedSqlizer sq.Sqlizer
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "req.body = bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.Eq{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body != bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpNotEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.NotEq{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body > bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpGt,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.Gt{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body < bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpLt,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.Lt{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body >= bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpGtEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.GtOrEq{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body <= bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpLtEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.LtOrEq{"req.body": "bar"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body =~ bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpRe,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.Expr("regexp(?, req.body)", "bar"),
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body !~ bar",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpNotRe,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
expectedSqlizer: sq.Expr("NOT regexp(?, req.body)", "bar"),
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body = bar AND res.body = yolo",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpAnd,
|
||||
Left: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
Right: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "res.body"},
|
||||
Right: &search.StringLiteral{Value: "yolo"},
|
||||
},
|
||||
},
|
||||
expectedSqlizer: sq.And{
|
||||
sq.Eq{"req.body": "bar"},
|
||||
sq.Eq{"res.body": "yolo"},
|
||||
},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body = bar AND res.body = yolo AND req.method = POST",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpAnd,
|
||||
Left: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
Right: &search.InfixExpression{
|
||||
Operator: search.TokOpAnd,
|
||||
Left: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "res.body"},
|
||||
Right: &search.StringLiteral{Value: "yolo"},
|
||||
},
|
||||
Right: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "req.method"},
|
||||
Right: &search.StringLiteral{Value: "POST"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedSqlizer: sq.And{
|
||||
sq.Eq{"req.body": "bar"},
|
||||
sq.And{
|
||||
sq.Eq{"res.body": "yolo"},
|
||||
sq.Eq{"req.method": "POST"},
|
||||
},
|
||||
},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "req.body = bar OR res.body = yolo",
|
||||
searchExpr: &search.InfixExpression{
|
||||
Operator: search.TokOpOr,
|
||||
Left: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "req.body"},
|
||||
Right: &search.StringLiteral{Value: "bar"},
|
||||
},
|
||||
Right: &search.InfixExpression{
|
||||
Operator: search.TokOpEq,
|
||||
Left: &search.StringLiteral{Value: "res.body"},
|
||||
Right: &search.StringLiteral{Value: "yolo"},
|
||||
},
|
||||
},
|
||||
expectedSqlizer: sq.Or{
|
||||
sq.Eq{"req.body": "bar"},
|
||||
sq.Eq{"res.body": "yolo"},
|
||||
},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "foo",
|
||||
searchExpr: &search.StringLiteral{
|
||||
Value: "foo",
|
||||
},
|
||||
expectedSqlizer: sq.Or{
|
||||
sq.Like{"req.body": "%foo%"},
|
||||
sq.Like{"req.id": "%foo%"},
|
||||
sq.Like{"req.method": "%foo%"},
|
||||
sq.Like{"req.proto": "%foo%"},
|
||||
sq.Like{"req.timestamp": "%foo%"},
|
||||
sq.Like{"req.url": "%foo%"},
|
||||
sq.Like{"res.body": "%foo%"},
|
||||
sq.Like{"res.id": "%foo%"},
|
||||
sq.Like{"res.proto": "%foo%"},
|
||||
sq.Like{"res.status_code": "%foo%"},
|
||||
sq.Like{"res.status_reason": "%foo%"},
|
||||
sq.Like{"res.timestamp": "%foo%"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := parseSearchExpr(tt.searchExpr)
|
||||
assertError(t, tt.expectedError, err)
|
||||
if !reflect.DeepEqual(tt.expectedSqlizer, got) {
|
||||
t.Errorf("expected: %#v, got: %#v", tt.expectedSqlizer, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertError(t *testing.T, exp, got error) {
|
||||
t.Helper()
|
||||
|
||||
switch {
|
||||
case exp == nil && got != nil:
|
||||
t.Fatalf("expected: nil, got: %v", got)
|
||||
case exp != nil && got == nil:
|
||||
t.Fatalf("expected: %v, got: nil", exp.Error())
|
||||
case exp != nil && got != nil && exp.Error() != got.Error():
|
||||
t.Fatalf("expected: %v, got: %v", exp.Error(), got.Error())
|
||||
}
|
||||
}
|
||||
@ -1,709 +0,0 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/99designs/gqlgen/graphql"
|
||||
sq "github.com/Masterminds/squirrel"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/mattn/go-sqlite3"
|
||||
|
||||
"github.com/dstotijn/hetty/pkg/proj"
|
||||
"github.com/dstotijn/hetty/pkg/reqlog"
|
||||
"github.com/dstotijn/hetty/pkg/scope"
|
||||
)
|
||||
|
||||
var regexpFn = func(pattern string, value interface{}) (bool, error) {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return regexp.MatchString(pattern, v)
|
||||
case int64:
|
||||
return regexp.MatchString(pattern, fmt.Sprintf("%v", v))
|
||||
case []byte:
|
||||
return regexp.Match(pattern, v)
|
||||
default:
|
||||
return false, fmt.Errorf("unsupported type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
// Client implements reqlog.Repository.
|
||||
type Client struct {
|
||||
db *sqlx.DB
|
||||
dbPath string
|
||||
activeProject string
|
||||
}
|
||||
|
||||
type httpRequestLogsQuery struct {
|
||||
requestCols []string
|
||||
requestHeaderCols []string
|
||||
responseHeaderCols []string
|
||||
joinResponse bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
sql.Register("sqlite3_with_regexp", &sqlite3.SQLiteDriver{
|
||||
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
||||
return conn.RegisterFunc("regexp", regexpFn, false)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func New(dbPath string) (*Client, error) {
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(dbPath, 0755); err != nil {
|
||||
return nil, fmt.Errorf("proj: could not create project directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &Client{
|
||||
dbPath: dbPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// OpenProject opens a project database.
|
||||
func (c *Client) OpenProject(name string) error {
|
||||
if c.db != nil {
|
||||
return errors.New("sqlite: there is already a project open")
|
||||
}
|
||||
|
||||
opts := make(url.Values)
|
||||
opts.Set("_foreign_keys", "1")
|
||||
|
||||
dbPath := filepath.Join(c.dbPath, name+".db")
|
||||
dsn := fmt.Sprintf("file:%v?%v", dbPath, opts.Encode())
|
||||
|
||||
db, err := sqlx.Open("sqlite3_with_regexp", dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite: could not open database: %w", err)
|
||||
}
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
return fmt.Errorf("sqlite: could not ping database: %w", err)
|
||||
}
|
||||
|
||||
if err := prepareSchema(db); err != nil {
|
||||
return fmt.Errorf("sqlite: could not prepare schema: %w", err)
|
||||
}
|
||||
|
||||
c.db = db
|
||||
c.activeProject = name
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Projects() ([]proj.Project, error) {
|
||||
files, err := ioutil.ReadDir(c.dbPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not read projects directory: %w", err)
|
||||
}
|
||||
|
||||
projects := make([]proj.Project, len(files))
|
||||
|
||||
for i, file := range files {
|
||||
projName := strings.TrimSuffix(file.Name(), ".db")
|
||||
projects[i] = proj.Project{
|
||||
Name: projName,
|
||||
IsActive: c.activeProject == projName,
|
||||
}
|
||||
}
|
||||
|
||||
return projects, nil
|
||||
}
|
||||
|
||||
func prepareSchema(db *sqlx.DB) error {
|
||||
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS http_requests (
|
||||
id INTEGER PRIMARY KEY,
|
||||
proto TEXT,
|
||||
url TEXT,
|
||||
method TEXT,
|
||||
body BLOB,
|
||||
timestamp DATETIME
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create http_requests table: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS http_responses (
|
||||
id INTEGER PRIMARY KEY,
|
||||
req_id INTEGER REFERENCES http_requests(id) ON DELETE CASCADE,
|
||||
proto TEXT,
|
||||
status_code INTEGER,
|
||||
status_reason TEXT,
|
||||
body BLOB,
|
||||
timestamp DATETIME
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create http_responses table: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS http_headers (
|
||||
id INTEGER PRIMARY KEY,
|
||||
req_id INTEGER REFERENCES http_requests(id) ON DELETE CASCADE,
|
||||
res_id INTEGER REFERENCES http_responses(id) ON DELETE CASCADE,
|
||||
key TEXT,
|
||||
value TEXT
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create http_headers table: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS settings (
|
||||
module TEXT PRIMARY KEY,
|
||||
settings TEXT
|
||||
)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create settings table: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close uses the underlying database if it's open.
|
||||
func (c *Client) Close() error {
|
||||
if c.db == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := c.db.Close(); err != nil {
|
||||
return fmt.Errorf("sqlite: could not close database: %w", err)
|
||||
}
|
||||
|
||||
c.db = nil
|
||||
c.activeProject = ""
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteProject(name string) error {
|
||||
if err := os.Remove(filepath.Join(c.dbPath, name+".db")); err != nil {
|
||||
return fmt.Errorf("sqlite: could not remove database file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var reqFieldToColumnMap = map[string]string{
|
||||
"proto": "proto AS req_proto",
|
||||
"url": "url",
|
||||
"method": "method",
|
||||
"body": "body AS req_body",
|
||||
"timestamp": "timestamp AS req_timestamp",
|
||||
}
|
||||
|
||||
var resFieldToColumnMap = map[string]string{
|
||||
"requestId": "req_id AS res_req_id",
|
||||
"proto": "proto AS res_proto",
|
||||
"statusCode": "status_code",
|
||||
"statusReason": "status_reason",
|
||||
"body": "body AS res_body",
|
||||
"timestamp": "timestamp AS res_timestamp",
|
||||
}
|
||||
|
||||
var headerFieldToColumnMap = map[string]string{
|
||||
"key": "key",
|
||||
"value": "value",
|
||||
}
|
||||
|
||||
func (c *Client) ClearRequestLogs(ctx context.Context) error {
|
||||
if c.db == nil {
|
||||
return proj.ErrNoProject
|
||||
}
|
||||
|
||||
_, err := c.db.Exec("DELETE FROM http_requests")
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite: could not delete requests: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) FindRequestLogs(
|
||||
ctx context.Context,
|
||||
filter reqlog.FindRequestsFilter,
|
||||
scope *scope.Scope,
|
||||
) (reqLogs []reqlog.Request, err error) {
|
||||
if c.db == nil {
|
||||
return nil, proj.ErrNoProject
|
||||
}
|
||||
|
||||
httpReqLogsQuery := parseHTTPRequestLogsQuery(ctx)
|
||||
|
||||
reqQuery := sq.
|
||||
Select(httpReqLogsQuery.requestCols...).
|
||||
From("http_requests req").
|
||||
OrderBy("req.id DESC")
|
||||
if httpReqLogsQuery.joinResponse {
|
||||
reqQuery = reqQuery.LeftJoin("http_responses res ON req.id = res.req_id")
|
||||
}
|
||||
|
||||
if filter.OnlyInScope && scope != nil {
|
||||
var ruleExpr []sq.Sqlizer
|
||||
|
||||
for _, rule := range scope.Rules() {
|
||||
if rule.URL != nil {
|
||||
ruleExpr = append(ruleExpr, sq.Expr("regexp(?, req.url)", rule.URL.String()))
|
||||
}
|
||||
}
|
||||
|
||||
if len(ruleExpr) > 0 {
|
||||
reqQuery = reqQuery.Where(sq.Or(ruleExpr))
|
||||
}
|
||||
}
|
||||
|
||||
if filter.SearchExpr != nil {
|
||||
sqlizer, err := parseSearchExpr(filter.SearchExpr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not parse search expression: %w", err)
|
||||
}
|
||||
|
||||
reqQuery = reqQuery.Where(sqlizer)
|
||||
}
|
||||
|
||||
sql, args, err := reqQuery.ToSql()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not parse query: %w", err)
|
||||
}
|
||||
|
||||
rows, err := c.db.QueryxContext(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not execute query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var dto httpRequest
|
||||
|
||||
err = rows.StructScan(&dto)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not scan row: %w", err)
|
||||
}
|
||||
|
||||
reqLogs = append(reqLogs, dto.toRequestLog())
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not iterate over rows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
if err := c.queryHeaders(ctx, httpReqLogsQuery, reqLogs); err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not query headers: %w", err)
|
||||
}
|
||||
|
||||
return reqLogs, nil
|
||||
}
|
||||
|
||||
func (c *Client) FindRequestLogByID(ctx context.Context, id int64) (reqlog.Request, error) {
|
||||
if c.db == nil {
|
||||
return reqlog.Request{}, proj.ErrNoProject
|
||||
}
|
||||
|
||||
httpReqLogsQuery := parseHTTPRequestLogsQuery(ctx)
|
||||
reqQuery := sq.
|
||||
Select(httpReqLogsQuery.requestCols...).
|
||||
From("http_requests req").
|
||||
Where("req.id = ?")
|
||||
|
||||
if httpReqLogsQuery.joinResponse {
|
||||
reqQuery = reqQuery.LeftJoin("http_responses res ON req.id = res.req_id")
|
||||
}
|
||||
|
||||
reqSQL, _, err := reqQuery.ToSql()
|
||||
if err != nil {
|
||||
return reqlog.Request{}, fmt.Errorf("sqlite: could not parse query: %w", err)
|
||||
}
|
||||
|
||||
row := c.db.QueryRowxContext(ctx, reqSQL, id)
|
||||
|
||||
var dto httpRequest
|
||||
|
||||
err = row.StructScan(&dto)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return reqlog.Request{}, reqlog.ErrRequestNotFound
|
||||
} else if err != nil {
|
||||
return reqlog.Request{}, fmt.Errorf("sqlite: could not scan row: %w", err)
|
||||
}
|
||||
|
||||
reqLog := dto.toRequestLog()
|
||||
reqLogs := []reqlog.Request{reqLog}
|
||||
|
||||
if err := c.queryHeaders(ctx, httpReqLogsQuery, reqLogs); err != nil {
|
||||
return reqlog.Request{}, fmt.Errorf("sqlite: could not query headers: %w", err)
|
||||
}
|
||||
|
||||
return reqLogs[0], nil
|
||||
}
|
||||
|
||||
func (c *Client) AddRequestLog(
|
||||
ctx context.Context,
|
||||
req http.Request,
|
||||
body []byte,
|
||||
timestamp time.Time,
|
||||
) (*reqlog.Request, error) {
|
||||
if c.db == nil {
|
||||
return nil, proj.ErrNoProject
|
||||
}
|
||||
|
||||
reqLog := &reqlog.Request{
|
||||
Request: req,
|
||||
Body: body,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
tx, err := c.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not start transaction: %w", err)
|
||||
}
|
||||
|
||||
defer tx.Rollback()
|
||||
|
||||
reqStmt, err := tx.PrepareContext(ctx, `INSERT INTO http_requests (
|
||||
proto,
|
||||
url,
|
||||
method,
|
||||
body,
|
||||
timestamp
|
||||
) VALUES (?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not prepare statement: %w", err)
|
||||
}
|
||||
defer reqStmt.Close()
|
||||
|
||||
result, err := reqStmt.ExecContext(ctx,
|
||||
reqLog.Request.Proto,
|
||||
reqLog.Request.URL.String(),
|
||||
reqLog.Request.Method,
|
||||
reqLog.Body,
|
||||
reqLog.Timestamp,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not execute statement: %w", err)
|
||||
}
|
||||
|
||||
reqID, err := result.LastInsertId()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not get last insert ID: %w", err)
|
||||
}
|
||||
|
||||
reqLog.ID = reqID
|
||||
|
||||
headerStmt, err := tx.PrepareContext(ctx, `INSERT INTO http_headers (
|
||||
req_id,
|
||||
key,
|
||||
value
|
||||
) VALUES (?, ?, ?)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not prepare statement: %w", err)
|
||||
}
|
||||
defer headerStmt.Close()
|
||||
|
||||
err = insertHeaders(ctx, headerStmt, reqID, reqLog.Request.Header)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not insert http headers: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return reqLog, nil
|
||||
}
|
||||
|
||||
func (c *Client) AddResponseLog(
|
||||
ctx context.Context,
|
||||
reqID int64,
|
||||
res http.Response,
|
||||
body []byte,
|
||||
timestamp time.Time,
|
||||
) (*reqlog.Response, error) {
|
||||
if c.db == nil {
|
||||
return nil, proj.ErrNoProject
|
||||
}
|
||||
|
||||
resLog := &reqlog.Response{
|
||||
RequestID: reqID,
|
||||
Response: res,
|
||||
Body: body,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
tx, err := c.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not start transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
resStmt, err := tx.PrepareContext(ctx, `INSERT INTO http_responses (
|
||||
req_id,
|
||||
proto,
|
||||
status_code,
|
||||
status_reason,
|
||||
body,
|
||||
timestamp
|
||||
) VALUES (?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not prepare statement: %w", err)
|
||||
}
|
||||
defer resStmt.Close()
|
||||
|
||||
var statusReason string
|
||||
if len(resLog.Response.Status) > 4 {
|
||||
statusReason = resLog.Response.Status[4:]
|
||||
}
|
||||
|
||||
result, err := resStmt.ExecContext(ctx,
|
||||
resLog.RequestID,
|
||||
resLog.Response.Proto,
|
||||
resLog.Response.StatusCode,
|
||||
statusReason,
|
||||
resLog.Body,
|
||||
resLog.Timestamp,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not execute statement: %w", err)
|
||||
}
|
||||
|
||||
resID, err := result.LastInsertId()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not get last insert ID: %w", err)
|
||||
}
|
||||
|
||||
resLog.ID = resID
|
||||
|
||||
headerStmt, err := tx.PrepareContext(ctx, `INSERT INTO http_headers (
|
||||
res_id,
|
||||
key,
|
||||
value
|
||||
) VALUES (?, ?, ?)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not prepare statement: %w", err)
|
||||
}
|
||||
defer headerStmt.Close()
|
||||
|
||||
err = insertHeaders(ctx, headerStmt, resID, resLog.Response.Header)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not insert http headers: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not commit transaction: %w", err)
|
||||
}
|
||||
|
||||
return resLog, nil
|
||||
}
|
||||
|
||||
func (c *Client) UpsertSettings(ctx context.Context, module string, settings interface{}) error {
|
||||
if c.db == nil {
|
||||
// TODO: Fix where `ErrNoProject` lives.
|
||||
return proj.ErrNoProject
|
||||
}
|
||||
|
||||
jsonSettings, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite: could not encode settings as JSON: %w", err)
|
||||
}
|
||||
|
||||
_, err = c.db.ExecContext(ctx,
|
||||
`INSERT INTO settings (module, settings) VALUES (?, ?)
|
||||
ON CONFLICT(module) DO UPDATE SET settings = ?`, module, jsonSettings, jsonSettings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite: could not insert scope settings: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) FindSettingsByModule(ctx context.Context, module string, settings interface{}) error {
|
||||
if c.db == nil {
|
||||
return proj.ErrNoProject
|
||||
}
|
||||
|
||||
var jsonSettings []byte
|
||||
|
||||
row := c.db.QueryRowContext(ctx, `SELECT settings FROM settings WHERE module = ?`, module)
|
||||
|
||||
err := row.Scan(&jsonSettings)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return proj.ErrNoSettings
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("sqlite: could not scan row: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(jsonSettings, &settings); err != nil {
|
||||
return fmt.Errorf("sqlite: could not decode settings from JSON: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertHeaders(ctx context.Context, stmt *sql.Stmt, id int64, headers http.Header) error {
|
||||
for key, values := range headers {
|
||||
for _, value := range values {
|
||||
if _, err := stmt.ExecContext(ctx, id, key, value); err != nil {
|
||||
return fmt.Errorf("could not execute statement: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findHeaders(ctx context.Context, stmt *sql.Stmt, id int64) (http.Header, error) {
|
||||
headers := make(http.Header)
|
||||
|
||||
rows, err := stmt.QueryContext(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not execute query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var key, value string
|
||||
|
||||
err := rows.Scan(&key, &value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not scan row: %w", err)
|
||||
}
|
||||
|
||||
headers.Add(key, value)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sqlite: could not iterate over rows: %w", err)
|
||||
}
|
||||
|
||||
return headers, nil
|
||||
}
|
||||
|
||||
func parseHTTPRequestLogsQuery(ctx context.Context) httpRequestLogsQuery {
|
||||
var (
|
||||
joinResponse bool
|
||||
reqHeaderCols, resHeaderCols []string
|
||||
)
|
||||
|
||||
opCtx := graphql.GetOperationContext(ctx)
|
||||
reqFields := graphql.CollectFieldsCtx(ctx, nil)
|
||||
reqCols := []string{"req.id AS req_id", "res.id AS res_id"}
|
||||
|
||||
for _, reqField := range reqFields {
|
||||
if col, ok := reqFieldToColumnMap[reqField.Name]; ok {
|
||||
reqCols = append(reqCols, "req."+col)
|
||||
}
|
||||
|
||||
if reqField.Name == "headers" {
|
||||
headerFields := graphql.CollectFields(opCtx, reqField.Selections, nil)
|
||||
for _, headerField := range headerFields {
|
||||
if col, ok := headerFieldToColumnMap[headerField.Name]; ok {
|
||||
reqHeaderCols = append(reqHeaderCols, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if reqField.Name == "response" {
|
||||
joinResponse = true
|
||||
resFields := graphql.CollectFields(opCtx, reqField.Selections, nil)
|
||||
|
||||
for _, resField := range resFields {
|
||||
if resField.Name == "headers" {
|
||||
reqCols = append(reqCols, "res.id AS res_id")
|
||||
headerFields := graphql.CollectFields(opCtx, resField.Selections, nil)
|
||||
|
||||
for _, headerField := range headerFields {
|
||||
if col, ok := headerFieldToColumnMap[headerField.Name]; ok {
|
||||
resHeaderCols = append(resHeaderCols, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if col, ok := resFieldToColumnMap[resField.Name]; ok {
|
||||
reqCols = append(reqCols, "res."+col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return httpRequestLogsQuery{
|
||||
requestCols: reqCols,
|
||||
requestHeaderCols: reqHeaderCols,
|
||||
responseHeaderCols: resHeaderCols,
|
||||
joinResponse: joinResponse,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) queryHeaders(
|
||||
ctx context.Context,
|
||||
query httpRequestLogsQuery,
|
||||
reqLogs []reqlog.Request,
|
||||
) error {
|
||||
if len(query.requestHeaderCols) > 0 {
|
||||
reqHeadersQuery, _, err := sq.
|
||||
Select(query.requestHeaderCols...).
|
||||
From("http_headers").Where("req_id = ?").
|
||||
ToSql()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse request headers query: %w", err)
|
||||
}
|
||||
|
||||
reqHeadersStmt, err := c.db.PrepareContext(ctx, reqHeadersQuery)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not prepare statement: %w", err)
|
||||
}
|
||||
defer reqHeadersStmt.Close()
|
||||
|
||||
for i := range reqLogs {
|
||||
headers, err := findHeaders(ctx, reqHeadersStmt, reqLogs[i].ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not query request headers: %w", err)
|
||||
}
|
||||
|
||||
reqLogs[i].Request.Header = headers
|
||||
}
|
||||
}
|
||||
|
||||
if len(query.responseHeaderCols) > 0 {
|
||||
resHeadersQuery, _, err := sq.
|
||||
Select(query.responseHeaderCols...).
|
||||
From("http_headers").Where("res_id = ?").
|
||||
ToSql()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse response headers query: %w", err)
|
||||
}
|
||||
|
||||
resHeadersStmt, err := c.db.PrepareContext(ctx, resHeadersQuery)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not prepare statement: %w", err)
|
||||
}
|
||||
defer resHeadersStmt.Close()
|
||||
|
||||
for i := range reqLogs {
|
||||
if reqLogs[i].Response == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
headers, err := findHeaders(ctx, resHeadersStmt, reqLogs[i].Response.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not query response headers: %w", err)
|
||||
}
|
||||
|
||||
reqLogs[i].Response.Response.Header = headers
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) IsOpen() bool {
|
||||
return c.db != nil
|
||||
}
|
||||
Reference in New Issue
Block a user