chrissewell/hetty
1
0
Fork 0
mirror of https://github.com/dstotijn/hetty.git synced 2025-07-01 18:47:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Selectively query DB based on GraphQL query field collection

Browse Source 
Fixes #5
This commit is contained in:
David Stotijn
2020-10-05 18:34:41 +02:00
parent 073bcea565
commit 5f4bff0155
12 changed files with 354 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
pkg/api/generated.go
View File

@ -59,12 +59,12 @@ type ComplexityRoot struct {
}
HTTPResponseLog struct {
Body func(childComplexity int) int
Headers func(childComplexity int) int
Proto func(childComplexity int) int
RequestID func(childComplexity int) int
Status func(childComplexity int) int
StatusCode func(childComplexity int) int
Body func(childComplexity int) int
Headers func(childComplexity int) int
Proto func(childComplexity int) int
RequestID func(childComplexity int) int
StatusCode func(childComplexity int) int
StatusReason func(childComplexity int) int
}
Query struct {
@ -191,13 +191,6 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.HTTPResponseLog.RequestID(childComplexity), true
case "HttpResponseLog.status":
if e.complexity.HTTPResponseLog.Status == nil {
break
}
return e.complexity.HTTPResponseLog.Status(childComplexity), true
case "HttpResponseLog.statusCode":
if e.complexity.HTTPResponseLog.StatusCode == nil {
break
@ -205,6 +198,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.HTTPResponseLog.StatusCode(childComplexity), true
case "HttpResponseLog.statusReason":
if e.complexity.HTTPResponseLog.StatusReason == nil {
break
}
return e.complexity.HTTPResponseLog.StatusReason(childComplexity), true
case "Query.httpRequestLog":
if e.complexity.Query.HTTPRequestLog == nil {
break
@ -288,8 +288,8 @@ var sources = []*ast.Source{
type HttpResponseLog {
requestId: ID!
proto: String!
status: String!
statusCode: Int!
statusReason: String!
body: String
headers: [HttpHeader!]!
}
@ -791,40 +791,6 @@ func (ec *executionContext) _HttpResponseLog_proto(ctx context.Context, field gr
return ec.marshalNString2string(ctx, field.Selections, res)
}
func (ec *executionContext) _HttpResponseLog_status(ctx context.Context, field graphql.CollectedField, obj *HTTPResponseLog) (ret graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
fc := &graphql.FieldContext{
Object: "HttpResponseLog",
Field: field,
Args: nil,
IsMethod: false,
}
ctx = graphql.WithFieldContext(ctx, fc)
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Status, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
ec.Errorf(ctx, "must not be null")
}
return graphql.Null
}
res := resTmp.(string)
fc.Result = res
return ec.marshalNString2string(ctx, field.Selections, res)
}
func (ec *executionContext) _HttpResponseLog_statusCode(ctx context.Context, field graphql.CollectedField, obj *HTTPResponseLog) (ret graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
@ -859,6 +825,40 @@ func (ec *executionContext) _HttpResponseLog_statusCode(ctx context.Context, fie
return ec.marshalNInt2int(ctx, field.Selections, res)
}
func (ec *executionContext) _HttpResponseLog_statusReason(ctx context.Context, field graphql.CollectedField, obj *HTTPResponseLog) (ret graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
fc := &graphql.FieldContext{
Object: "HttpResponseLog",
Field: field,
Args: nil,
IsMethod: false,
}
ctx = graphql.WithFieldContext(ctx, fc)
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.StatusReason, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
ec.Errorf(ctx, "must not be null")
}
return graphql.Null
}
res := resTmp.(string)
fc.Result = res
return ec.marshalNString2string(ctx, field.Selections, res)
}
func (ec *executionContext) _HttpResponseLog_body(ctx context.Context, field graphql.CollectedField, obj *HTTPResponseLog) (ret graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
@ -2237,13 +2237,13 @@ func (ec *executionContext) _HttpResponseLog(ctx context.Context, sel ast.Select
if out.Values[i] == graphql.Null {
invalids++
}
case "status":
out.Values[i] = ec._HttpResponseLog_status(ctx, field, obj)
case "statusCode":
out.Values[i] = ec._HttpResponseLog_statusCode(ctx, field, obj)
if out.Values[i] == graphql.Null {
invalids++
}
case "statusCode":
out.Values[i] = ec._HttpResponseLog_statusCode(ctx, field, obj)
case "statusReason":
out.Values[i] = ec._HttpResponseLog_statusReason(ctx, field, obj)
if out.Values[i] == graphql.Null {
invalids++
}

12
pkg/api/models_gen.go
View File

@ -26,12 +26,12 @@ type HTTPRequestLog struct {
}
type HTTPResponseLog struct {
RequestID int64 `json:"requestId"`
Proto string `json:"proto"`
Status string `json:"status"`
StatusCode int `json:"statusCode"`
Body *string `json:"body"`
Headers []HTTPHeader `json:"headers"`
RequestID int64 `json:"requestId"`
Proto string `json:"proto"`
StatusCode int `json:"statusCode"`
StatusReason string `json:"statusReason"`
Body *string `json:"body"`
Headers []HTTPHeader `json:"headers"`
}
type HTTPMethod string

15
pkg/api/resolvers.go
View File

@ -5,6 +5,7 @@ package api
import (
"context"
"fmt"
"strings"
"github.com/dstotijn/hetty/pkg/reqlog"
)
@ -54,18 +55,21 @@ func (r *queryResolver) HTTPRequestLog(ctx context.Context, id int64) (*HTTPRequ
func parseRequestLog(req reqlog.Request) (HTTPRequestLog, error) {
method := HTTPMethod(req.Request.Method)
if !method.IsValid() {
if method != "" && !method.IsValid() {
return HTTPRequestLog{}, fmt.Errorf("request has invalid method: %v", method)
}
log := HTTPRequestLog{
ID: req.ID,
URL: req.Request.URL.String(),
Proto: req.Request.Proto,
Method: method,
Timestamp: req.Timestamp,
}
if req.Request.URL != nil {
log.URL = req.Request.URL.String()
}
if len(req.Body) > 0 {
reqBody := string(req.Body)
log.Body = &reqBody
@ -85,11 +89,14 @@ func parseRequestLog(req reqlog.Request) (HTTPRequestLog, error) {
if req.Response != nil {
log.Response = &HTTPResponseLog{
RequestID: req.ID,
RequestID: req.Response.RequestID,
Proto: req.Response.Response.Proto,
Status: req.Response.Response.Status,
StatusCode: req.Response.Response.StatusCode,
}
statusReasonSubs := strings.SplitN(req.Response.Response.Status, " ", 2)
if len(statusReasonSubs) == 2 {
log.Response.StatusReason = statusReasonSubs[1]
}
if len(req.Response.Body) > 0 {
resBody := string(req.Response.Body)
log.Response.Body = &resBody

2
pkg/api/schema.graphql
View File

@ -12,8 +12,8 @@ type HttpRequestLog {
type HttpResponseLog {
requestId: ID!
proto: String!
status: String!
statusCode: Int!
statusReason: String!
body: String
headers: [HttpHeader!]!
}

83
pkg/db/sqlite/dto.go
View File

@ -1,11 +1,82 @@
package sqlite
import "time"
import (
"database/sql"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
"github.com/dstotijn/hetty/pkg/reqlog"
)
type reqURL url.URL
type httpRequest struct {
ID int64 `db:"req_id"`
Proto string `db:"req_proto"`
URL reqURL `db:"url"`
Method string `db:"method"`
Body []byte `db:"req_body"`
Timestamp time.Time `db:"req_timestamp"`
httpResponse
}
type httpResponse struct {
ID *int64
Proto *string
StatusCode *int
Body *[]byte
Timestamp *time.Time
ID sql.NullInt64 `db:"res_id"`
RequestID sql.NullInt64 `db:"res_req_id"`
Proto sql.NullString `db:"res_proto"`
StatusCode sql.NullInt64 `db:"status_code"`
StatusReason sql.NullString `db:"status_reason"`
Body []byte `db:"res_body"`
Timestamp sql.NullTime `db:"res_timestamp"`
}
// Value implements driver.Valuer.
func (u *reqURL) Scan(value interface{}) error {
rawURL, ok := value.(string)
if !ok {
return errors.New("sqlite: cannot scan non-string value")
}
parsed, err := url.Parse(rawURL)
if err != nil {
return fmt.Errorf("sqlite: could not parse URL: %v", err)
}
*u = reqURL(*parsed)
return nil
}
func (dto httpRequest) toRequestLog() reqlog.Request {
u := url.URL(dto.URL)
reqLog := reqlog.Request{
ID: dto.ID,
Request: http.Request{
Proto: dto.Proto,
Method: dto.Method,
URL: &u,
},
Body: dto.Body,
Timestamp: dto.Timestamp,
}
if dto.httpResponse.ID.Valid {
reqLog.Response = &reqlog.Response{
ID: dto.httpResponse.ID.Int64,
RequestID: dto.httpResponse.RequestID.Int64,
Response: http.Response{
Status: strconv.FormatInt(dto.StatusCode.Int64, 10) + " " + dto.StatusReason.String,
StatusCode: int(dto.StatusCode.Int64),
Proto: dto.httpResponse.Proto.String,
},
Body: dto.httpResponse.Body,
Timestamp: dto.httpResponse.Timestamp.Time,
}
}
return reqLog
}

350
pkg/db/sqlite/sqlite.go
View File

@ -8,19 +8,29 @@ import (
"net/url"
"os"
"path/filepath"
"strconv"
"time"
"github.com/dstotijn/hetty/pkg/reqlog"
"github.com/dstotijn/hetty/pkg/scope"
"github.com/99designs/gqlgen/graphql"
sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx"
// Register sqlite3 for use via database/sql.
_ "github.com/mattn/go-sqlite3"
)
// Client implements reqlog.Repository.
type Client struct {
db *sql.DB
db *sqlx.DB
}
type httpRequestLogsQuery struct {
requestCols []string
requestHeaderCols []string
responseHeaderCols []string
joinResponse bool
}
// New returns a new Client.
@ -36,7 +46,7 @@ func New(filename string) (*Client, error) {
opts.Set("_foreign_keys", "1")
dsn := fmt.Sprintf("file:%v?%v", filename, opts.Encode())
db, err := sql.Open("sqlite3", dsn)
db, err := sqlx.Open("sqlite3", dsn)
if err != nil {
return nil, err
}
@ -99,213 +109,107 @@ func (c *Client) Close() error {
return c.db.Close()
}
var reqFieldToColumnMap = map[string]string{
"proto": "proto AS req_proto",
"url": "url",
"method": "method",
"body": "body AS req_body",
"timestamp": "timestamp AS req_timestamp",
}
var resFieldToColumnMap = map[string]string{
"requestId": "req_id AS res_req_id",
"proto": "proto AS res_proto",
"statusCode": "status_code",
"statusReason": "status_reason",
"body": "body AS res_body",
"timestamp": "timestamp AS res_timestamp",
}
var headerFieldToColumnMap = map[string]string{
"key": "key",
"value": "value",
}
func (c *Client) FindRequestLogs(
ctx context.Context,
opts reqlog.FindRequestsOptions,
scope *scope.Scope,
) (reqLogs []reqlog.Request, err error) {
// TODO: Pass GraphQL field collections upstream, so we can query only
// requested fields.
// TODO: Use opts and scope to filter.
reqQuery := `SELECT
req.id,
req.proto,
req.url,
req.method,
req.body,
req.timestamp,
res.id,
res.proto,
res.status_code,
res.status_reason,
res.body,
res.timestamp
FROM http_requests req
LEFT JOIN http_responses res ON req.id = res.req_id
ORDER BY req.id DESC`
httpReqLogsQuery := parseHTTPRequestLogsQuery(ctx)
rows, err := c.db.QueryContext(ctx, reqQuery)
reqQuery := sq.
Select(httpReqLogsQuery.requestCols...).
From("http_requests req").
OrderBy("req.id DESC")
if httpReqLogsQuery.joinResponse {
reqQuery = reqQuery.LeftJoin("http_responses res ON req.id = res.req_id")
}
sql, _, err := reqQuery.ToSql()
if err != nil {
return nil, fmt.Errorf("sqlite: could not parse query: %v", err)
}
rows, err := c.db.QueryxContext(ctx, sql, nil)
if err != nil {
return nil, fmt.Errorf("sqlite: could not execute query: %v", err)
}
defer rows.Close()
for rows.Next() {
var reqLog reqlog.Request
var resDTO httpResponse
var statusReason *string
var rawURL string
err := rows.Scan(
&reqLog.ID,
&reqLog.Request.Proto,
&rawURL,
&reqLog.Request.Method,
&reqLog.Body,
&reqLog.Timestamp,
&resDTO.ID,
&resDTO.Proto,
&resDTO.StatusCode,
&statusReason,
&resDTO.Body,
&resDTO.Timestamp,
)
var dto httpRequest
err = rows.StructScan(&dto)
if err != nil {
return nil, fmt.Errorf("sqlite: could not scan row: %v", err)
}
u, err := url.Parse(rawURL)
if err != nil {
return nil, fmt.Errorf("sqlite: could not parse URL: %v", err)
}
reqLog.Request.URL = u
if resDTO.ID != nil {
status := strconv.Itoa(*resDTO.StatusCode) + " " + *statusReason
reqLog.Response = &reqlog.Response{
ID: *resDTO.ID,
RequestID: reqLog.ID,
Response: http.Response{
Status: status,
StatusCode: *resDTO.StatusCode,
Proto: *resDTO.Proto,
},
Body: *resDTO.Body,
Timestamp: *resDTO.Timestamp,
}
}
reqLogs = append(reqLogs, reqLog)
reqLogs = append(reqLogs, dto.toRequestLog())
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("sqlite: could not iterate over rows: %v", err)
}
rows.Close()
reqHeadersStmt, err := c.db.PrepareContext(ctx, `SELECT key, value FROM http_headers WHERE req_id = ?`)
if err != nil {
return nil, fmt.Errorf("sqlite: could not prepare statement: %v", err)
}
defer reqHeadersStmt.Close()
resHeadersStmt, err := c.db.PrepareContext(ctx, `SELECT key, value FROM http_headers WHERE res_id = ?`)
if err != nil {
return nil, fmt.Errorf("sqlite: could not prepare statement: %v", err)
}
defer resHeadersStmt.Close()
for _, reqLog := range reqLogs {
headers, err := findHeaders(ctx, reqHeadersStmt, reqLog.ID)
if err != nil {
return nil, fmt.Errorf("sqlite: could not query request headers: %v", err)
}
reqLog.Request.Header = headers
if reqLog.Response != nil {
headers, err := findHeaders(ctx, resHeadersStmt, reqLog.Response.ID)
if err != nil {
return nil, fmt.Errorf("sqlite: could not query response headers: %v", err)
}
reqLog.Response.Response.Header = headers
}
if err := c.queryHeaders(ctx, httpReqLogsQuery, reqLogs); err != nil {
return nil, fmt.Errorf("sqlite: could not query headers: %v", err)
}
return reqLogs, nil
}
func (c *Client) FindRequestLogByID(ctx context.Context, id int64) (reqlog.Request, error) {
// TODO: Pass GraphQL field collections upstream, so we can query only
// requested fields.
reqQuery := `SELECT
req.id,
req.proto,
req.url,
req.method,
req.body,
req.timestamp,
res.id,
res.proto,
res.status_code,
res.status_reason,
res.body,
res.timestamp
FROM http_requests req
LEFT JOIN http_responses res ON req.id = res.req_id
WHERE req_id = ?
ORDER BY req.id DESC`
httpReqLogsQuery := parseHTTPRequestLogsQuery(ctx)
var reqLog reqlog.Request
var resDTO httpResponse
var statusReason *string
var rawURL string
reqQuery := sq.
Select(httpReqLogsQuery.requestCols...).
From("http_requests req").
Where("req.id = ?")
if httpReqLogsQuery.joinResponse {
reqQuery = reqQuery.LeftJoin("http_responses res ON req.id = res.req_id")
}
err := c.db.QueryRowContext(ctx, reqQuery, id).Scan(
&reqLog.ID,
&reqLog.Request.Proto,
&rawURL,
&reqLog.Request.Method,
&reqLog.Body,
&reqLog.Timestamp,
&resDTO.ID,
&resDTO.Proto,
&resDTO.StatusCode,
&statusReason,
&resDTO.Body,
&resDTO.Timestamp,
)
reqSQL, _, err := reqQuery.ToSql()
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not parse query: %v", err)
}
row := c.db.QueryRowxContext(ctx, reqSQL, id)
var dto httpRequest
err = row.StructScan(&dto)
if err == sql.ErrNoRows {
return reqlog.Request{}, reqlog.ErrRequestNotFound
}
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not scan row: %v", err)
}
reqLog := dto.toRequestLog()
u, err := url.Parse(rawURL)
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not parse URL: %v", err)
}
reqLog.Request.URL = u
if resDTO.ID != nil {
status := strconv.Itoa(*resDTO.StatusCode) + " " + *statusReason
reqLog.Response = &reqlog.Response{
ID: *resDTO.ID,
RequestID: reqLog.ID,
Response: http.Response{
Status: status,
StatusCode: *resDTO.StatusCode,
Proto: *resDTO.Proto,
},
Body: *resDTO.Body,
Timestamp: *resDTO.Timestamp,
}
reqLogs := []reqlog.Request{reqLog}
if err := c.queryHeaders(ctx, httpReqLogsQuery, reqLogs); err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not query headers: %v", err)
}
reqHeadersStmt, err := c.db.PrepareContext(ctx, `SELECT key, value FROM http_headers WHERE req_id = ?`)
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not prepare statement: %v", err)
}
defer reqHeadersStmt.Close()
resHeadersStmt, err := c.db.PrepareContext(ctx, `SELECT key, value FROM http_headers WHERE res_id = ?`)
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not prepare statement: %v", err)
}
defer resHeadersStmt.Close()
headers, err := findHeaders(ctx, reqHeadersStmt, reqLog.ID)
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not query request headers: %v", err)
}
reqLog.Request.Header = headers
if reqLog.Response != nil {
headers, err := findHeaders(ctx, resHeadersStmt, reqLog.Response.ID)
if err != nil {
return reqlog.Request{}, fmt.Errorf("sqlite: could not query response headers: %v", err)
}
reqLog.Response.Response.Header = headers
}
return reqLog, nil
return reqLogs[0], nil
}
func (c *Client) AddRequestLog(
@ -321,7 +225,7 @@ func (c *Client) AddRequestLog(
Timestamp: timestamp,
}
tx, err := c.db.BeginTx(ctx, nil)
tx, err := c.db.BeginTxx(ctx, nil)
if err != nil {
return nil, fmt.Errorf("sqlite: could not start transaction: %v", err)
}
@ -491,3 +395,103 @@ func findHeaders(ctx context.Context, stmt *sql.Stmt, id int64) (http.Header, er
return headers, nil
}
func parseHTTPRequestLogsQuery(ctx context.Context) httpRequestLogsQuery {
var joinResponse bool
var reqHeaderCols, resHeaderCols []string
opCtx := graphql.GetOperationContext(ctx)
reqFields := graphql.CollectFieldsCtx(ctx, nil)
reqCols := []string{"req.id AS req_id", "res.id AS res_id"}
for _, reqField := range reqFields {
if col, ok := reqFieldToColumnMap[reqField.Name]; ok {
reqCols = append(reqCols, "req."+col)
}
if reqField.Name == "headers" {
headerFields := graphql.CollectFields(opCtx, reqField.Selections, nil)
for _, headerField := range headerFields {
if col, ok := headerFieldToColumnMap[headerField.Name]; ok {
reqHeaderCols = append(reqHeaderCols, col)
}
}
}
if reqField.Name == "response" {
joinResponse = true
resFields := graphql.CollectFields(opCtx, reqField.Selections, nil)
for _, resField := range resFields {
if resField.Name == "headers" {
reqCols = append(reqCols, "res.id AS res_id")
headerFields := graphql.CollectFields(opCtx, resField.Selections, nil)
for _, headerField := range headerFields {
if col, ok := headerFieldToColumnMap[headerField.Name]; ok {
resHeaderCols = append(resHeaderCols, col)
}
}
}
if col, ok := resFieldToColumnMap[resField.Name]; ok {
reqCols = append(reqCols, "res."+col)
}
}
}
}
return httpRequestLogsQuery{
requestCols: reqCols,
requestHeaderCols: reqHeaderCols,
responseHeaderCols: resHeaderCols,
joinResponse: joinResponse,
}
}
func (c *Client) queryHeaders(
ctx context.Context,
query httpRequestLogsQuery,
reqLogs []reqlog.Request,
) error {
if len(query.requestHeaderCols) > 0 {
reqHeadersQuery, _, err := sq.
Select(query.requestHeaderCols...).
From("http_headers").Where("req_id = ?").
ToSql()
if err != nil {
return fmt.Errorf("could not parse request headers query: %v", err)
}
reqHeadersStmt, err := c.db.PrepareContext(ctx, reqHeadersQuery)
if err != nil {
return fmt.Errorf("could not prepare statement: %v", err)
}
defer reqHeadersStmt.Close()
for i := range reqLogs {
headers, err := findHeaders(ctx, reqHeadersStmt, reqLogs[i].ID)
if err != nil {
return fmt.Errorf("could not query request headers: %v", err)
}
reqLogs[i].Request.Header = headers
}
}
if len(query.responseHeaderCols) > 0 {
resHeadersQuery, _, err := sq.
Select(query.responseHeaderCols...).
From("http_headers").Where("res_id = ?").
ToSql()
if err != nil {
return fmt.Errorf("could not parse response headers query: %v", err)
}
resHeadersStmt, err := c.db.PrepareContext(ctx, resHeadersQuery)
if err != nil {
return fmt.Errorf("could not prepare statement: %v", err)
}
defer resHeadersStmt.Close()
for i := range reqLogs {
headers, err := findHeaders(ctx, resHeadersStmt, reqLogs[i].Response.ID)
if err != nil {
return fmt.Errorf("could not query response headers: %v", err)
}
reqLogs[i].Response.Response.Header = headers
}
}
return nil
}