TyleoDvDelaware3217/GoDoxy
1
0
Fork 0
mirror of https://github.com/yusing/godoxy.git synced 2025-05-20 12:42:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix accesslog and serialization

Browse source
This commit is contained in:
yusing 2024-12-18 09:57:29 +08:00
parent 87279688e6
commit 57a7c04a4c
10 changed files with 99 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/net/http/accesslog/access_logger_test.go
View file

@ -81,7 +81,7 @@ func TestAccessLoggerCombined(t *testing.T) {
func TestAccessLoggerRedactQuery(t *testing.T) {
config := DefaultConfig()
config.Format = FormatCommon
config.Fields.Query.DefaultMode = FieldModeRedact
config.Fields.Query.Default = FieldModeRedact
ExpectEqual(t, fmtLog(config),
fmt.Sprintf("%s %s - - [%s] \"%s %s %s\" %d %d",
host, remote, TestTimeNow, method, uriRedacted, proto, status, contentLength,

30
internal/net/http/accesslog/config.go
View file

@ -5,22 +5,22 @@ import "github.com/yusing/go-proxy/internal/utils"
type (
Format string
Filters struct {
StatusCodes LogFilter[*StatusCodeRange]
Method LogFilter[HTTPMethod]
Headers LogFilter[*HTTPHeader] // header exists or header == value
CIDR LogFilter[*CIDR]
StatusCodes LogFilter[*StatusCodeRange] `json:"status_codes"`
Method LogFilter[HTTPMethod] `json:"method"`
Headers LogFilter[*HTTPHeader] `json:"headers"` // header exists or header == value
CIDR LogFilter[*CIDR] `json:"cidr"`
}
Fields struct {
Headers FieldConfig
Query FieldConfig
Cookies FieldConfig
Headers FieldConfig `json:"headers"`
Query FieldConfig `json:"query"`
Cookies FieldConfig `json:"cookies"`
}
Config struct {
BufferSize uint
Format Format `validate:"oneof=common combined json"`
Path string `validate:"required"`
Filters Filters
Fields Fields
BufferSize uint `json:"buffer_size" validate:"gte=1"`
Format Format `json:"format" validate:"oneof=common combined json"`
Path string `json:"path" validate:"required"`
Filters Filters `json:"filters"`
Fields Fields `json:"fields"`
}
)
@ -38,13 +38,13 @@ func DefaultConfig() *Config {
Format: FormatCombined,
Fields: Fields{
Headers: FieldConfig{
DefaultMode: FieldModeDrop,
Default: FieldModeDrop,
},
Query: FieldConfig{
DefaultMode: FieldModeKeep,
Default: FieldModeKeep,
},
Cookies: FieldConfig{
DefaultMode: FieldModeDrop,
Default: FieldModeDrop,
},
},
}

12
internal/net/http/accesslog/config_test.go
View file

@ -19,11 +19,11 @@ func TestNewConfig(t *testing.T) {
"proxy.filters.headers.values": "foo=bar, baz",
"proxy.filters.headers.negative": "true",
"proxy.filters.cidr.values": "192.168.10.0/24",
"proxy.fields.headers.default_mode": "keep",
"proxy.fields.headers.default": "keep",
"proxy.fields.headers.config.foo": "redact",
"proxy.fields.query.default_mode": "drop",
"proxy.fields.query.default": "drop",
"proxy.fields.query.config.foo": "keep",
"proxy.fields.cookies.default_mode": "redact",
"proxy.fields.cookies.default": "redact",
"proxy.fields.cookies.config.foo": "keep",
}
parsed, err := docker.ParseLabels(labels)
@ -44,10 +44,10 @@ func TestNewConfig(t *testing.T) {
ExpectTrue(t, config.Filters.Headers.Negative)
ExpectEqual(t, len(config.Filters.CIDR.Values), 1)
ExpectEqual(t, config.Filters.CIDR.Values[0].String(), "192.168.10.0/24")
ExpectEqual(t, config.Fields.Headers.DefaultMode, FieldModeKeep)
ExpectEqual(t, config.Fields.Headers.Default, FieldModeKeep)
ExpectEqual(t, config.Fields.Headers.Config["foo"], FieldModeRedact)
ExpectEqual(t, config.Fields.Query.DefaultMode, FieldModeDrop)
ExpectEqual(t, config.Fields.Query.Default, FieldModeDrop)
ExpectEqual(t, config.Fields.Query.Config["foo"], FieldModeKeep)
ExpectEqual(t, config.Fields.Cookies.DefaultMode, FieldModeRedact)
ExpectEqual(t, config.Fields.Cookies.Default, FieldModeRedact)
ExpectEqual(t, config.Fields.Cookies.Config["foo"], FieldModeKeep)
}

12
internal/net/http/accesslog/fields.go
View file

@ -7,8 +7,8 @@ import (
type (
FieldConfig struct {
DefaultMode FieldMode `validate:"oneof=keep drop redact"`
Config map[string]FieldMode `validate:"dive,oneof=keep drop redact"`
Default FieldMode `json:"default" validate:"oneof=keep drop redact"`
Config map[string]FieldMode `json:"config" validate:"dive,oneof=keep drop redact"`
}
FieldMode string
)
@ -23,7 +23,7 @@ const (
func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string]V {
if len(cfg.Config) == 0 {
switch cfg.DefaultMode {
switch cfg.Default {
case FieldModeKeep:
return m
case FieldModeDrop:
@ -46,7 +46,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string
var mode FieldMode
var ok bool
if mode, ok = cfg.Config[k]; !ok {
mode = cfg.DefaultMode
mode = cfg.Default
}
switch mode {
case FieldModeKeep:
@ -60,7 +60,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string
func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) string, convert func(V) VReturn, redact func(V) VReturn) map[string]VReturn {
if len(s) == 0 ||
len(cfg.Config) == 0 && cfg.DefaultMode == FieldModeDrop {
len(cfg.Config) == 0 && cfg.Default == FieldModeDrop {
return nil
}
newMap := make(map[string]VReturn, len(s))
@ -69,7 +69,7 @@ func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) st
var ok bool
k := getKey(v)
if mode, ok = cfg.Config[k]; !ok {
mode = cfg.DefaultMode
mode = cfg.Default
}
switch mode {
case FieldModeKeep:

48
internal/net/http/accesslog/fields_test.go
View file

@ -11,21 +11,45 @@ import (
// stored in JSONLogEntry.Cookies instead.
func TestAccessLoggerJSONKeepHeaders(t *testing.T) {
config := DefaultConfig()
config.Fields.Headers.DefaultMode = FieldModeKeep
config.Fields.Headers.Default = FieldModeKeep
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
for k, v := range req.Header {
if k != "Cookie" {
ExpectDeepEqual(t, entry.Headers[k], v)
}
}
config.Fields.Headers.Config = map[string]FieldMode{
"Referer": FieldModeRedact,
"User-Agent": FieldModeDrop,
}
entry = getJSONEntry(t, config)
ExpectDeepEqual(t, entry.Headers["Referer"], []string{RedactedValue})
ExpectDeepEqual(t, entry.Headers["User-Agent"], nil)
}
func TestAccessLoggerJSONDropHeaders(t *testing.T) {
config := DefaultConfig()
config.Fields.Headers.Default = FieldModeDrop
entry := getJSONEntry(t, config)
for k := range req.Header {
ExpectDeepEqual(t, entry.Headers[k], nil)
}
config.Fields.Headers.Config = map[string]FieldMode{
"Referer": FieldModeKeep,
"User-Agent": FieldModeRedact,
}
entry = getJSONEntry(t, config)
ExpectDeepEqual(t, entry.Headers["Referer"], []string{req.Header.Get("Referer")})
ExpectDeepEqual(t, entry.Headers["User-Agent"], []string{RedactedValue})
}
func TestAccessLoggerJSONRedactHeaders(t *testing.T) {
config := DefaultConfig()
config.Fields.Headers.DefaultMode = FieldModeRedact
config.Fields.Headers.Default = FieldModeRedact
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
for k := range req.Header {
if k != "Cookie" {
ExpectDeepEqual(t, entry.Headers[k], []string{RedactedValue})
@ -35,10 +59,10 @@ func TestAccessLoggerJSONRedactHeaders(t *testing.T) {
func TestAccessLoggerJSONKeepCookies(t *testing.T) {
config := DefaultConfig()
config.Fields.Headers.DefaultMode = FieldModeKeep
config.Fields.Cookies.DefaultMode = FieldModeKeep
config.Fields.Headers.Default = FieldModeKeep
config.Fields.Cookies.Default = FieldModeKeep
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
for _, cookie := range req.Cookies() {
ExpectEqual(t, entry.Cookies[cookie.Name], cookie.Value)
}
@ -46,10 +70,10 @@ func TestAccessLoggerJSONKeepCookies(t *testing.T) {
func TestAccessLoggerJSONRedactCookies(t *testing.T) {
config := DefaultConfig()
config.Fields.Headers.DefaultMode = FieldModeKeep
config.Fields.Cookies.DefaultMode = FieldModeRedact
config.Fields.Headers.Default = FieldModeKeep
config.Fields.Cookies.Default = FieldModeRedact
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
for _, cookie := range req.Cookies() {
ExpectEqual(t, entry.Cookies[cookie.Name], RedactedValue)
}
@ -57,7 +81,7 @@ func TestAccessLoggerJSONRedactCookies(t *testing.T) {
func TestAccessLoggerJSONDropQuery(t *testing.T) {
config := DefaultConfig()
config.Fields.Query.DefaultMode = FieldModeDrop
config.Fields.Query.Default = FieldModeDrop
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, entry.Query["foo"], nil)
ExpectDeepEqual(t, entry.Query["bar"], nil)
@ -65,7 +89,7 @@ func TestAccessLoggerJSONDropQuery(t *testing.T) {
func TestAccessLoggerJSONRedactQuery(t *testing.T) {
config := DefaultConfig()
config.Fields.Query.DefaultMode = FieldModeRedact
config.Fields.Query.Default = FieldModeRedact
entry := getJSONEntry(t, config)
ExpectDeepEqual(t, entry.Query["foo"], []string{RedactedValue})
ExpectDeepEqual(t, entry.Query["bar"], []string{RedactedValue})

16
internal/net/http/accesslog/filter.go
View file

@ -6,6 +6,7 @@ import (
"strings"
E "github.com/yusing/go-proxy/internal/error"
"github.com/yusing/go-proxy/internal/net/types"
)
type (
@ -21,9 +22,7 @@ type (
HTTPHeader struct {
Key, Value string
}
CIDR struct {
*net.IPNet
}
CIDR struct{ types.CIDR }
)
var ErrInvalidHTTPHeaderFilter = E.New("invalid http header filter")
@ -80,16 +79,7 @@ func (k *HTTPHeader) Fulfill(req *http.Request, res *http.Response) bool {
return false
}
func (cidr *CIDR) Parse(v string) error {
_, ipnet, err := net.ParseCIDR(v)
if err != nil {
return err
}
cidr.IPNet = ipnet
return nil
}
func (cidr *CIDR) Fulfill(req *http.Request, res *http.Response) bool {
func (cidr CIDR) Fulfill(req *http.Request, res *http.Response) bool {
ip, _, err := net.SplitHostPort(req.RemoteAddr)
if err != nil {
ip = req.RemoteAddr

2
internal/net/http/accesslog/formatter.go
View file

@ -123,7 +123,7 @@ func (f JSONFormatter) Format(line *bytes.Buffer, req *http.Request, res *http.R
entry.Error = res.Status
}
if entry.ContentType != "" {
if entry.ContentType == "" {
// try to get content type from request
entry.ContentType = req.Header.Get("Content-Type")
}

2
internal/net/http/middleware/cloudflare_real_ip.go
View file

@ -122,7 +122,7 @@ func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line)
}
*cfCIDRs = append(*cfCIDRs, cidr)
*cfCIDRs = append(*cfCIDRs, (*types.CIDR)(cidr))
}
return nil

29
internal/net/types/cidr.go
View file

@ -2,6 +2,33 @@ package types
import (
"net"
"strings"
)
type CIDR = net.IPNet
//nolint:recvcheck
type CIDR net.IPNet
func (cidr *CIDR) Parse(v string) error {
if !strings.Contains(v, "/") {
v += "/32" // single IP
}
_, ipnet, err := net.ParseCIDR(v)
if err != nil {
return err
}
cidr.IP = ipnet.IP
cidr.Mask = ipnet.Mask
return nil
}
func (cidr CIDR) Contains(ip net.IP) bool {
return (*net.IPNet)(&cidr).Contains(ip)
}
func (cidr CIDR) String() string {
return (*net.IPNet)(&cidr).String()
}
func (cidr CIDR) MarshalText() ([]byte, error) {
return []byte(cidr.String()), nil
}

14
internal/utils/serialization.go
View file

@ -3,7 +3,6 @@ package utils
import (
"encoding/json"
"errors"
"net"
"os"
"reflect"
"strconv"
@ -250,7 +249,7 @@ func Deserialize(src SerializedObject, dst any) E.Error {
tmp := New(mapVT).Elem()
err := Convert(reflect.ValueOf(src[k]), tmp)
if err == nil {
dstV.SetMapIndex(reflect.ValueOf(strutils.ToLowerNoSnake(k)), tmp)
dstV.SetMapIndex(reflect.ValueOf(k), tmp)
} else {
errs.Add(err.Subject(k))
}
@ -367,6 +366,7 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E
dst.Set(New(dstT.Elem()))
}
dst = dst.Elem()
dstT = dst.Type()
}
if dst.Kind() == reflect.String {
dst.SetString(src)
@ -384,16 +384,6 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E
}
dst.Set(reflect.ValueOf(d))
return
case reflect.TypeFor[net.IPNet]():
if !strings.Contains(src, "/") {
src += "/32" // single IP
}
_, ipnet, err := net.ParseCIDR(src)
if err != nil {
return true, E.From(err)
}
dst.Set(reflect.ValueOf(ipnet).Elem())
return
default:
}
if dstKind := dst.Kind(); isIntFloat(dstKind) {