diff --git a/internal/net/http/accesslog/access_logger_test.go b/internal/net/http/accesslog/access_logger_test.go index fa690dc..6818ebe 100644 --- a/internal/net/http/accesslog/access_logger_test.go +++ b/internal/net/http/accesslog/access_logger_test.go @@ -81,7 +81,7 @@ func TestAccessLoggerCombined(t *testing.T) { func TestAccessLoggerRedactQuery(t *testing.T) { config := DefaultConfig() config.Format = FormatCommon - config.Fields.Query.DefaultMode = FieldModeRedact + config.Fields.Query.Default = FieldModeRedact ExpectEqual(t, fmtLog(config), fmt.Sprintf("%s %s - - [%s] \"%s %s %s\" %d %d", host, remote, TestTimeNow, method, uriRedacted, proto, status, contentLength, diff --git a/internal/net/http/accesslog/config.go b/internal/net/http/accesslog/config.go index 1554dfe..2760803 100644 --- a/internal/net/http/accesslog/config.go +++ b/internal/net/http/accesslog/config.go @@ -5,22 +5,22 @@ import "github.com/yusing/go-proxy/internal/utils" type ( Format string Filters struct { - StatusCodes LogFilter[*StatusCodeRange] - Method LogFilter[HTTPMethod] - Headers LogFilter[*HTTPHeader] // header exists or header == value - CIDR LogFilter[*CIDR] + StatusCodes LogFilter[*StatusCodeRange] `json:"status_codes"` + Method LogFilter[HTTPMethod] `json:"method"` + Headers LogFilter[*HTTPHeader] `json:"headers"` // header exists or header == value + CIDR LogFilter[*CIDR] `json:"cidr"` } Fields struct { - Headers FieldConfig - Query FieldConfig - Cookies FieldConfig + Headers FieldConfig `json:"headers"` + Query FieldConfig `json:"query"` + Cookies FieldConfig `json:"cookies"` } Config struct { - BufferSize uint - Format Format `validate:"oneof=common combined json"` - Path string `validate:"required"` - Filters Filters - Fields Fields + BufferSize uint `json:"buffer_size" validate:"gte=1"` + Format Format `json:"format" validate:"oneof=common combined json"` + Path string `json:"path" validate:"required"` + Filters Filters `json:"filters"` + Fields Fields `json:"fields"` } ) @@ -38,13 +38,13 @@ func DefaultConfig() *Config { Format: FormatCombined, Fields: Fields{ Headers: FieldConfig{ - DefaultMode: FieldModeDrop, + Default: FieldModeDrop, }, Query: FieldConfig{ - DefaultMode: FieldModeKeep, + Default: FieldModeKeep, }, Cookies: FieldConfig{ - DefaultMode: FieldModeDrop, + Default: FieldModeDrop, }, }, } diff --git a/internal/net/http/accesslog/config_test.go b/internal/net/http/accesslog/config_test.go index f7e0989..125b501 100644 --- a/internal/net/http/accesslog/config_test.go +++ b/internal/net/http/accesslog/config_test.go @@ -19,11 +19,11 @@ func TestNewConfig(t *testing.T) { "proxy.filters.headers.values": "foo=bar, baz", "proxy.filters.headers.negative": "true", "proxy.filters.cidr.values": "192.168.10.0/24", - "proxy.fields.headers.default_mode": "keep", + "proxy.fields.headers.default": "keep", "proxy.fields.headers.config.foo": "redact", - "proxy.fields.query.default_mode": "drop", + "proxy.fields.query.default": "drop", "proxy.fields.query.config.foo": "keep", - "proxy.fields.cookies.default_mode": "redact", + "proxy.fields.cookies.default": "redact", "proxy.fields.cookies.config.foo": "keep", } parsed, err := docker.ParseLabels(labels) @@ -44,10 +44,10 @@ func TestNewConfig(t *testing.T) { ExpectTrue(t, config.Filters.Headers.Negative) ExpectEqual(t, len(config.Filters.CIDR.Values), 1) ExpectEqual(t, config.Filters.CIDR.Values[0].String(), "192.168.10.0/24") - ExpectEqual(t, config.Fields.Headers.DefaultMode, FieldModeKeep) + ExpectEqual(t, config.Fields.Headers.Default, FieldModeKeep) ExpectEqual(t, config.Fields.Headers.Config["foo"], FieldModeRedact) - ExpectEqual(t, config.Fields.Query.DefaultMode, FieldModeDrop) + ExpectEqual(t, config.Fields.Query.Default, FieldModeDrop) ExpectEqual(t, config.Fields.Query.Config["foo"], FieldModeKeep) - ExpectEqual(t, config.Fields.Cookies.DefaultMode, FieldModeRedact) + ExpectEqual(t, config.Fields.Cookies.Default, FieldModeRedact) ExpectEqual(t, config.Fields.Cookies.Config["foo"], FieldModeKeep) } diff --git a/internal/net/http/accesslog/fields.go b/internal/net/http/accesslog/fields.go index 7ad85c7..f785b94 100644 --- a/internal/net/http/accesslog/fields.go +++ b/internal/net/http/accesslog/fields.go @@ -7,8 +7,8 @@ import ( type ( FieldConfig struct { - DefaultMode FieldMode `validate:"oneof=keep drop redact"` - Config map[string]FieldMode `validate:"dive,oneof=keep drop redact"` + Default FieldMode `json:"default" validate:"oneof=keep drop redact"` + Config map[string]FieldMode `json:"config" validate:"dive,oneof=keep drop redact"` } FieldMode string ) @@ -23,7 +23,7 @@ const ( func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string]V { if len(cfg.Config) == 0 { - switch cfg.DefaultMode { + switch cfg.Default { case FieldModeKeep: return m case FieldModeDrop: @@ -46,7 +46,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string var mode FieldMode var ok bool if mode, ok = cfg.Config[k]; !ok { - mode = cfg.DefaultMode + mode = cfg.Default } switch mode { case FieldModeKeep: @@ -60,7 +60,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) string, convert func(V) VReturn, redact func(V) VReturn) map[string]VReturn { if len(s) == 0 || - len(cfg.Config) == 0 && cfg.DefaultMode == FieldModeDrop { + len(cfg.Config) == 0 && cfg.Default == FieldModeDrop { return nil } newMap := make(map[string]VReturn, len(s)) @@ -69,7 +69,7 @@ func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) st var ok bool k := getKey(v) if mode, ok = cfg.Config[k]; !ok { - mode = cfg.DefaultMode + mode = cfg.Default } switch mode { case FieldModeKeep: diff --git a/internal/net/http/accesslog/fields_test.go b/internal/net/http/accesslog/fields_test.go index 241901f..feac44d 100644 --- a/internal/net/http/accesslog/fields_test.go +++ b/internal/net/http/accesslog/fields_test.go @@ -11,21 +11,45 @@ import ( // stored in JSONLogEntry.Cookies instead. func TestAccessLoggerJSONKeepHeaders(t *testing.T) { config := DefaultConfig() - config.Fields.Headers.DefaultMode = FieldModeKeep + config.Fields.Headers.Default = FieldModeKeep entry := getJSONEntry(t, config) - ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0) for k, v := range req.Header { if k != "Cookie" { ExpectDeepEqual(t, entry.Headers[k], v) } } + + config.Fields.Headers.Config = map[string]FieldMode{ + "Referer": FieldModeRedact, + "User-Agent": FieldModeDrop, + } + entry = getJSONEntry(t, config) + ExpectDeepEqual(t, entry.Headers["Referer"], []string{RedactedValue}) + ExpectDeepEqual(t, entry.Headers["User-Agent"], nil) +} + +func TestAccessLoggerJSONDropHeaders(t *testing.T) { + config := DefaultConfig() + config.Fields.Headers.Default = FieldModeDrop + entry := getJSONEntry(t, config) + for k := range req.Header { + ExpectDeepEqual(t, entry.Headers[k], nil) + } + + config.Fields.Headers.Config = map[string]FieldMode{ + "Referer": FieldModeKeep, + "User-Agent": FieldModeRedact, + } + entry = getJSONEntry(t, config) + ExpectDeepEqual(t, entry.Headers["Referer"], []string{req.Header.Get("Referer")}) + ExpectDeepEqual(t, entry.Headers["User-Agent"], []string{RedactedValue}) } func TestAccessLoggerJSONRedactHeaders(t *testing.T) { config := DefaultConfig() - config.Fields.Headers.DefaultMode = FieldModeRedact + config.Fields.Headers.Default = FieldModeRedact entry := getJSONEntry(t, config) - ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0) + ExpectEqual(t, len(entry.Headers["Cookie"]), 0) for k := range req.Header { if k != "Cookie" { ExpectDeepEqual(t, entry.Headers[k], []string{RedactedValue}) @@ -35,10 +59,10 @@ func TestAccessLoggerJSONRedactHeaders(t *testing.T) { func TestAccessLoggerJSONKeepCookies(t *testing.T) { config := DefaultConfig() - config.Fields.Headers.DefaultMode = FieldModeKeep - config.Fields.Cookies.DefaultMode = FieldModeKeep + config.Fields.Headers.Default = FieldModeKeep + config.Fields.Cookies.Default = FieldModeKeep entry := getJSONEntry(t, config) - ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0) + ExpectEqual(t, len(entry.Headers["Cookie"]), 0) for _, cookie := range req.Cookies() { ExpectEqual(t, entry.Cookies[cookie.Name], cookie.Value) } @@ -46,10 +70,10 @@ func TestAccessLoggerJSONKeepCookies(t *testing.T) { func TestAccessLoggerJSONRedactCookies(t *testing.T) { config := DefaultConfig() - config.Fields.Headers.DefaultMode = FieldModeKeep - config.Fields.Cookies.DefaultMode = FieldModeRedact + config.Fields.Headers.Default = FieldModeKeep + config.Fields.Cookies.Default = FieldModeRedact entry := getJSONEntry(t, config) - ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0) + ExpectEqual(t, len(entry.Headers["Cookie"]), 0) for _, cookie := range req.Cookies() { ExpectEqual(t, entry.Cookies[cookie.Name], RedactedValue) } @@ -57,7 +81,7 @@ func TestAccessLoggerJSONRedactCookies(t *testing.T) { func TestAccessLoggerJSONDropQuery(t *testing.T) { config := DefaultConfig() - config.Fields.Query.DefaultMode = FieldModeDrop + config.Fields.Query.Default = FieldModeDrop entry := getJSONEntry(t, config) ExpectDeepEqual(t, entry.Query["foo"], nil) ExpectDeepEqual(t, entry.Query["bar"], nil) @@ -65,7 +89,7 @@ func TestAccessLoggerJSONDropQuery(t *testing.T) { func TestAccessLoggerJSONRedactQuery(t *testing.T) { config := DefaultConfig() - config.Fields.Query.DefaultMode = FieldModeRedact + config.Fields.Query.Default = FieldModeRedact entry := getJSONEntry(t, config) ExpectDeepEqual(t, entry.Query["foo"], []string{RedactedValue}) ExpectDeepEqual(t, entry.Query["bar"], []string{RedactedValue}) diff --git a/internal/net/http/accesslog/filter.go b/internal/net/http/accesslog/filter.go index c92457a..00fc1b7 100644 --- a/internal/net/http/accesslog/filter.go +++ b/internal/net/http/accesslog/filter.go @@ -6,6 +6,7 @@ import ( "strings" E "github.com/yusing/go-proxy/internal/error" + "github.com/yusing/go-proxy/internal/net/types" ) type ( @@ -21,9 +22,7 @@ type ( HTTPHeader struct { Key, Value string } - CIDR struct { - *net.IPNet - } + CIDR struct{ types.CIDR } ) var ErrInvalidHTTPHeaderFilter = E.New("invalid http header filter") @@ -80,16 +79,7 @@ func (k *HTTPHeader) Fulfill(req *http.Request, res *http.Response) bool { return false } -func (cidr *CIDR) Parse(v string) error { - _, ipnet, err := net.ParseCIDR(v) - if err != nil { - return err - } - cidr.IPNet = ipnet - return nil -} - -func (cidr *CIDR) Fulfill(req *http.Request, res *http.Response) bool { +func (cidr CIDR) Fulfill(req *http.Request, res *http.Response) bool { ip, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { ip = req.RemoteAddr diff --git a/internal/net/http/accesslog/formatter.go b/internal/net/http/accesslog/formatter.go index 1851f28..9ad9350 100644 --- a/internal/net/http/accesslog/formatter.go +++ b/internal/net/http/accesslog/formatter.go @@ -123,7 +123,7 @@ func (f JSONFormatter) Format(line *bytes.Buffer, req *http.Request, res *http.R entry.Error = res.Status } - if entry.ContentType != "" { + if entry.ContentType == "" { // try to get content type from request entry.ContentType = req.Header.Get("Content-Type") } diff --git a/internal/net/http/middleware/cloudflare_real_ip.go b/internal/net/http/middleware/cloudflare_real_ip.go index 970d33e..bd75770 100644 --- a/internal/net/http/middleware/cloudflare_real_ip.go +++ b/internal/net/http/middleware/cloudflare_real_ip.go @@ -122,7 +122,7 @@ func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error { return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line) } - *cfCIDRs = append(*cfCIDRs, cidr) + *cfCIDRs = append(*cfCIDRs, (*types.CIDR)(cidr)) } return nil diff --git a/internal/net/types/cidr.go b/internal/net/types/cidr.go index 09638b4..1aa00b9 100644 --- a/internal/net/types/cidr.go +++ b/internal/net/types/cidr.go @@ -2,6 +2,33 @@ package types import ( "net" + "strings" ) -type CIDR = net.IPNet +//nolint:recvcheck +type CIDR net.IPNet + +func (cidr *CIDR) Parse(v string) error { + if !strings.Contains(v, "/") { + v += "/32" // single IP + } + _, ipnet, err := net.ParseCIDR(v) + if err != nil { + return err + } + cidr.IP = ipnet.IP + cidr.Mask = ipnet.Mask + return nil +} + +func (cidr CIDR) Contains(ip net.IP) bool { + return (*net.IPNet)(&cidr).Contains(ip) +} + +func (cidr CIDR) String() string { + return (*net.IPNet)(&cidr).String() +} + +func (cidr CIDR) MarshalText() ([]byte, error) { + return []byte(cidr.String()), nil +} diff --git a/internal/utils/serialization.go b/internal/utils/serialization.go index a4f6202..a78f905 100644 --- a/internal/utils/serialization.go +++ b/internal/utils/serialization.go @@ -3,7 +3,6 @@ package utils import ( "encoding/json" "errors" - "net" "os" "reflect" "strconv" @@ -250,7 +249,7 @@ func Deserialize(src SerializedObject, dst any) E.Error { tmp := New(mapVT).Elem() err := Convert(reflect.ValueOf(src[k]), tmp) if err == nil { - dstV.SetMapIndex(reflect.ValueOf(strutils.ToLowerNoSnake(k)), tmp) + dstV.SetMapIndex(reflect.ValueOf(k), tmp) } else { errs.Add(err.Subject(k)) } @@ -367,6 +366,7 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E dst.Set(New(dstT.Elem())) } dst = dst.Elem() + dstT = dst.Type() } if dst.Kind() == reflect.String { dst.SetString(src) @@ -384,16 +384,6 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E } dst.Set(reflect.ValueOf(d)) return - case reflect.TypeFor[net.IPNet](): - if !strings.Contains(src, "/") { - src += "/32" // single IP - } - _, ipnet, err := net.ParseCIDR(src) - if err != nil { - return true, E.From(err) - } - dst.Set(reflect.ValueOf(ipnet).Elem()) - return default: } if dstKind := dst.Kind(); isIntFloat(dstKind) {