fix accesslog and serialization

internal/net/http/accesslog/access_logger_test.go

@@ -81,7 +81,7 @@ func TestAccessLoggerCombined(t *testing.T) {
 func TestAccessLoggerRedactQuery(t *testing.T) {
 	config := DefaultConfig()
 	config.Format = FormatCommon
-	config.Fields.Query.DefaultMode = FieldModeRedact
+	config.Fields.Query.Default = FieldModeRedact
 	ExpectEqual(t, fmtLog(config),
 		fmt.Sprintf("%s %s - - [%s] \"%s %s %s\" %d %d",
 			host, remote, TestTimeNow, method, uriRedacted, proto, status, contentLength,

internal/net/http/accesslog/config.go

@@ -5,22 +5,22 @@ import "github.com/yusing/go-proxy/internal/utils"
 type (
 	Format  string
 	Filters struct {
-		StatusCodes LogFilter[*StatusCodeRange]
+		StatusCodes LogFilter[*StatusCodeRange] `json:"status_codes"`
-		Method      LogFilter[HTTPMethod]
+		Method      LogFilter[HTTPMethod]       `json:"method"`
-		Headers     LogFilter[*HTTPHeader] // header exists or header == value
+		Headers     LogFilter[*HTTPHeader]      `json:"headers"` // header exists or header == value
-		CIDR        LogFilter[*CIDR]
+		CIDR        LogFilter[*CIDR]            `json:"cidr"`
 	}
 	Fields struct {
-		Headers FieldConfig
+		Headers FieldConfig `json:"headers"`
-		Query   FieldConfig
+		Query   FieldConfig `json:"query"`
-		Cookies FieldConfig
+		Cookies FieldConfig `json:"cookies"`
 	}
 	Config struct {
-		BufferSize uint
+		BufferSize uint    `json:"buffer_size" validate:"gte=1"`
-		Format     Format `validate:"oneof=common combined json"`
+		Format     Format  `json:"format" validate:"oneof=common combined json"`
-		Path       string `validate:"required"`
+		Path       string  `json:"path" validate:"required"`
-		Filters    Filters
+		Filters    Filters `json:"filters"`
-		Fields     Fields
+		Fields     Fields  `json:"fields"`
 	}
 )
 
@@ -38,13 +38,13 @@ func DefaultConfig() *Config {
 		Format:     FormatCombined,
 		Fields: Fields{
 			Headers: FieldConfig{
-				DefaultMode: FieldModeDrop,
+				Default: FieldModeDrop,
 			},
 			Query: FieldConfig{
-				DefaultMode: FieldModeKeep,
+				Default: FieldModeKeep,
 			},
 			Cookies: FieldConfig{
-				DefaultMode: FieldModeDrop,
+				Default: FieldModeDrop,
 			},
 		},
 	}

internal/net/http/accesslog/config_test.go

@@ -19,11 +19,11 @@ func TestNewConfig(t *testing.T) {
 		"proxy.filters.headers.values":      "foo=bar, baz",
 		"proxy.filters.headers.negative":    "true",
 		"proxy.filters.cidr.values":         "192.168.10.0/24",
-		"proxy.fields.headers.default_mode": "keep",
+		"proxy.fields.headers.default":      "keep",
 		"proxy.fields.headers.config.foo":   "redact",
-		"proxy.fields.query.default_mode":   "drop",
+		"proxy.fields.query.default":        "drop",
 		"proxy.fields.query.config.foo":     "keep",
-		"proxy.fields.cookies.default_mode": "redact",
+		"proxy.fields.cookies.default":      "redact",
 		"proxy.fields.cookies.config.foo":   "keep",
 	}
 	parsed, err := docker.ParseLabels(labels)
@@ -44,10 +44,10 @@ func TestNewConfig(t *testing.T) {
 	ExpectTrue(t, config.Filters.Headers.Negative)
 	ExpectEqual(t, len(config.Filters.CIDR.Values), 1)
 	ExpectEqual(t, config.Filters.CIDR.Values[0].String(), "192.168.10.0/24")
-	ExpectEqual(t, config.Fields.Headers.DefaultMode, FieldModeKeep)
+	ExpectEqual(t, config.Fields.Headers.Default, FieldModeKeep)
 	ExpectEqual(t, config.Fields.Headers.Config["foo"], FieldModeRedact)
-	ExpectEqual(t, config.Fields.Query.DefaultMode, FieldModeDrop)
+	ExpectEqual(t, config.Fields.Query.Default, FieldModeDrop)
 	ExpectEqual(t, config.Fields.Query.Config["foo"], FieldModeKeep)
-	ExpectEqual(t, config.Fields.Cookies.DefaultMode, FieldModeRedact)
+	ExpectEqual(t, config.Fields.Cookies.Default, FieldModeRedact)
 	ExpectEqual(t, config.Fields.Cookies.Config["foo"], FieldModeKeep)
 }

internal/net/http/accesslog/fields.go

@@ -7,8 +7,8 @@ import (
 
 type (
 	FieldConfig struct {
-		DefaultMode FieldMode            `validate:"oneof=keep drop redact"`
+		Default FieldMode            `json:"default" validate:"oneof=keep drop redact"`
-		Config      map[string]FieldMode `validate:"dive,oneof=keep drop redact"`
+		Config  map[string]FieldMode `json:"config" validate:"dive,oneof=keep drop redact"`
 	}
 	FieldMode string
 )
@@ -23,7 +23,7 @@ const (
 
 func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string]V {
 	if len(cfg.Config) == 0 {
-		switch cfg.DefaultMode {
+		switch cfg.Default {
 		case FieldModeKeep:
 			return m
 		case FieldModeDrop:
@@ -46,7 +46,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string
 		var mode FieldMode
 		var ok bool
 		if mode, ok = cfg.Config[k]; !ok {
-			mode = cfg.DefaultMode
+			mode = cfg.Default
 		}
 		switch mode {
 		case FieldModeKeep:
@@ -60,7 +60,7 @@ func processMap[V any](cfg *FieldConfig, m map[string]V, redactedV V) map[string
 
 func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) string, convert func(V) VReturn, redact func(V) VReturn) map[string]VReturn {
 	if len(s) == 0 ||
-		len(cfg.Config) == 0 && cfg.DefaultMode == FieldModeDrop {
+		len(cfg.Config) == 0 && cfg.Default == FieldModeDrop {
 		return nil
 	}
 	newMap := make(map[string]VReturn, len(s))
@@ -69,7 +69,7 @@ func processSlice[V any, VReturn any](cfg *FieldConfig, s []V, getKey func(V) st
 		var ok bool
 		k := getKey(v)
 		if mode, ok = cfg.Config[k]; !ok {
-			mode = cfg.DefaultMode
+			mode = cfg.Default
 		}
 		switch mode {
 		case FieldModeKeep:

internal/net/http/accesslog/fields_test.go

@@ -11,21 +11,45 @@ import (
 // stored in JSONLogEntry.Cookies instead.
 func TestAccessLoggerJSONKeepHeaders(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Headers.DefaultMode = FieldModeKeep
+	config.Fields.Headers.Default = FieldModeKeep
 	entry := getJSONEntry(t, config)
-	ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
 	for k, v := range req.Header {
 		if k != "Cookie" {
 			ExpectDeepEqual(t, entry.Headers[k], v)
 		}
 	}
+
+	config.Fields.Headers.Config = map[string]FieldMode{
+		"Referer":    FieldModeRedact,
+		"User-Agent": FieldModeDrop,
+	}
+	entry = getJSONEntry(t, config)
+	ExpectDeepEqual(t, entry.Headers["Referer"], []string{RedactedValue})
+	ExpectDeepEqual(t, entry.Headers["User-Agent"], nil)
+}
+
+func TestAccessLoggerJSONDropHeaders(t *testing.T) {
+	config := DefaultConfig()
+	config.Fields.Headers.Default = FieldModeDrop
+	entry := getJSONEntry(t, config)
+	for k := range req.Header {
+		ExpectDeepEqual(t, entry.Headers[k], nil)
+	}
+
+	config.Fields.Headers.Config = map[string]FieldMode{
+		"Referer":    FieldModeKeep,
+		"User-Agent": FieldModeRedact,
+	}
+	entry = getJSONEntry(t, config)
+	ExpectDeepEqual(t, entry.Headers["Referer"], []string{req.Header.Get("Referer")})
+	ExpectDeepEqual(t, entry.Headers["User-Agent"], []string{RedactedValue})
 }
 
 func TestAccessLoggerJSONRedactHeaders(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Headers.DefaultMode = FieldModeRedact
+	config.Fields.Headers.Default = FieldModeRedact
 	entry := getJSONEntry(t, config)
-	ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
+	ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
 	for k := range req.Header {
 		if k != "Cookie" {
 			ExpectDeepEqual(t, entry.Headers[k], []string{RedactedValue})
@@ -35,10 +59,10 @@ func TestAccessLoggerJSONRedactHeaders(t *testing.T) {
 
 func TestAccessLoggerJSONKeepCookies(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Headers.DefaultMode = FieldModeKeep
+	config.Fields.Headers.Default = FieldModeKeep
-	config.Fields.Cookies.DefaultMode = FieldModeKeep
+	config.Fields.Cookies.Default = FieldModeKeep
 	entry := getJSONEntry(t, config)
-	ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
+	ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
 	for _, cookie := range req.Cookies() {
 		ExpectEqual(t, entry.Cookies[cookie.Name], cookie.Value)
 	}
@@ -46,10 +70,10 @@ func TestAccessLoggerJSONKeepCookies(t *testing.T) {
 
 func TestAccessLoggerJSONRedactCookies(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Headers.DefaultMode = FieldModeKeep
+	config.Fields.Headers.Default = FieldModeKeep
-	config.Fields.Cookies.DefaultMode = FieldModeRedact
+	config.Fields.Cookies.Default = FieldModeRedact
 	entry := getJSONEntry(t, config)
-	ExpectDeepEqual(t, len(entry.Headers["Cookie"]), 0)
+	ExpectEqual(t, len(entry.Headers["Cookie"]), 0)
 	for _, cookie := range req.Cookies() {
 		ExpectEqual(t, entry.Cookies[cookie.Name], RedactedValue)
 	}
@@ -57,7 +81,7 @@ func TestAccessLoggerJSONRedactCookies(t *testing.T) {
 
 func TestAccessLoggerJSONDropQuery(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Query.DefaultMode = FieldModeDrop
+	config.Fields.Query.Default = FieldModeDrop
 	entry := getJSONEntry(t, config)
 	ExpectDeepEqual(t, entry.Query["foo"], nil)
 	ExpectDeepEqual(t, entry.Query["bar"], nil)
@@ -65,7 +89,7 @@ func TestAccessLoggerJSONDropQuery(t *testing.T) {
 
 func TestAccessLoggerJSONRedactQuery(t *testing.T) {
 	config := DefaultConfig()
-	config.Fields.Query.DefaultMode = FieldModeRedact
+	config.Fields.Query.Default = FieldModeRedact
 	entry := getJSONEntry(t, config)
 	ExpectDeepEqual(t, entry.Query["foo"], []string{RedactedValue})
 	ExpectDeepEqual(t, entry.Query["bar"], []string{RedactedValue})

internal/net/http/accesslog/filter.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	E "github.com/yusing/go-proxy/internal/error"
+	"github.com/yusing/go-proxy/internal/net/types"
 )
 
 type (
@@ -21,9 +22,7 @@ type (
 	HTTPHeader struct {
 		Key, Value string
 	}
-	CIDR struct {
+	CIDR struct{ types.CIDR }
-		*net.IPNet
-	}
 )
 
 var ErrInvalidHTTPHeaderFilter = E.New("invalid http header filter")
@@ -80,16 +79,7 @@ func (k *HTTPHeader) Fulfill(req *http.Request, res *http.Response) bool {
 	return false
 }
 
-func (cidr *CIDR) Parse(v string) error {
+func (cidr CIDR) Fulfill(req *http.Request, res *http.Response) bool {
-	_, ipnet, err := net.ParseCIDR(v)
-	if err != nil {
-		return err
-	}
-	cidr.IPNet = ipnet
-	return nil
-}
-
-func (cidr *CIDR) Fulfill(req *http.Request, res *http.Response) bool {
 	ip, _, err := net.SplitHostPort(req.RemoteAddr)
 	if err != nil {
 		ip = req.RemoteAddr

internal/net/http/accesslog/formatter.go

@@ -123,7 +123,7 @@ func (f JSONFormatter) Format(line *bytes.Buffer, req *http.Request, res *http.R
 		entry.Error = res.Status
 	}
 
-	if entry.ContentType != "" {
+	if entry.ContentType == "" {
 		// try to get content type from request
 		entry.ContentType = req.Header.Get("Content-Type")
 	}

internal/net/http/middleware/cloudflare_real_ip.go

@@ -122,7 +122,7 @@ func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
 			return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line)
 		}
 
-		*cfCIDRs = append(*cfCIDRs, cidr)
+		*cfCIDRs = append(*cfCIDRs, (*types.CIDR)(cidr))
 	}
 
 	return nil

internal/net/types/cidr.go

@@ -2,6 +2,33 @@ package types
 
 import (
 	"net"
+	"strings"
 )
 
-type CIDR = net.IPNet
+//nolint:recvcheck
+type CIDR net.IPNet
+
+func (cidr *CIDR) Parse(v string) error {
+	if !strings.Contains(v, "/") {
+		v += "/32" // single IP
+	}
+	_, ipnet, err := net.ParseCIDR(v)
+	if err != nil {
+		return err
+	}
+	cidr.IP = ipnet.IP
+	cidr.Mask = ipnet.Mask
+	return nil
+}
+
+func (cidr CIDR) Contains(ip net.IP) bool {
+	return (*net.IPNet)(&cidr).Contains(ip)
+}
+
+func (cidr CIDR) String() string {
+	return (*net.IPNet)(&cidr).String()
+}
+
+func (cidr CIDR) MarshalText() ([]byte, error) {
+	return []byte(cidr.String()), nil
+}

internal/utils/serialization.go

@@ -3,7 +3,6 @@ package utils
 import (
 	"encoding/json"
 	"errors"
-	"net"
 	"os"
 	"reflect"
 	"strconv"
@@ -250,7 +249,7 @@ func Deserialize(src SerializedObject, dst any) E.Error {
 			tmp := New(mapVT).Elem()
 			err := Convert(reflect.ValueOf(src[k]), tmp)
 			if err == nil {
-				dstV.SetMapIndex(reflect.ValueOf(strutils.ToLowerNoSnake(k)), tmp)
+				dstV.SetMapIndex(reflect.ValueOf(k), tmp)
 			} else {
 				errs.Add(err.Subject(k))
 			}
@@ -367,6 +366,7 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E
 			dst.Set(New(dstT.Elem()))
 		}
 		dst = dst.Elem()
+		dstT = dst.Type()
 	}
 	if dst.Kind() == reflect.String {
 		dst.SetString(src)
@@ -384,16 +384,6 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr E.E
 		}
 		dst.Set(reflect.ValueOf(d))
 		return
-	case reflect.TypeFor[net.IPNet]():
-		if !strings.Contains(src, "/") {
-			src += "/32" // single IP
-		}
-		_, ipnet, err := net.ParseCIDR(src)
-		if err != nil {
-			return true, E.From(err)
-		}
-		dst.Set(reflect.ValueOf(ipnet).Elem())
-		return
 	default:
 	}
 	if dstKind := dst.Kind(); isIntFloat(dstKind) {