added real_ip and cloudflare_real_ip middlewares, fixed that some middlewares does not work properly

Makefile

@@ -15,7 +15,7 @@ build:
 		go build -ldflags '${BUILD_FLAG}' -pgo=auto -o bin/go-proxy ./cmd
 
 test:
-	go test ./internal/...
+	GOPROXY_TEST=1 go test ./internal/...
 
 up:
 	docker compose up -d
@@ -32,6 +32,9 @@ get:
 debug:
 	make BUILD_FLAG="" build && sudo GOPROXY_DEBUG=1 bin/go-proxy
 
+run-test:
+	make BUILD_FLAG="" build && sudo GOPROXY_TEST=1 bin/go-proxy
+
 run:
 	make build && sudo bin/go-proxy
 

internal/common/env.go

@@ -11,7 +11,8 @@ import (
 
 var (
 	NoSchemaValidation = GetEnvBool("GOPROXY_NO_SCHEMA_VALIDATION", false)
-	IsDebug            = GetEnvBool("GOPROXY_DEBUG", false)
+	IsTest             = GetEnvBool("GOPROXY_TEST", false)
+	IsDebug            = GetEnvBool("GOPROXY_DEBUG", IsTest)
 
 	ProxyHTTPAddr,
 	ProxyHTTPHost,

internal/error/builder.go

@@ -50,7 +50,7 @@ func (b Builder) Build() NestedError {
 	if len(b.errors) == 0 {
 		return nil
 	} else if len(b.errors) == 1 {
-		return b.errors[0]
+		return b.errors[0].Subjectf("%s", b.message)
 	}
 	return Join(b.message, b.errors...)
 }

internal/net/http/middleware/cloudflare_real_ip.go

@@ -0,0 +1,118 @@
+package middleware
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/yusing/go-proxy/internal/common"
+	E "github.com/yusing/go-proxy/internal/error"
+)
+
+const (
+	cfIPv4CIDRsEndpoint        = "https://www.cloudflare.com/ips-v4"
+	cfIPv6CIDRsEndpoint        = "https://www.cloudflare.com/ips-v6"
+	cfCIDRsUpdateInterval      = time.Hour
+	cfCIDRsUpdateRetryInterval = 3 * time.Second
+)
+
+var (
+	cfCIDRsLastUpdate time.Time
+	cfCIDRsMu         sync.Mutex
+	cfCIDRsLogger     = logrus.WithField("middleware", "CloudflareRealIP")
+)
+
+var CloudflareRealIP = &realIP{
+	m: &Middleware{
+		withOptions: NewCloudflareRealIP,
+	},
+}
+
+func NewCloudflareRealIP(_ OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) {
+	cri := new(realIP)
+	cri.m = &Middleware{
+		impl: cri,
+		rewrite: func(r *Request) {
+			cidrs := tryFetchCFCIDR()
+			if cidrs != nil {
+				cri.From = cidrs
+			}
+			cri.setRealIP(r)
+		},
+	}
+	cri.realIPOpts = &realIPOpts{
+		Header:    "CF-Connecting-IP",
+		Recursive: true,
+	}
+	return cri.m, nil
+}
+
+func tryFetchCFCIDR() (cfCIDRs []*net.IPNet) {
+	if time.Since(cfCIDRsLastUpdate) < cfCIDRsUpdateInterval {
+		return
+	}
+
+	cfCIDRsMu.Lock()
+	defer cfCIDRsMu.Unlock()
+
+	if time.Since(cfCIDRsLastUpdate) < cfCIDRsUpdateInterval {
+		return
+	}
+
+	if common.IsTest {
+		cfCIDRs = []*net.IPNet{
+			{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 0, 0, 0)},
+			{IP: net.IPv4(10, 0, 0, 0), Mask: net.IPv4Mask(255, 0, 0, 0)},
+			{IP: net.IPv4(172, 16, 0, 0), Mask: net.IPv4Mask(255, 255, 0, 0)},
+			{IP: net.IPv4(192, 168, 0, 0), Mask: net.IPv4Mask(255, 255, 255, 0)},
+		}
+	} else {
+		cfCIDRs = make([]*net.IPNet, 0, 30)
+		err := errors.Join(
+			fetchUpdateCFIPRange(cfIPv4CIDRsEndpoint, cfCIDRs),
+			fetchUpdateCFIPRange(cfIPv6CIDRsEndpoint, cfCIDRs),
+		)
+		if err != nil {
+			cfCIDRsLastUpdate = time.Now().Add(-cfCIDRsUpdateRetryInterval - cfCIDRsUpdateInterval)
+			cfCIDRsLogger.Errorf("failed to update cloudflare range: %s, retry in %s", err, cfCIDRsUpdateRetryInterval)
+			return nil
+		}
+	}
+
+	cfCIDRsLastUpdate = time.Now()
+	cfCIDRsLogger.Debugf("cloudflare CIDR range updated")
+	return
+}
+
+func fetchUpdateCFIPRange(endpoint string, cfCIDRs []*net.IPNet) error {
+	resp, err := http.Get(endpoint)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	for _, line := range strings.Split(string(body), "\n") {
+		if line == "" {
+			continue
+		}
+		_, cidr, err := net.ParseCIDR(line)
+		if err != nil {
+			return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line)
+		} else {
+			cfCIDRs = append(cfCIDRs, cidr)
+		}
+	}
+
+	return nil
+}

internal/net/http/middleware/custom_error_page.go

@@ -11,7 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/yusing/go-proxy/internal/api/v1/error_page"
 	"github.com/yusing/go-proxy/internal/common"
-	gpHTTP "github.com/yusing/go-proxy/internal/http"
+	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
 )
 
 var CustomErrorPage = &Middleware{

internal/net/http/middleware/forward_auth.go

@@ -17,8 +17,7 @@ import (
 	"github.com/yusing/go-proxy/internal/common"
 	D "github.com/yusing/go-proxy/internal/docker"
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/http"
-	U "github.com/yusing/go-proxy/internal/utils"
+	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
 )
 
 type (
@@ -44,50 +43,50 @@ const (
 	xForwardedPort   = "X-Forwarded-Port"
 )
 
-var ForwardAuth = newForwardAuth()
-var faLogger = logrus.WithField("middleware", "ForwardAuth")
-
-func newForwardAuth() (fa *forwardAuth) {
-	fa = new(forwardAuth)
+var ForwardAuth = func() *forwardAuth {
+	fa := new(forwardAuth)
 	fa.m = new(Middleware)
 	fa.m.labelParserMap = D.ValueParserMap{
 		"trust_forward_header":         D.BoolParser,
 		"auth_response_headers":        D.YamlStringListParser,
 		"add_auth_cookies_to_response": D.YamlStringListParser,
 	}
-	fa.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) {
-		tr, ok := rp.Transport.(*http.Transport)
-		if ok {
-			tr = tr.Clone()
-		} else {
-			tr = common.DefaultTransport.Clone()
-		}
+	fa.m.withOptions = NewForwardAuthfunc
+	return fa
+}()
+var faLogger = logrus.WithField("middleware", "ForwardAuth")
 
-		faWithOpts := new(forwardAuth)
-		faWithOpts.forwardAuthOpts = new(forwardAuthOpts)
-		faWithOpts.client = http.Client{
-			CheckRedirect: func(r *Request, via []*Request) error {
-				return http.ErrUseLastResponse
-			},
-			Timeout:   30 * time.Second,
-			Transport: tr,
-		}
-		faWithOpts.m = &Middleware{
-			impl:   faWithOpts,
-			before: faWithOpts.forward,
-		}
-
-		err := U.Deserialize(optsRaw, faWithOpts.forwardAuthOpts)
-		if err != nil {
-			return nil, E.FailWith("set options", err)
-		}
-		_, err = E.Check(url.Parse(faWithOpts.Address))
-		if err != nil {
-			return nil, E.Invalid("address", faWithOpts.Address)
-		}
-		return faWithOpts.m, nil
+func NewForwardAuthfunc(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) {
+	tr, ok := rp.Transport.(*http.Transport)
+	if ok {
+		tr = tr.Clone()
+	} else {
+		tr = common.DefaultTransport.Clone()
 	}
-	return
+
+	faWithOpts := new(forwardAuth)
+	faWithOpts.forwardAuthOpts = new(forwardAuthOpts)
+	faWithOpts.client = http.Client{
+		CheckRedirect: func(r *Request, via []*Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout:   30 * time.Second,
+		Transport: tr,
+	}
+	faWithOpts.m = &Middleware{
+		impl:   faWithOpts,
+		before: faWithOpts.forward,
+	}
+
+	err := Deserialize(optsRaw, faWithOpts.forwardAuthOpts)
+	if err != nil {
+		return nil, E.FailWith("set options", err)
+	}
+	_, err = E.Check(url.Parse(faWithOpts.Address))
+	if err != nil {
+		return nil, E.Invalid("address", faWithOpts.Address)
+	}
+	return faWithOpts.m, nil
 }
 
 func (fa *forwardAuth) forward(next http.Handler, w ResponseWriter, req *Request) {

internal/net/http/middleware/middleware.go

@@ -5,7 +5,8 @@ import (
 
 	D "github.com/yusing/go-proxy/internal/docker"
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/http"
+	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	U "github.com/yusing/go-proxy/internal/utils"
 )
 
 type (
@@ -20,7 +21,7 @@ type (
 	Cookie         = http.Cookie
 
 	BeforeFunc         func(next http.Handler, w ResponseWriter, r *Request)
-	RewriteFunc        func(req *ProxyRequest)
+	RewriteFunc        func(req *Request)
 	ModifyResponseFunc func(resp *Response) error
 	CloneWithOptFunc   func(opts OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError)
 
@@ -42,6 +43,8 @@ type (
 	}
 )
 
+var Deserialize = U.Deserialize
+
 func (m *Middleware) Name() string {
 	return m.name
 }
@@ -80,7 +83,7 @@ func PatchReverseProxy(rp *ReverseProxy, middlewares map[string]OptionsRaw) (res
 	for name, opts := range middlewares {
 		m, ok := Get(name)
 		if !ok {
-			invalidM.Addf("%s", name)
+			invalidM.Add(E.NotExist("middleware", name))
 			continue
 		}
 
@@ -118,13 +121,12 @@ func PatchReverseProxy(rp *ReverseProxy, middlewares map[string]OptionsRaw) (res
 	}
 
 	if len(rewrites) > 0 {
-		if rp.Rewrite != nil {
-			rewrites = append([]RewriteFunc{rp.Rewrite}, rewrites...)
-		}
-		rp.Rewrite = func(req *ProxyRequest) {
+		origServeHTTP = rp.ServeHTTP
+		rp.ServeHTTP = func(w http.ResponseWriter, r *http.Request) {
 			for _, rewrite := range rewrites {
-				rewrite(req)
+				rewrite(r)
 			}
+			origServeHTTP(w, r)
 		}
 	}
 

internal/net/http/middleware/middlewares.go

@@ -17,14 +17,16 @@ func Get(name string) (middleware *Middleware, ok bool) {
 // initialize middleware names and label parsers
 func init() {
 	middlewares = map[string]*Middleware{
-		"set_x_forwarded":   SetXForwarded,
-		"add_x_forwarded":   AddXForwarded,
-		"redirect_http":     RedirectHTTP,
-		"forward_auth":      ForwardAuth.m,
-		"modify_response":   ModifyResponse.m,
-		"modify_request":    ModifyRequest.m,
-		"error_page":        CustomErrorPage,
-		"custom_error_page": CustomErrorPage,
+		"set_x_forwarded":    SetXForwarded,
+		"add_x_forwarded":    AddXForwarded,
+		"redirect_http":      RedirectHTTP,
+		"forward_auth":       ForwardAuth.m,
+		"modify_response":    ModifyResponse.m,
+		"modify_request":     ModifyRequest.m,
+		"error_page":         CustomErrorPage,
+		"custom_error_page":  CustomErrorPage,
+		"real_ip":            RealIP.m,
+		"cloudflare_real_ip": CloudflareRealIP.m,
 	}
 	names := make(map[*Middleware][]string)
 	for name, m := range middlewares {

internal/net/http/middleware/modify_request.go

@@ -0,0 +1,57 @@
+package middleware
+
+import (
+	D "github.com/yusing/go-proxy/internal/docker"
+	E "github.com/yusing/go-proxy/internal/error"
+)
+
+type (
+	modifyRequest struct {
+		*modifyRequestOpts
+		m *Middleware
+	}
+	// order: set_headers -> add_headers -> hide_headers
+	modifyRequestOpts struct {
+		SetHeaders  map[string]string
+		AddHeaders  map[string]string
+		HideHeaders []string
+	}
+)
+
+var ModifyRequest = func() *modifyRequest {
+	mr := new(modifyRequest)
+	mr.m = new(Middleware)
+	mr.m.labelParserMap = D.ValueParserMap{
+		"set_headers":  D.YamlLikeMappingParser(true),
+		"add_headers":  D.YamlLikeMappingParser(true),
+		"hide_headers": D.YamlStringListParser,
+	}
+	mr.m.withOptions = NewModifyRequest
+	return mr
+}()
+
+func NewModifyRequest(optsRaw OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) {
+	mr := new(modifyRequest)
+	mr.m = &Middleware{
+		impl:    mr,
+		rewrite: mr.modifyRequest,
+	}
+	mr.modifyRequestOpts = new(modifyRequestOpts)
+	err := Deserialize(optsRaw, mr.modifyRequestOpts)
+	if err != nil {
+		return nil, E.FailWith("set options", err)
+	}
+	return mr.m, nil
+}
+
+func (mr *modifyRequest) modifyRequest(req *Request) {
+	for k, v := range mr.SetHeaders {
+		req.Header.Set(k, v)
+	}
+	for k, v := range mr.AddHeaders {
+		req.Header.Add(k, v)
+	}
+	for _, k := range mr.HideHeaders {
+		req.Header.Del(k)
+	}
+}

internal/net/http/middleware/modify_response.go

@@ -5,7 +5,6 @@ import (
 
 	D "github.com/yusing/go-proxy/internal/docker"
 	E "github.com/yusing/go-proxy/internal/error"
-	U "github.com/yusing/go-proxy/internal/utils"
 )
 
 type (
@@ -21,9 +20,7 @@ type (
 	}
 )
 
-var ModifyResponse = newModifyResponse()
-
-func newModifyResponse() (mr *modifyResponse) {
+var ModifyResponse = func() (mr *modifyResponse) {
 	mr = new(modifyResponse)
 	mr.m = new(Middleware)
 	mr.m.labelParserMap = D.ValueParserMap{
@@ -31,20 +28,22 @@ func newModifyResponse() (mr *modifyResponse) {
 		"add_headers":  D.YamlLikeMappingParser(true),
 		"hide_headers": D.YamlStringListParser,
 	}
-	mr.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) {
-		mrWithOpts := new(modifyResponse)
-		mrWithOpts.m = &Middleware{
-			impl:           mrWithOpts,
-			modifyResponse: mrWithOpts.modifyResponse,
-		}
-		mrWithOpts.modifyResponseOpts = new(modifyResponseOpts)
-		err := U.Deserialize(optsRaw, mrWithOpts.modifyResponseOpts)
-		if err != nil {
-			return nil, E.FailWith("set options", err)
-		}
-		return mrWithOpts.m, nil
-	}
+	mr.m.withOptions = NewModifyResponse
 	return
+}()
+
+func NewModifyResponse(optsRaw OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) {
+	mr := new(modifyResponse)
+	mr.m = &Middleware{
+		impl:           mr,
+		modifyResponse: mr.modifyResponse,
+	}
+	mr.modifyResponseOpts = new(modifyResponseOpts)
+	err := Deserialize(optsRaw, mr.modifyResponseOpts)
+	if err != nil {
+		return nil, E.FailWith("set options", err)
+	}
+	return mr.m, nil
 }
 
 func (mr *modifyResponse) modifyResponse(resp *http.Response) error {

internal/net/http/middleware/real_ip.go

@@ -0,0 +1,157 @@
+package middleware
+
+import (
+	"net"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	D "github.com/yusing/go-proxy/internal/docker"
+	E "github.com/yusing/go-proxy/internal/error"
+)
+
+// https://nginx.org/en/docs/http/ngx_http_realip_module.html
+
+type realIP struct {
+	*realIPOpts
+	m *Middleware
+}
+
+type realIPOpts struct {
+	// Header is the name of the header to use for the real client IP
+	Header string
+	// From is a list of Address / CIDRs to trust
+	From []*net.IPNet
+	/*
+		If recursive search is disabled,
+		the original client address that matches one of the trusted addresses is replaced by
+		the last address sent in the request header field defined by the Header field.
+		If recursive search is enabled,
+		the original client address that matches one of the trusted addresses is replaced by
+		the last non-trusted address sent in the request header field.
+	*/
+	Recursive bool
+}
+
+var RealIP = &realIP{
+	m: &Middleware{
+		labelParserMap: D.ValueParserMap{
+			"from":      CIDRListParser,
+			"recursive": D.BoolParser,
+		},
+		withOptions: NewRealIP,
+	},
+}
+
+var realIPOptsDefault = func() *realIPOpts {
+	return &realIPOpts{
+		Header: "X-Real-IP",
+		From: []*net.IPNet{
+			{IP: net.IPv4(127, 0, 0, 1), Mask: net.CIDRMask(8, 32)},
+			{IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
+			{IP: net.IPv4(172, 16, 0, 0), Mask: net.CIDRMask(12, 32)},
+			{IP: net.IPv4(192, 168, 0, 0), Mask: net.CIDRMask(16, 32)},
+			{IP: net.ParseIP("fc00::"), Mask: net.CIDRMask(7, 128)},
+			{IP: net.ParseIP("fe80::"), Mask: net.CIDRMask(10, 128)},
+		},
+	}
+}
+
+var realIPLogger = logrus.WithField("middleware", "RealIP")
+
+func NewRealIP(opts OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) {
+	riWithOpts := new(realIP)
+	riWithOpts.m = &Middleware{
+		impl:    riWithOpts,
+		rewrite: riWithOpts.setRealIP,
+	}
+	riWithOpts.realIPOpts = realIPOptsDefault()
+	err := Deserialize(opts, riWithOpts.realIPOpts)
+	if err != nil {
+		return nil, E.FailWith("set options", err)
+	}
+	return riWithOpts.m, nil
+}
+
+func CIDRListParser(s string) (any, E.NestedError) {
+	sl, err := D.YamlStringListParser(s)
+	if err != nil {
+		return nil, err
+	}
+
+	b := E.NewBuilder("invalid CIDR(s)")
+
+	CIDRs := sl.([]string)
+	res := make([]*net.IPNet, 0, len(CIDRs))
+
+	for _, cidr := range CIDRs {
+		if !strings.Contains(cidr, "/") {
+			cidr += "/32" // single IP
+		}
+		_, ipnet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			b.Add(E.Invalid("CIDR", cidr))
+			continue
+		}
+		res = append(res, ipnet)
+	}
+	return res, b.Build()
+}
+
+func (ri *realIP) isInCIDRList(ip net.IP) bool {
+	for _, CIDR := range ri.From {
+		if CIDR.Contains(ip) {
+			return true
+		}
+	}
+	// not in any CIDR
+	return false
+}
+
+func (ri *realIP) setRealIP(req *Request) {
+	clientIPStr, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		realIPLogger.Debugf("failed to split host port from %s: %s", req.RemoteAddr, err)
+	}
+	clientIP := net.ParseIP(clientIPStr)
+
+	var isTrusted = false
+	for _, CIDR := range ri.From {
+		if CIDR.Contains(clientIP) {
+			isTrusted = true
+			break
+		}
+	}
+	if !isTrusted {
+		realIPLogger.Debugf("client ip %s is not trusted", clientIP)
+		return
+	}
+
+	var realIPs = req.Header.Values(ri.Header)
+	var lastNonTrustedIP string
+
+	if len(realIPs) == 0 {
+		realIPLogger.Debugf("no real ip found in header %q", ri.Header)
+		return
+	}
+
+	if !ri.Recursive {
+		lastNonTrustedIP = realIPs[len(realIPs)-1]
+	} else {
+		for _, r := range realIPs {
+			if !ri.isInCIDRList(net.ParseIP(r)) {
+				lastNonTrustedIP = r
+			}
+		}
+		if lastNonTrustedIP == "" {
+			realIPLogger.Debugf("no non-trusted ip found in header %q", ri.Header)
+			return
+		}
+	}
+
+	req.RemoteAddr = lastNonTrustedIP
+	req.Header.Set(ri.Header, lastNonTrustedIP)
+	req.Header.Set("X-Real-IP", lastNonTrustedIP)
+	req.Header.Set("X-Forwarded-For", lastNonTrustedIP)
+
+	realIPLogger.Debugf("real ip %s", lastNonTrustedIP)
+}

internal/net/http/middleware/test_utils.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/http"
+	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
 )
 
 //go:embed test_data/sample_headers.json

internal/net/http/middleware/x_forwarded.go

@@ -0,0 +1,44 @@
+package middleware
+
+import (
+	"net"
+	"strings"
+)
+
+var AddXForwarded = &Middleware{
+	rewrite: func(req *Request) {
+		clientIP, _, err := net.SplitHostPort(req.RemoteAddr)
+		if err == nil {
+			req.Header.Set("X-Forwarded-For", clientIP)
+		} else {
+			req.Header.Del("X-Forwarded-For")
+		}
+		req.Header.Set("X-Forwarded-Host", req.Host)
+		if req.TLS == nil {
+			req.Header.Set("X-Forwarded-Proto", "http")
+		} else {
+			req.Header.Set("X-Forwarded-Proto", "https")
+		}
+	},
+}
+
+var SetXForwarded = &Middleware{
+	rewrite: func(req *Request) {
+		clientIP, _, err := net.SplitHostPort(req.RemoteAddr)
+		if err == nil {
+			prior := req.Header["X-Forwarded-For"]
+			if len(prior) > 0 {
+				clientIP = strings.Join(prior, ", ") + ", " + clientIP
+			}
+			req.Header.Set("X-Forwarded-For", clientIP)
+		} else {
+			req.Header.Del("X-Forwarded-For")
+		}
+		req.Header.Set("X-Forwarded-Host", req.Host)
+		if req.TLS == nil {
+			req.Header.Set("X-Forwarded-Proto", "http")
+		} else {
+			req.Header.Set("X-Forwarded-Proto", "https")
+		}
+	},
+}

internal/net/http/reverse_proxy_mod.go

@@ -15,7 +15,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/textproto"
@@ -58,39 +57,6 @@ type ProxyRequest struct {
 //		r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"]
 //		r.SetXForwarded()
 //	}
-func (r *ProxyRequest) SetXForwarded() {
-	clientIP, _, err := net.SplitHostPort(r.In.RemoteAddr)
-	if err == nil {
-		r.Out.Header.Set("X-Forwarded-For", clientIP)
-	} else {
-		r.Out.Header.Del("X-Forwarded-For")
-	}
-	r.Out.Header.Set("X-Forwarded-Host", r.In.Host)
-	if r.In.TLS == nil {
-		r.Out.Header.Set("X-Forwarded-Proto", "http")
-	} else {
-		r.Out.Header.Set("X-Forwarded-Proto", "https")
-	}
-}
-
-func (r *ProxyRequest) AddXForwarded() {
-	clientIP, _, err := net.SplitHostPort(r.In.RemoteAddr)
-	if err == nil {
-		prior := r.Out.Header["X-Forwarded-For"]
-		if len(prior) > 0 {
-			clientIP = strings.Join(prior, ", ") + ", " + clientIP
-		}
-		r.Out.Header.Set("X-Forwarded-For", clientIP)
-	} else {
-		r.Out.Header.Del("X-Forwarded-For")
-	}
-	r.Out.Header.Set("X-Forwarded-Host", r.In.Host)
-	if r.In.TLS == nil {
-		r.Out.Header.Set("X-Forwarded-Proto", "http")
-	} else {
-		r.Out.Header.Set("X-Forwarded-Proto", "https")
-	}
-}
 
 // ReverseProxy is an HTTP Handler that takes an incoming request and
 // sends it to another server, proxying the response back to the

internal/route/http.go

@@ -13,10 +13,10 @@ import (
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/docker/idlewatcher"
 	E "github.com/yusing/go-proxy/internal/error"
-	. "github.com/yusing/go-proxy/internal/http"
+	. "github.com/yusing/go-proxy/internal/net/http"
+	"github.com/yusing/go-proxy/internal/net/http/middleware"
 	P "github.com/yusing/go-proxy/internal/proxy"
 	PT "github.com/yusing/go-proxy/internal/proxy/fields"
-	"github.com/yusing/go-proxy/internal/route/middleware"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
 

internal/route/middleware/modify_request.go

@@ -1,58 +0,0 @@
-package middleware
-
-import (
-	D "github.com/yusing/go-proxy/internal/docker"
-	E "github.com/yusing/go-proxy/internal/error"
-	U "github.com/yusing/go-proxy/internal/utils"
-)
-
-type (
-	modifyRequest struct {
-		*modifyRequestOpts
-		m *Middleware
-	}
-	// order: set_headers -> add_headers -> hide_headers
-	modifyRequestOpts struct {
-		SetHeaders  map[string]string
-		AddHeaders  map[string]string
-		HideHeaders []string
-	}
-)
-
-var ModifyRequest = newModifyRequest()
-
-func newModifyRequest() (mr *modifyRequest) {
-	mr = new(modifyRequest)
-	mr.m = new(Middleware)
-	mr.m.labelParserMap = D.ValueParserMap{
-		"set_headers":  D.YamlLikeMappingParser(true),
-		"add_headers":  D.YamlLikeMappingParser(true),
-		"hide_headers": D.YamlStringListParser,
-	}
-	mr.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) {
-		mrWithOpts := new(modifyRequest)
-		mrWithOpts.m = &Middleware{
-			impl:    mrWithOpts,
-			rewrite: mrWithOpts.modifyRequest,
-		}
-		mrWithOpts.modifyRequestOpts = new(modifyRequestOpts)
-		err := U.Deserialize(optsRaw, mrWithOpts.modifyRequestOpts)
-		if err != nil {
-			return nil, E.FailWith("set options", err)
-		}
-		return mrWithOpts.m, nil
-	}
-	return
-}
-
-func (mr *modifyRequest) modifyRequest(req *ProxyRequest) {
-	for k, v := range mr.SetHeaders {
-		req.Out.Header.Set(k, v)
-	}
-	for k, v := range mr.AddHeaders {
-		req.Out.Header.Add(k, v)
-	}
-	for _, k := range mr.HideHeaders {
-		req.Out.Header.Del(k)
-	}
-}

internal/route/middleware/x_forwarded.go

@@ -1,9 +0,0 @@
-package middleware
-
-var AddXForwarded = &Middleware{
-	rewrite: (*ProxyRequest).AddXForwarded,
-}
-
-var SetXForwarded = &Middleware{
-	rewrite: (*ProxyRequest).SetXForwarded,
-}

internal/utils/serialization.go

@@ -107,6 +107,9 @@ func Serialize(data any) (SerializedObject, E.NestedError) {
 }
 
 func Deserialize(src SerializedObject, target any) E.NestedError {
+	if src == nil || target == nil {
+		return nil
+	}
 	// convert data fields to lower no-snake
 	// convert target fields to lower no-snake
 	// then check if the field of data is in the target