From 860e914b9072c9734744679e707324d8191e60bc Mon Sep 17 00:00:00 2001 From: yusing Date: Mon, 30 Sep 2024 04:03:48 +0800 Subject: [PATCH] added real_ip and cloudflare_real_ip middlewares, fixed that some middlewares does not work properly --- Makefile | 5 +- internal/common/env.go | 3 +- internal/error/builder.go | 2 +- internal/{ => net}/http/content_type.go | 0 internal/{ => net}/http/header_utils.go | 0 .../net/http/middleware/cloudflare_real_ip.go | 118 +++++++++++++ .../http}/middleware/custom_error_page.go | 2 +- .../http}/middleware/forward_auth.go | 75 +++++---- .../http}/middleware/middleware.go | 18 +- .../http}/middleware/middlewares.go | 18 +- .../net/http/middleware/modify_request.go | 57 +++++++ .../http}/middleware/modify_request_test.go | 0 .../http}/middleware/modify_response.go | 33 ++-- .../http}/middleware/modify_response_test.go | 0 internal/net/http/middleware/real_ip.go | 157 ++++++++++++++++++ .../http}/middleware/redirect_http.go | 0 .../http}/middleware/redirect_http_test.go | 0 .../middleware/test_data/sample_headers.json | 0 .../http}/middleware/test_utils.go | 2 +- internal/net/http/middleware/x_forwarded.go | 44 +++++ .../{ => net}/http/modify_response_writer.go | 0 internal/{ => net}/http/reverse_proxy_mod.go | 34 ---- internal/{ => net}/http/status_code.go | 0 internal/route/http.go | 4 +- internal/route/middleware/modify_request.go | 58 ------- internal/route/middleware/x_forwarded.go | 9 - internal/utils/serialization.go | 3 + 27 files changed, 463 insertions(+), 179 deletions(-) rename internal/{ => net}/http/content_type.go (100%) rename internal/{ => net}/http/header_utils.go (100%) create mode 100644 internal/net/http/middleware/cloudflare_real_ip.go rename internal/{route => net/http}/middleware/custom_error_page.go (97%) rename internal/{route => net/http}/middleware/forward_auth.go (83%) rename internal/{route => net/http}/middleware/middleware.go (90%) rename internal/{route => net/http}/middleware/middlewares.go (68%) create mode 100644 internal/net/http/middleware/modify_request.go rename internal/{route => net/http}/middleware/modify_request_test.go (100%) rename internal/{route => net/http}/middleware/modify_response.go (60%) rename internal/{route => net/http}/middleware/modify_response_test.go (100%) create mode 100644 internal/net/http/middleware/real_ip.go rename internal/{route => net/http}/middleware/redirect_http.go (100%) rename internal/{route => net/http}/middleware/redirect_http_test.go (100%) rename internal/{route => net/http}/middleware/test_data/sample_headers.json (100%) rename internal/{route => net/http}/middleware/test_utils.go (97%) create mode 100644 internal/net/http/middleware/x_forwarded.go rename internal/{ => net}/http/modify_response_writer.go (100%) rename internal/{ => net}/http/reverse_proxy_mod.go (94%) rename internal/{ => net}/http/status_code.go (100%) delete mode 100644 internal/route/middleware/modify_request.go delete mode 100644 internal/route/middleware/x_forwarded.go diff --git a/Makefile b/Makefile index ec76b20..7f730ba 100755 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ build: go build -ldflags '${BUILD_FLAG}' -pgo=auto -o bin/go-proxy ./cmd test: - go test ./internal/... + GOPROXY_TEST=1 go test ./internal/... up: docker compose up -d @@ -32,6 +32,9 @@ get: debug: make BUILD_FLAG="" build && sudo GOPROXY_DEBUG=1 bin/go-proxy +run-test: + make BUILD_FLAG="" build && sudo GOPROXY_TEST=1 bin/go-proxy + run: make build && sudo bin/go-proxy diff --git a/internal/common/env.go b/internal/common/env.go index 1c0c07b..e1db32d 100644 --- a/internal/common/env.go +++ b/internal/common/env.go @@ -11,7 +11,8 @@ import ( var ( NoSchemaValidation = GetEnvBool("GOPROXY_NO_SCHEMA_VALIDATION", false) - IsDebug = GetEnvBool("GOPROXY_DEBUG", false) + IsTest = GetEnvBool("GOPROXY_TEST", false) + IsDebug = GetEnvBool("GOPROXY_DEBUG", IsTest) ProxyHTTPAddr, ProxyHTTPHost, diff --git a/internal/error/builder.go b/internal/error/builder.go index 6bc2442..086287d 100644 --- a/internal/error/builder.go +++ b/internal/error/builder.go @@ -50,7 +50,7 @@ func (b Builder) Build() NestedError { if len(b.errors) == 0 { return nil } else if len(b.errors) == 1 { - return b.errors[0] + return b.errors[0].Subjectf("%s", b.message) } return Join(b.message, b.errors...) } diff --git a/internal/http/content_type.go b/internal/net/http/content_type.go similarity index 100% rename from internal/http/content_type.go rename to internal/net/http/content_type.go diff --git a/internal/http/header_utils.go b/internal/net/http/header_utils.go similarity index 100% rename from internal/http/header_utils.go rename to internal/net/http/header_utils.go diff --git a/internal/net/http/middleware/cloudflare_real_ip.go b/internal/net/http/middleware/cloudflare_real_ip.go new file mode 100644 index 0000000..8f07ec1 --- /dev/null +++ b/internal/net/http/middleware/cloudflare_real_ip.go @@ -0,0 +1,118 @@ +package middleware + +import ( + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" + "github.com/yusing/go-proxy/internal/common" + E "github.com/yusing/go-proxy/internal/error" +) + +const ( + cfIPv4CIDRsEndpoint = "https://www.cloudflare.com/ips-v4" + cfIPv6CIDRsEndpoint = "https://www.cloudflare.com/ips-v6" + cfCIDRsUpdateInterval = time.Hour + cfCIDRsUpdateRetryInterval = 3 * time.Second +) + +var ( + cfCIDRsLastUpdate time.Time + cfCIDRsMu sync.Mutex + cfCIDRsLogger = logrus.WithField("middleware", "CloudflareRealIP") +) + +var CloudflareRealIP = &realIP{ + m: &Middleware{ + withOptions: NewCloudflareRealIP, + }, +} + +func NewCloudflareRealIP(_ OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) { + cri := new(realIP) + cri.m = &Middleware{ + impl: cri, + rewrite: func(r *Request) { + cidrs := tryFetchCFCIDR() + if cidrs != nil { + cri.From = cidrs + } + cri.setRealIP(r) + }, + } + cri.realIPOpts = &realIPOpts{ + Header: "CF-Connecting-IP", + Recursive: true, + } + return cri.m, nil +} + +func tryFetchCFCIDR() (cfCIDRs []*net.IPNet) { + if time.Since(cfCIDRsLastUpdate) < cfCIDRsUpdateInterval { + return + } + + cfCIDRsMu.Lock() + defer cfCIDRsMu.Unlock() + + if time.Since(cfCIDRsLastUpdate) < cfCIDRsUpdateInterval { + return + } + + if common.IsTest { + cfCIDRs = []*net.IPNet{ + {IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 0, 0, 0)}, + {IP: net.IPv4(10, 0, 0, 0), Mask: net.IPv4Mask(255, 0, 0, 0)}, + {IP: net.IPv4(172, 16, 0, 0), Mask: net.IPv4Mask(255, 255, 0, 0)}, + {IP: net.IPv4(192, 168, 0, 0), Mask: net.IPv4Mask(255, 255, 255, 0)}, + } + } else { + cfCIDRs = make([]*net.IPNet, 0, 30) + err := errors.Join( + fetchUpdateCFIPRange(cfIPv4CIDRsEndpoint, cfCIDRs), + fetchUpdateCFIPRange(cfIPv6CIDRsEndpoint, cfCIDRs), + ) + if err != nil { + cfCIDRsLastUpdate = time.Now().Add(-cfCIDRsUpdateRetryInterval - cfCIDRsUpdateInterval) + cfCIDRsLogger.Errorf("failed to update cloudflare range: %s, retry in %s", err, cfCIDRsUpdateRetryInterval) + return nil + } + } + + cfCIDRsLastUpdate = time.Now() + cfCIDRsLogger.Debugf("cloudflare CIDR range updated") + return +} + +func fetchUpdateCFIPRange(endpoint string, cfCIDRs []*net.IPNet) error { + resp, err := http.Get(endpoint) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + for _, line := range strings.Split(string(body), "\n") { + if line == "" { + continue + } + _, cidr, err := net.ParseCIDR(line) + if err != nil { + return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line) + } else { + cfCIDRs = append(cfCIDRs, cidr) + } + } + + return nil +} diff --git a/internal/route/middleware/custom_error_page.go b/internal/net/http/middleware/custom_error_page.go similarity index 97% rename from internal/route/middleware/custom_error_page.go rename to internal/net/http/middleware/custom_error_page.go index 9df41f8..41e682f 100644 --- a/internal/route/middleware/custom_error_page.go +++ b/internal/net/http/middleware/custom_error_page.go @@ -11,7 +11,7 @@ import ( "github.com/sirupsen/logrus" "github.com/yusing/go-proxy/internal/api/v1/error_page" "github.com/yusing/go-proxy/internal/common" - gpHTTP "github.com/yusing/go-proxy/internal/http" + gpHTTP "github.com/yusing/go-proxy/internal/net/http" ) var CustomErrorPage = &Middleware{ diff --git a/internal/route/middleware/forward_auth.go b/internal/net/http/middleware/forward_auth.go similarity index 83% rename from internal/route/middleware/forward_auth.go rename to internal/net/http/middleware/forward_auth.go index 6232bd1..27256bc 100644 --- a/internal/route/middleware/forward_auth.go +++ b/internal/net/http/middleware/forward_auth.go @@ -17,8 +17,7 @@ import ( "github.com/yusing/go-proxy/internal/common" D "github.com/yusing/go-proxy/internal/docker" E "github.com/yusing/go-proxy/internal/error" - gpHTTP "github.com/yusing/go-proxy/internal/http" - U "github.com/yusing/go-proxy/internal/utils" + gpHTTP "github.com/yusing/go-proxy/internal/net/http" ) type ( @@ -44,50 +43,50 @@ const ( xForwardedPort = "X-Forwarded-Port" ) -var ForwardAuth = newForwardAuth() -var faLogger = logrus.WithField("middleware", "ForwardAuth") - -func newForwardAuth() (fa *forwardAuth) { - fa = new(forwardAuth) +var ForwardAuth = func() *forwardAuth { + fa := new(forwardAuth) fa.m = new(Middleware) fa.m.labelParserMap = D.ValueParserMap{ "trust_forward_header": D.BoolParser, "auth_response_headers": D.YamlStringListParser, "add_auth_cookies_to_response": D.YamlStringListParser, } - fa.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) { - tr, ok := rp.Transport.(*http.Transport) - if ok { - tr = tr.Clone() - } else { - tr = common.DefaultTransport.Clone() - } + fa.m.withOptions = NewForwardAuthfunc + return fa +}() +var faLogger = logrus.WithField("middleware", "ForwardAuth") - faWithOpts := new(forwardAuth) - faWithOpts.forwardAuthOpts = new(forwardAuthOpts) - faWithOpts.client = http.Client{ - CheckRedirect: func(r *Request, via []*Request) error { - return http.ErrUseLastResponse - }, - Timeout: 30 * time.Second, - Transport: tr, - } - faWithOpts.m = &Middleware{ - impl: faWithOpts, - before: faWithOpts.forward, - } - - err := U.Deserialize(optsRaw, faWithOpts.forwardAuthOpts) - if err != nil { - return nil, E.FailWith("set options", err) - } - _, err = E.Check(url.Parse(faWithOpts.Address)) - if err != nil { - return nil, E.Invalid("address", faWithOpts.Address) - } - return faWithOpts.m, nil +func NewForwardAuthfunc(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) { + tr, ok := rp.Transport.(*http.Transport) + if ok { + tr = tr.Clone() + } else { + tr = common.DefaultTransport.Clone() } - return + + faWithOpts := new(forwardAuth) + faWithOpts.forwardAuthOpts = new(forwardAuthOpts) + faWithOpts.client = http.Client{ + CheckRedirect: func(r *Request, via []*Request) error { + return http.ErrUseLastResponse + }, + Timeout: 30 * time.Second, + Transport: tr, + } + faWithOpts.m = &Middleware{ + impl: faWithOpts, + before: faWithOpts.forward, + } + + err := Deserialize(optsRaw, faWithOpts.forwardAuthOpts) + if err != nil { + return nil, E.FailWith("set options", err) + } + _, err = E.Check(url.Parse(faWithOpts.Address)) + if err != nil { + return nil, E.Invalid("address", faWithOpts.Address) + } + return faWithOpts.m, nil } func (fa *forwardAuth) forward(next http.Handler, w ResponseWriter, req *Request) { diff --git a/internal/route/middleware/middleware.go b/internal/net/http/middleware/middleware.go similarity index 90% rename from internal/route/middleware/middleware.go rename to internal/net/http/middleware/middleware.go index e02cb80..2737d67 100644 --- a/internal/route/middleware/middleware.go +++ b/internal/net/http/middleware/middleware.go @@ -5,7 +5,8 @@ import ( D "github.com/yusing/go-proxy/internal/docker" E "github.com/yusing/go-proxy/internal/error" - gpHTTP "github.com/yusing/go-proxy/internal/http" + gpHTTP "github.com/yusing/go-proxy/internal/net/http" + U "github.com/yusing/go-proxy/internal/utils" ) type ( @@ -20,7 +21,7 @@ type ( Cookie = http.Cookie BeforeFunc func(next http.Handler, w ResponseWriter, r *Request) - RewriteFunc func(req *ProxyRequest) + RewriteFunc func(req *Request) ModifyResponseFunc func(resp *Response) error CloneWithOptFunc func(opts OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) @@ -42,6 +43,8 @@ type ( } ) +var Deserialize = U.Deserialize + func (m *Middleware) Name() string { return m.name } @@ -80,7 +83,7 @@ func PatchReverseProxy(rp *ReverseProxy, middlewares map[string]OptionsRaw) (res for name, opts := range middlewares { m, ok := Get(name) if !ok { - invalidM.Addf("%s", name) + invalidM.Add(E.NotExist("middleware", name)) continue } @@ -118,13 +121,12 @@ func PatchReverseProxy(rp *ReverseProxy, middlewares map[string]OptionsRaw) (res } if len(rewrites) > 0 { - if rp.Rewrite != nil { - rewrites = append([]RewriteFunc{rp.Rewrite}, rewrites...) - } - rp.Rewrite = func(req *ProxyRequest) { + origServeHTTP = rp.ServeHTTP + rp.ServeHTTP = func(w http.ResponseWriter, r *http.Request) { for _, rewrite := range rewrites { - rewrite(req) + rewrite(r) } + origServeHTTP(w, r) } } diff --git a/internal/route/middleware/middlewares.go b/internal/net/http/middleware/middlewares.go similarity index 68% rename from internal/route/middleware/middlewares.go rename to internal/net/http/middleware/middlewares.go index 31b8c68..87e0824 100644 --- a/internal/route/middleware/middlewares.go +++ b/internal/net/http/middleware/middlewares.go @@ -17,14 +17,16 @@ func Get(name string) (middleware *Middleware, ok bool) { // initialize middleware names and label parsers func init() { middlewares = map[string]*Middleware{ - "set_x_forwarded": SetXForwarded, - "add_x_forwarded": AddXForwarded, - "redirect_http": RedirectHTTP, - "forward_auth": ForwardAuth.m, - "modify_response": ModifyResponse.m, - "modify_request": ModifyRequest.m, - "error_page": CustomErrorPage, - "custom_error_page": CustomErrorPage, + "set_x_forwarded": SetXForwarded, + "add_x_forwarded": AddXForwarded, + "redirect_http": RedirectHTTP, + "forward_auth": ForwardAuth.m, + "modify_response": ModifyResponse.m, + "modify_request": ModifyRequest.m, + "error_page": CustomErrorPage, + "custom_error_page": CustomErrorPage, + "real_ip": RealIP.m, + "cloudflare_real_ip": CloudflareRealIP.m, } names := make(map[*Middleware][]string) for name, m := range middlewares { diff --git a/internal/net/http/middleware/modify_request.go b/internal/net/http/middleware/modify_request.go new file mode 100644 index 0000000..b497459 --- /dev/null +++ b/internal/net/http/middleware/modify_request.go @@ -0,0 +1,57 @@ +package middleware + +import ( + D "github.com/yusing/go-proxy/internal/docker" + E "github.com/yusing/go-proxy/internal/error" +) + +type ( + modifyRequest struct { + *modifyRequestOpts + m *Middleware + } + // order: set_headers -> add_headers -> hide_headers + modifyRequestOpts struct { + SetHeaders map[string]string + AddHeaders map[string]string + HideHeaders []string + } +) + +var ModifyRequest = func() *modifyRequest { + mr := new(modifyRequest) + mr.m = new(Middleware) + mr.m.labelParserMap = D.ValueParserMap{ + "set_headers": D.YamlLikeMappingParser(true), + "add_headers": D.YamlLikeMappingParser(true), + "hide_headers": D.YamlStringListParser, + } + mr.m.withOptions = NewModifyRequest + return mr +}() + +func NewModifyRequest(optsRaw OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) { + mr := new(modifyRequest) + mr.m = &Middleware{ + impl: mr, + rewrite: mr.modifyRequest, + } + mr.modifyRequestOpts = new(modifyRequestOpts) + err := Deserialize(optsRaw, mr.modifyRequestOpts) + if err != nil { + return nil, E.FailWith("set options", err) + } + return mr.m, nil +} + +func (mr *modifyRequest) modifyRequest(req *Request) { + for k, v := range mr.SetHeaders { + req.Header.Set(k, v) + } + for k, v := range mr.AddHeaders { + req.Header.Add(k, v) + } + for _, k := range mr.HideHeaders { + req.Header.Del(k) + } +} diff --git a/internal/route/middleware/modify_request_test.go b/internal/net/http/middleware/modify_request_test.go similarity index 100% rename from internal/route/middleware/modify_request_test.go rename to internal/net/http/middleware/modify_request_test.go diff --git a/internal/route/middleware/modify_response.go b/internal/net/http/middleware/modify_response.go similarity index 60% rename from internal/route/middleware/modify_response.go rename to internal/net/http/middleware/modify_response.go index ea1aafb..1a77d65 100644 --- a/internal/route/middleware/modify_response.go +++ b/internal/net/http/middleware/modify_response.go @@ -5,7 +5,6 @@ import ( D "github.com/yusing/go-proxy/internal/docker" E "github.com/yusing/go-proxy/internal/error" - U "github.com/yusing/go-proxy/internal/utils" ) type ( @@ -21,9 +20,7 @@ type ( } ) -var ModifyResponse = newModifyResponse() - -func newModifyResponse() (mr *modifyResponse) { +var ModifyResponse = func() (mr *modifyResponse) { mr = new(modifyResponse) mr.m = new(Middleware) mr.m.labelParserMap = D.ValueParserMap{ @@ -31,20 +28,22 @@ func newModifyResponse() (mr *modifyResponse) { "add_headers": D.YamlLikeMappingParser(true), "hide_headers": D.YamlStringListParser, } - mr.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) { - mrWithOpts := new(modifyResponse) - mrWithOpts.m = &Middleware{ - impl: mrWithOpts, - modifyResponse: mrWithOpts.modifyResponse, - } - mrWithOpts.modifyResponseOpts = new(modifyResponseOpts) - err := U.Deserialize(optsRaw, mrWithOpts.modifyResponseOpts) - if err != nil { - return nil, E.FailWith("set options", err) - } - return mrWithOpts.m, nil - } + mr.m.withOptions = NewModifyResponse return +}() + +func NewModifyResponse(optsRaw OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) { + mr := new(modifyResponse) + mr.m = &Middleware{ + impl: mr, + modifyResponse: mr.modifyResponse, + } + mr.modifyResponseOpts = new(modifyResponseOpts) + err := Deserialize(optsRaw, mr.modifyResponseOpts) + if err != nil { + return nil, E.FailWith("set options", err) + } + return mr.m, nil } func (mr *modifyResponse) modifyResponse(resp *http.Response) error { diff --git a/internal/route/middleware/modify_response_test.go b/internal/net/http/middleware/modify_response_test.go similarity index 100% rename from internal/route/middleware/modify_response_test.go rename to internal/net/http/middleware/modify_response_test.go diff --git a/internal/net/http/middleware/real_ip.go b/internal/net/http/middleware/real_ip.go new file mode 100644 index 0000000..b624f8c --- /dev/null +++ b/internal/net/http/middleware/real_ip.go @@ -0,0 +1,157 @@ +package middleware + +import ( + "net" + "strings" + + "github.com/sirupsen/logrus" + D "github.com/yusing/go-proxy/internal/docker" + E "github.com/yusing/go-proxy/internal/error" +) + +// https://nginx.org/en/docs/http/ngx_http_realip_module.html + +type realIP struct { + *realIPOpts + m *Middleware +} + +type realIPOpts struct { + // Header is the name of the header to use for the real client IP + Header string + // From is a list of Address / CIDRs to trust + From []*net.IPNet + /* + If recursive search is disabled, + the original client address that matches one of the trusted addresses is replaced by + the last address sent in the request header field defined by the Header field. + If recursive search is enabled, + the original client address that matches one of the trusted addresses is replaced by + the last non-trusted address sent in the request header field. + */ + Recursive bool +} + +var RealIP = &realIP{ + m: &Middleware{ + labelParserMap: D.ValueParserMap{ + "from": CIDRListParser, + "recursive": D.BoolParser, + }, + withOptions: NewRealIP, + }, +} + +var realIPOptsDefault = func() *realIPOpts { + return &realIPOpts{ + Header: "X-Real-IP", + From: []*net.IPNet{ + {IP: net.IPv4(127, 0, 0, 1), Mask: net.CIDRMask(8, 32)}, + {IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(8, 32)}, + {IP: net.IPv4(172, 16, 0, 0), Mask: net.CIDRMask(12, 32)}, + {IP: net.IPv4(192, 168, 0, 0), Mask: net.CIDRMask(16, 32)}, + {IP: net.ParseIP("fc00::"), Mask: net.CIDRMask(7, 128)}, + {IP: net.ParseIP("fe80::"), Mask: net.CIDRMask(10, 128)}, + }, + } +} + +var realIPLogger = logrus.WithField("middleware", "RealIP") + +func NewRealIP(opts OptionsRaw, _ *ReverseProxy) (*Middleware, E.NestedError) { + riWithOpts := new(realIP) + riWithOpts.m = &Middleware{ + impl: riWithOpts, + rewrite: riWithOpts.setRealIP, + } + riWithOpts.realIPOpts = realIPOptsDefault() + err := Deserialize(opts, riWithOpts.realIPOpts) + if err != nil { + return nil, E.FailWith("set options", err) + } + return riWithOpts.m, nil +} + +func CIDRListParser(s string) (any, E.NestedError) { + sl, err := D.YamlStringListParser(s) + if err != nil { + return nil, err + } + + b := E.NewBuilder("invalid CIDR(s)") + + CIDRs := sl.([]string) + res := make([]*net.IPNet, 0, len(CIDRs)) + + for _, cidr := range CIDRs { + if !strings.Contains(cidr, "/") { + cidr += "/32" // single IP + } + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + b.Add(E.Invalid("CIDR", cidr)) + continue + } + res = append(res, ipnet) + } + return res, b.Build() +} + +func (ri *realIP) isInCIDRList(ip net.IP) bool { + for _, CIDR := range ri.From { + if CIDR.Contains(ip) { + return true + } + } + // not in any CIDR + return false +} + +func (ri *realIP) setRealIP(req *Request) { + clientIPStr, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + realIPLogger.Debugf("failed to split host port from %s: %s", req.RemoteAddr, err) + } + clientIP := net.ParseIP(clientIPStr) + + var isTrusted = false + for _, CIDR := range ri.From { + if CIDR.Contains(clientIP) { + isTrusted = true + break + } + } + if !isTrusted { + realIPLogger.Debugf("client ip %s is not trusted", clientIP) + return + } + + var realIPs = req.Header.Values(ri.Header) + var lastNonTrustedIP string + + if len(realIPs) == 0 { + realIPLogger.Debugf("no real ip found in header %q", ri.Header) + return + } + + if !ri.Recursive { + lastNonTrustedIP = realIPs[len(realIPs)-1] + } else { + for _, r := range realIPs { + if !ri.isInCIDRList(net.ParseIP(r)) { + lastNonTrustedIP = r + } + } + if lastNonTrustedIP == "" { + realIPLogger.Debugf("no non-trusted ip found in header %q", ri.Header) + return + } + } + + req.RemoteAddr = lastNonTrustedIP + req.Header.Set(ri.Header, lastNonTrustedIP) + req.Header.Set("X-Real-IP", lastNonTrustedIP) + req.Header.Set("X-Forwarded-For", lastNonTrustedIP) + + realIPLogger.Debugf("real ip %s", lastNonTrustedIP) +} diff --git a/internal/route/middleware/redirect_http.go b/internal/net/http/middleware/redirect_http.go similarity index 100% rename from internal/route/middleware/redirect_http.go rename to internal/net/http/middleware/redirect_http.go diff --git a/internal/route/middleware/redirect_http_test.go b/internal/net/http/middleware/redirect_http_test.go similarity index 100% rename from internal/route/middleware/redirect_http_test.go rename to internal/net/http/middleware/redirect_http_test.go diff --git a/internal/route/middleware/test_data/sample_headers.json b/internal/net/http/middleware/test_data/sample_headers.json similarity index 100% rename from internal/route/middleware/test_data/sample_headers.json rename to internal/net/http/middleware/test_data/sample_headers.json diff --git a/internal/route/middleware/test_utils.go b/internal/net/http/middleware/test_utils.go similarity index 97% rename from internal/route/middleware/test_utils.go rename to internal/net/http/middleware/test_utils.go index bb49e70..a1f89dd 100644 --- a/internal/route/middleware/test_utils.go +++ b/internal/net/http/middleware/test_utils.go @@ -10,7 +10,7 @@ import ( "net/url" E "github.com/yusing/go-proxy/internal/error" - gpHTTP "github.com/yusing/go-proxy/internal/http" + gpHTTP "github.com/yusing/go-proxy/internal/net/http" ) //go:embed test_data/sample_headers.json diff --git a/internal/net/http/middleware/x_forwarded.go b/internal/net/http/middleware/x_forwarded.go new file mode 100644 index 0000000..624996b --- /dev/null +++ b/internal/net/http/middleware/x_forwarded.go @@ -0,0 +1,44 @@ +package middleware + +import ( + "net" + "strings" +) + +var AddXForwarded = &Middleware{ + rewrite: func(req *Request) { + clientIP, _, err := net.SplitHostPort(req.RemoteAddr) + if err == nil { + req.Header.Set("X-Forwarded-For", clientIP) + } else { + req.Header.Del("X-Forwarded-For") + } + req.Header.Set("X-Forwarded-Host", req.Host) + if req.TLS == nil { + req.Header.Set("X-Forwarded-Proto", "http") + } else { + req.Header.Set("X-Forwarded-Proto", "https") + } + }, +} + +var SetXForwarded = &Middleware{ + rewrite: func(req *Request) { + clientIP, _, err := net.SplitHostPort(req.RemoteAddr) + if err == nil { + prior := req.Header["X-Forwarded-For"] + if len(prior) > 0 { + clientIP = strings.Join(prior, ", ") + ", " + clientIP + } + req.Header.Set("X-Forwarded-For", clientIP) + } else { + req.Header.Del("X-Forwarded-For") + } + req.Header.Set("X-Forwarded-Host", req.Host) + if req.TLS == nil { + req.Header.Set("X-Forwarded-Proto", "http") + } else { + req.Header.Set("X-Forwarded-Proto", "https") + } + }, +} diff --git a/internal/http/modify_response_writer.go b/internal/net/http/modify_response_writer.go similarity index 100% rename from internal/http/modify_response_writer.go rename to internal/net/http/modify_response_writer.go diff --git a/internal/http/reverse_proxy_mod.go b/internal/net/http/reverse_proxy_mod.go similarity index 94% rename from internal/http/reverse_proxy_mod.go rename to internal/net/http/reverse_proxy_mod.go index 57d018f..0570944 100644 --- a/internal/http/reverse_proxy_mod.go +++ b/internal/net/http/reverse_proxy_mod.go @@ -15,7 +15,6 @@ import ( "errors" "fmt" "io" - "net" "net/http" "net/http/httptrace" "net/textproto" @@ -58,39 +57,6 @@ type ProxyRequest struct { // r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"] // r.SetXForwarded() // } -func (r *ProxyRequest) SetXForwarded() { - clientIP, _, err := net.SplitHostPort(r.In.RemoteAddr) - if err == nil { - r.Out.Header.Set("X-Forwarded-For", clientIP) - } else { - r.Out.Header.Del("X-Forwarded-For") - } - r.Out.Header.Set("X-Forwarded-Host", r.In.Host) - if r.In.TLS == nil { - r.Out.Header.Set("X-Forwarded-Proto", "http") - } else { - r.Out.Header.Set("X-Forwarded-Proto", "https") - } -} - -func (r *ProxyRequest) AddXForwarded() { - clientIP, _, err := net.SplitHostPort(r.In.RemoteAddr) - if err == nil { - prior := r.Out.Header["X-Forwarded-For"] - if len(prior) > 0 { - clientIP = strings.Join(prior, ", ") + ", " + clientIP - } - r.Out.Header.Set("X-Forwarded-For", clientIP) - } else { - r.Out.Header.Del("X-Forwarded-For") - } - r.Out.Header.Set("X-Forwarded-Host", r.In.Host) - if r.In.TLS == nil { - r.Out.Header.Set("X-Forwarded-Proto", "http") - } else { - r.Out.Header.Set("X-Forwarded-Proto", "https") - } -} // ReverseProxy is an HTTP Handler that takes an incoming request and // sends it to another server, proxying the response back to the diff --git a/internal/http/status_code.go b/internal/net/http/status_code.go similarity index 100% rename from internal/http/status_code.go rename to internal/net/http/status_code.go diff --git a/internal/route/http.go b/internal/route/http.go index eecdec4..b24b0b0 100755 --- a/internal/route/http.go +++ b/internal/route/http.go @@ -13,10 +13,10 @@ import ( "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/docker/idlewatcher" E "github.com/yusing/go-proxy/internal/error" - . "github.com/yusing/go-proxy/internal/http" + . "github.com/yusing/go-proxy/internal/net/http" + "github.com/yusing/go-proxy/internal/net/http/middleware" P "github.com/yusing/go-proxy/internal/proxy" PT "github.com/yusing/go-proxy/internal/proxy/fields" - "github.com/yusing/go-proxy/internal/route/middleware" F "github.com/yusing/go-proxy/internal/utils/functional" ) diff --git a/internal/route/middleware/modify_request.go b/internal/route/middleware/modify_request.go deleted file mode 100644 index a722c7f..0000000 --- a/internal/route/middleware/modify_request.go +++ /dev/null @@ -1,58 +0,0 @@ -package middleware - -import ( - D "github.com/yusing/go-proxy/internal/docker" - E "github.com/yusing/go-proxy/internal/error" - U "github.com/yusing/go-proxy/internal/utils" -) - -type ( - modifyRequest struct { - *modifyRequestOpts - m *Middleware - } - // order: set_headers -> add_headers -> hide_headers - modifyRequestOpts struct { - SetHeaders map[string]string - AddHeaders map[string]string - HideHeaders []string - } -) - -var ModifyRequest = newModifyRequest() - -func newModifyRequest() (mr *modifyRequest) { - mr = new(modifyRequest) - mr.m = new(Middleware) - mr.m.labelParserMap = D.ValueParserMap{ - "set_headers": D.YamlLikeMappingParser(true), - "add_headers": D.YamlLikeMappingParser(true), - "hide_headers": D.YamlStringListParser, - } - mr.m.withOptions = func(optsRaw OptionsRaw, rp *ReverseProxy) (*Middleware, E.NestedError) { - mrWithOpts := new(modifyRequest) - mrWithOpts.m = &Middleware{ - impl: mrWithOpts, - rewrite: mrWithOpts.modifyRequest, - } - mrWithOpts.modifyRequestOpts = new(modifyRequestOpts) - err := U.Deserialize(optsRaw, mrWithOpts.modifyRequestOpts) - if err != nil { - return nil, E.FailWith("set options", err) - } - return mrWithOpts.m, nil - } - return -} - -func (mr *modifyRequest) modifyRequest(req *ProxyRequest) { - for k, v := range mr.SetHeaders { - req.Out.Header.Set(k, v) - } - for k, v := range mr.AddHeaders { - req.Out.Header.Add(k, v) - } - for _, k := range mr.HideHeaders { - req.Out.Header.Del(k) - } -} diff --git a/internal/route/middleware/x_forwarded.go b/internal/route/middleware/x_forwarded.go deleted file mode 100644 index 902fb8e..0000000 --- a/internal/route/middleware/x_forwarded.go +++ /dev/null @@ -1,9 +0,0 @@ -package middleware - -var AddXForwarded = &Middleware{ - rewrite: (*ProxyRequest).AddXForwarded, -} - -var SetXForwarded = &Middleware{ - rewrite: (*ProxyRequest).SetXForwarded, -} diff --git a/internal/utils/serialization.go b/internal/utils/serialization.go index b526752..9b54876 100644 --- a/internal/utils/serialization.go +++ b/internal/utils/serialization.go @@ -107,6 +107,9 @@ func Serialize(data any) (SerializedObject, E.NestedError) { } func Deserialize(src SerializedObject, target any) E.NestedError { + if src == nil || target == nil { + return nil + } // convert data fields to lower no-snake // convert target fields to lower no-snake // then check if the field of data is in the target