diff --git a/.env.example b/.env.example index 107435e..1fcd644 100644 --- a/.env.example +++ b/.env.example @@ -2,21 +2,24 @@ TZ=ETC/UTC # generate secret with `openssl rand -base64 32` -GOPROXY_API_JWT_SECRET= +GODOXY_API_JWT_SECRET= # the JWT token time-to-live -GOPROXY_API_JWT_TOKEN_TTL=1h +GODOXY_API_JWT_TOKEN_TTL=1h # API/WebUI login credentials -GOPROXY_API_USER=admin -GOPROXY_API_PASSWORD=password +GODOXY_API_USER=admin +GODOXY_API_PASSWORD=password # Proxy listening address -GOPROXY_HTTP_ADDR=:80 -GOPROXY_HTTPS_ADDR=:443 +GODOXY_HTTP_ADDR=:80 +GODOXY_HTTPS_ADDR=:443 # API listening address -GOPROXY_API_ADDR=127.0.0.1:8888 +GODOXY_API_ADDR=127.0.0.1:8888 + +# Prometheus Metrics listening address (uncomment to enable) +#GODOXY_PROMETHEUS_ADDR=:8889 # Debug mode -GOPROXY_DEBUG=false +GODOXY_DEBUG=false diff --git a/README.md b/README.md index a9b1f07..b89de49 100755 --- a/README.md +++ b/README.md @@ -79,13 +79,13 @@ Setup DNS Records point to machine which runs `GoDoxy`, e.g. - set random JWT secret ```shell - sed -i "s|GOPROXY_API_JWT_SECRET=.*|GOPROXY_API_JWT_SECRET=$(openssl rand -base64 32)|g" .env + sed -i "s|API_JWT_SECRET=.*|API_JWT_SECRET=$(openssl rand -base64 32)|g" .env ``` - change username and password for WebUI authentication ```shell - sed -i "s|GOPROXY_API_USERNAME=.*|GOPROXY_API_USERNAME=admin|g" .env - sed -i "s|GOPROXY_API_PASSWORD=.*|GOPROXY_API_PASSWORD=some-strong-password|g" .env + sed -i "s|API_USERNAME=.*|API_USERNAME=admin|g" .env + sed -i "s|API_PASSWORD=.*|API_PASSWORD=some-strong-password|g" .env ``` 4. _(Optional)_ setup `docker-socket-proxy` other docker nodes (see [Multi docker nodes setup](https://github.com/yusing/go-proxy/wiki/Configurations#multi-docker-nodes-setup)) then add them inside `config.yml` diff --git a/cmd/main.go b/cmd/main.go index f658e7c..2668ebd 100755 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,6 +16,7 @@ import ( "github.com/yusing/go-proxy/internal/config" E "github.com/yusing/go-proxy/internal/error" "github.com/yusing/go-proxy/internal/logging" + "github.com/yusing/go-proxy/internal/metrics" "github.com/yusing/go-proxy/internal/net/http/middleware" R "github.com/yusing/go-proxy/internal/route" "github.com/yusing/go-proxy/internal/server" @@ -126,7 +127,7 @@ func main() { logging.Info().Msg("autocert not configured") } - proxyServer := server.InitProxyServer(server.Options{ + server.StartServer(server.Options{ Name: "proxy", CertProvider: autocert, HTTPAddr: common.ProxyHTTPAddr, @@ -134,7 +135,7 @@ func main() { Handler: http.HandlerFunc(R.ProxyHandler), RedirectToHTTPS: config.Value().RedirectToHTTPS, }) - apiServer := server.InitAPIServer(server.Options{ + server.StartServer(server.Options{ Name: "api", CertProvider: autocert, HTTPAddr: common.APIHTTPAddr, @@ -142,8 +143,15 @@ func main() { RedirectToHTTPS: config.Value().RedirectToHTTPS, }) - proxyServer.Start() - apiServer.Start() + if common.PrometheusEnabled { + server.StartServer(server.Options{ + Name: "metrics", + CertProvider: autocert, + HTTPAddr: common.MetricsHTTPAddr, + Handler: metrics.NewHandler(), + RedirectToHTTPS: config.Value().RedirectToHTTPS, + }) + } // wait for signal <-sig diff --git a/go.mod b/go.mod index 2302e57..0e63e33 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/coder/websocket v1.8.12 github.com/docker/cli v27.3.1+incompatible github.com/docker/docker v27.3.1+incompatible - github.com/fsnotify/fsnotify v1.7.0 + github.com/fsnotify/fsnotify v1.8.0 github.com/go-acme/lego/v4 v4.19.2 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gotify/server/v2 v2.5.0 + github.com/prometheus/client_golang v1.20.5 github.com/puzpuzpuz/xsync/v3 v3.4.0 github.com/rs/zerolog v1.33.0 github.com/santhosh-tekuri/jsonschema v1.2.4 @@ -21,7 +22,9 @@ require ( require ( github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/cloudflare-go v0.108.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -34,17 +37,21 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/miekg/dns v1.1.62 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/ovh/go-ovh v1.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect @@ -59,6 +66,7 @@ require ( golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/tools v0.26.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gotest.tools/v3 v3.5.1 // indirect ) diff --git a/go.sum b/go.sum index 94cd447..edc720b 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/cloudflare-go v0.108.0 h1:C4Skfjd8I8X3uEOGmQUT4/iGyZcWdkIU7HwvMoLkEE0= github.com/cloudflare/cloudflare-go v0.108.0/go.mod h1:m492eNahT/9MsN7Ppnoge8AaI7QhVFtEgVm3I9HJFeU= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= @@ -11,7 +15,6 @@ github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3C github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -28,8 +31,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-acme/lego/v4 v4.19.2 h1:Y8hrmMvWETdqzzkRly7m98xtPJJivWFsgWi8fcvZo+Y= github.com/go-acme/lego/v4 v4.19.2/go.mod h1:wtDe3dDkmV4/oI2nydpNXSJpvV10J9RCyZ6MbYxNtlQ= github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= @@ -61,10 +64,14 @@ github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nu github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -81,21 +88,29 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI= github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4= github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -181,8 +196,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/api/v1/stats.go b/internal/api/v1/stats.go index d98ad7e..99ba304 100644 --- a/internal/api/v1/stats.go +++ b/internal/api/v1/stats.go @@ -10,7 +10,6 @@ import ( U "github.com/yusing/go-proxy/internal/api/v1/utils" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/config" - "github.com/yusing/go-proxy/internal/server" "github.com/yusing/go-proxy/internal/utils/strutils" ) @@ -33,7 +32,6 @@ func StatsWS(w http.ResponseWriter, r *http.Request) { } originPats = append(originPats, localAddresses...) } - U.LogInfo(r).Msgf("websocket API request from origins: %s", originPats) if common.IsDebug { originPats = []string{"*"} } @@ -62,9 +60,11 @@ func StatsWS(w http.ResponseWriter, r *http.Request) { } } +var startTime = time.Now() + func getStats() map[string]any { return map[string]any{ "proxies": config.Statistics(), - "uptime": strutils.FormatDuration(server.GetProxyServer().Uptime()), + "uptime": strutils.FormatDuration(time.Since(startTime)), } } diff --git a/internal/common/env.go b/internal/common/env.go index 9b615f6..93bcd35 100644 --- a/internal/common/env.go +++ b/internal/common/env.go @@ -19,6 +19,7 @@ var ( IsDebug = GetEnvBool("DEBUG", IsTest) IsDebugSkipAuth = GetEnvBool("DEBUG_SKIP_AUTH", false) IsTrace = GetEnvBool("TRACE", false) && IsDebug + IsProduction = !IsTest && !IsDebug ProxyHTTPAddr, ProxyHTTPHost, @@ -35,6 +36,12 @@ var ( APIHTTPPort, APIHTTPURL = GetAddrEnv("API_ADDR", "127.0.0.1:8888", "http") + MetricsHTTPAddr, + MetricsHTTPHost, + MetricsHTTPPort, + MetricsHTTPURL = GetAddrEnv("PROMETHEUS_ADDR", "", "http") + PrometheusEnabled = MetricsHTTPURL != "" + APIJWTSecret = decodeJWTKey(GetEnvString("API_JWT_SECRET", "")) APIJWTTokenTTL = GetDurationEnv("API_JWT_TOKEN_TTL", time.Hour) APIUser = GetEnvString("API_USER", "admin") @@ -79,6 +86,9 @@ func GetEnvBool(key string, defaultValue bool) bool { func GetAddrEnv(key, defaultValue, scheme string) (addr, host, port, fullURL string) { addr = GetEnvString(key, defaultValue) + if addr == "" { + return + } host, port, err := net.SplitHostPort(addr) if err != nil { log.Fatal().Msgf("env %s: invalid address: %s", key, addr) diff --git a/internal/config/query.go b/internal/config/query.go index 9f04c60..76877eb 100644 --- a/internal/config/query.go +++ b/internal/config/query.go @@ -9,7 +9,6 @@ import ( "github.com/yusing/go-proxy/internal/proxy/entry" "github.com/yusing/go-proxy/internal/route" proxy "github.com/yusing/go-proxy/internal/route/provider" - F "github.com/yusing/go-proxy/internal/utils/functional" "github.com/yusing/go-proxy/internal/utils/strutils" ) @@ -139,13 +138,12 @@ func Statistics() map[string]any { providerStats := make(map[string]proxy.ProviderStats) instance.providers.RangeAll(func(name string, p *proxy.Provider) { - providerStats[name] = p.Statistics() - }) + stats := p.Statistics() + providerStats[name] = stats - for _, stats := range providerStats { nTotalRPs += stats.NumRPs nTotalStreams += stats.NumStreams - } + }) return map[string]any{ "num_total_streams": nTotalStreams, @@ -153,14 +151,3 @@ func Statistics() map[string]any { "providers": providerStats, } } - -func FindRoute(alias string) *route.Route { - return F.MapFind(instance.providers, - func(p *proxy.Provider) (*route.Route, bool) { - if route, ok := p.GetRoute(alias); ok { - return route, true - } - return nil, false - }, - ) -} diff --git a/internal/docker/idlewatcher/types/waker.go b/internal/docker/idlewatcher/types/waker.go index 914e30e..0048851 100644 --- a/internal/docker/idlewatcher/types/waker.go +++ b/internal/docker/idlewatcher/types/waker.go @@ -11,4 +11,5 @@ type Waker interface { health.HealthMonitor http.Handler net.Stream + Wake() error } diff --git a/internal/docker/idlewatcher/watcher.go b/internal/docker/idlewatcher/watcher.go index 69593c2..5426d38 100644 --- a/internal/docker/idlewatcher/watcher.go +++ b/internal/docker/idlewatcher/watcher.go @@ -98,6 +98,10 @@ func registerWatcher(providerSubtask task.Task, entry entry.Entry, waker *waker) return w, nil } +func (w *Watcher) Wake() error { + return w.wakeIfStopped() +} + // WakeDebug logs a debug message related to waking the container. func (w *Watcher) WakeDebug() *zerolog.Event { return w.Debug().Str("action", "wake") diff --git a/internal/metrics/http_handler.go b/internal/metrics/http_handler.go new file mode 100644 index 0000000..31fb3cc --- /dev/null +++ b/internal/metrics/http_handler.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func NewHandler() http.Handler { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + return mux +} diff --git a/internal/metrics/http_metrics.go b/internal/metrics/http_metrics.go new file mode 100644 index 0000000..fde8ea1 --- /dev/null +++ b/internal/metrics/http_metrics.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/yusing/go-proxy/internal/common" +) + +type ( + RouteMetrics struct { + HTTPReqTotal, + HTTP2xx3xx, + HTTP4xx, + HTTP5xx *Counter + HTTPReqElapsed *Gauge + } + HTTPRouteMetricLabels struct { + Service, Method, Host, Visitor, Path string + } +) + +var rm RouteMetrics + +const ( + routerNamespace = "router" + routerHTTPSubsystem = "http" +) + +func GetRouteMetrics() *RouteMetrics { + return &rm +} + +func (lbl HTTPRouteMetricLabels) toPromLabels() prometheus.Labels { + return prometheus.Labels{ + "service": lbl.Service, + "method": lbl.Method, + "host": lbl.Host, + "visitor": lbl.Visitor, + "path": lbl.Path, + } +} + +func init() { + if !common.PrometheusEnabled { + return + } + lbls := []string{"service", "method", "host", "visitor", "path"} + partitionsHelp := ", partitioned by " + strings.Join(lbls, ", ") + rm = RouteMetrics{ + HTTPReqTotal: NewCounter(prometheus.CounterOpts{ + Namespace: routerNamespace, + Subsystem: routerHTTPSubsystem, + Name: "req_total", + Help: "How many requests processed" + partitionsHelp, + }), + HTTP2xx3xx: NewCounter(prometheus.CounterOpts{ + Namespace: routerNamespace, + Subsystem: routerHTTPSubsystem, + Name: "req_ok_count", + Help: "How many 2xx-3xx requests processed" + partitionsHelp, + }, lbls...), + HTTP4xx: NewCounter(prometheus.CounterOpts{ + Namespace: routerNamespace, + Subsystem: routerHTTPSubsystem, + Name: "req_4xx_count", + Help: "How many 4xx requests processed" + partitionsHelp, + }, lbls...), + HTTP5xx: NewCounter(prometheus.CounterOpts{ + Namespace: routerNamespace, + Subsystem: routerHTTPSubsystem, + Name: "req_5xx_count", + Help: "How many 5xx requests processed" + partitionsHelp, + }, lbls...), + HTTPReqElapsed: NewGauge(prometheus.GaugeOpts{ + Namespace: routerNamespace, + Subsystem: routerHTTPSubsystem, + Name: "req_elapsed_ms", + Help: "How long it took to process the request" + partitionsHelp, + }, lbls...), + } +} diff --git a/internal/metrics/metric.go b/internal/metrics/metric.go new file mode 100644 index 0000000..6106b0f --- /dev/null +++ b/internal/metrics/metric.go @@ -0,0 +1,73 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +type ( + Counter struct { + collector prometheus.Counter + mv *prometheus.CounterVec + } + Gauge struct { + collector prometheus.Gauge + mv *prometheus.GaugeVec + } + Labels interface { + toPromLabels() prometheus.Labels + } +) + +func NewCounter(opts prometheus.CounterOpts, labels ...string) *Counter { + m := &Counter{ + mv: prometheus.NewCounterVec(opts, labels), + } + if len(labels) == 0 { + m.collector = m.mv.WithLabelValues() + m.collector.Add(0) + } + prometheus.MustRegister(m) + return m +} + +func NewGauge(opts prometheus.GaugeOpts, labels ...string) *Gauge { + m := &Gauge{ + mv: prometheus.NewGaugeVec(opts, labels), + } + if len(labels) == 0 { + m.collector = m.mv.WithLabelValues() + m.collector.Set(0) + } + prometheus.MustRegister(m) + return m +} + +func (c *Counter) Collect(ch chan<- prometheus.Metric) { + c.mv.Collect(ch) +} + +func (c *Counter) Describe(ch chan<- *prometheus.Desc) { + c.mv.Describe(ch) +} + +func (c *Counter) Inc() { + c.collector.Inc() +} + +func (c *Counter) With(l Labels) prometheus.Counter { + return c.mv.With(l.toPromLabels()) +} + +func (g *Gauge) Collect(ch chan<- prometheus.Metric) { + g.mv.Collect(ch) +} + +func (g *Gauge) Describe(ch chan<- *prometheus.Desc) { + g.mv.Describe(ch) +} + +func (g *Gauge) Set(v float64) { + g.collector.Set(v) +} + +func (g *Gauge) With(l Labels) prometheus.Gauge { + return g.mv.With(l.toPromLabels()) +} diff --git a/internal/metrics/router_metrics.go b/internal/metrics/router_metrics.go new file mode 100644 index 0000000..9228e8d --- /dev/null +++ b/internal/metrics/router_metrics.go @@ -0,0 +1,20 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +func InitRouterMetrics(getRPsCount func() int, getStreamsCount func() int) { + prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "entrypoint", + Name: "num_reverse_proxies", + Help: "The number of reverse proxies", + }, func() float64 { + return float64(getRPsCount()) + })) + prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "entrypoint", + Name: "num_streams", + Help: "The number of streams", + }, func() float64 { + return float64(getStreamsCount()) + })) +} diff --git a/internal/net/http/dummy_response_writer.go b/internal/net/http/dummy_response_writer.go deleted file mode 100644 index 0e5a1a9..0000000 --- a/internal/net/http/dummy_response_writer.go +++ /dev/null @@ -1,15 +0,0 @@ -package http - -import "net/http" - -type DummyResponseWriter struct{} - -func (w DummyResponseWriter) Header() http.Header { - return make(http.Header) -} - -func (w DummyResponseWriter) Write([]byte) (_ int, _ error) { - return -} - -func (w DummyResponseWriter) WriteHeader(int) {} diff --git a/internal/net/http/loadbalancer/loadbalancer.go b/internal/net/http/loadbalancer/loadbalancer.go index edda918..e8f38fe 100644 --- a/internal/net/http/loadbalancer/loadbalancer.go +++ b/internal/net/http/loadbalancer/loadbalancer.go @@ -1,7 +1,6 @@ package loadbalancer import ( - "context" "net/http" "sync" "time" @@ -10,7 +9,6 @@ import ( "github.com/yusing/go-proxy/internal/common" idlewatcher "github.com/yusing/go-proxy/internal/docker/idlewatcher/types" E "github.com/yusing/go-proxy/internal/error" - gphttp "github.com/yusing/go-proxy/internal/net/http" "github.com/yusing/go-proxy/internal/net/http/middleware" "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/watcher/health" @@ -225,18 +223,15 @@ func (lb *LoadBalancer) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } if r.Header.Get(common.HeaderCheckRedirect) != "" { - ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second) - defer cancel() - // send dummy request to wake all servers - var dummyRW gphttp.DummyResponseWriter + // wake all servers for _, srv := range srvs { // wake only if server implements Waker - _, ok := srv.handler.(idlewatcher.Waker) - if !ok { - continue + waker, ok := srv.handler.(idlewatcher.Waker) + if ok { + if err := waker.Wake(); err != nil { + lb.Err(err).Msgf("failed to wake server %s", srv.Name) + } } - wakeReq := r.Clone(ctx) - srv.ServeHTTP(dummyRW, wakeReq) } } lb.impl.ServeHTTP(srvs, rw, r) diff --git a/internal/net/http/middleware/middleware.go b/internal/net/http/middleware/middleware.go index 04f9e8e..e6fbb73 100644 --- a/internal/net/http/middleware/middleware.go +++ b/internal/net/http/middleware/middleware.go @@ -157,8 +157,8 @@ func patchReverseProxy(rpName string, rp *ReverseProxy, middlewares []*Middlewar mid := BuildMiddlewareFromChain(rpName, middlewares) if mid.before != nil { - ori := rp.ServeHTTP - rp.ServeHTTP = func(w http.ResponseWriter, r *http.Request) { + ori := rp.HandlerFunc + rp.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { mid.before(ori, w, r) } } diff --git a/internal/net/http/reverse_proxy_mod.go b/internal/net/http/reverse_proxy_mod.go index f763be2..b56706c 100644 --- a/internal/net/http/reverse_proxy_mod.go +++ b/internal/net/http/reverse_proxy_mod.go @@ -10,6 +10,7 @@ package http // Copyright (c) 2024 yusing import ( + "bufio" "bytes" "context" "errors" @@ -22,8 +23,11 @@ import ( "net/url" "strings" "sync" + "time" "github.com/rs/zerolog" + "github.com/yusing/go-proxy/internal/common" + "github.com/yusing/go-proxy/internal/metrics" "github.com/yusing/go-proxy/internal/net/types" U "github.com/yusing/go-proxy/internal/utils" "golang.org/x/net/http/httpguts" @@ -86,12 +90,52 @@ type ReverseProxy struct { // implementation is used. ModifyResponse func(*http.Response) error - ServeHTTP http.HandlerFunc + HandlerFunc http.HandlerFunc TargetName string TargetURL types.URL } +type httpMetricLogger struct { + http.ResponseWriter + labels metrics.HTTPRouteMetricLabels +} + +// WriteHeader implements http.ResponseWriter. +func (l *httpMetricLogger) WriteHeader(status int) { + l.ResponseWriter.WriteHeader(status) + go func() { + m := metrics.GetRouteMetrics() + m.HTTPReqTotal.Inc() + + // ignore 1xx + switch { + case status >= 500: + m.HTTP5xx.With(l.labels).Inc() + case status >= 400: + m.HTTP4xx.With(l.labels).Inc() + case status >= 200: + m.HTTP2xx3xx.With(l.labels).Inc() + } + }() +} + +// Hijack hijacks the connection. +func (l *httpMetricLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if h, ok := l.ResponseWriter.(http.Hijacker); ok { + return h.Hijack() + } + + return nil, nil, fmt.Errorf("not a hijacker: %T", l.ResponseWriter) +} + +// Flush sends any buffered data to the client. +func (l *httpMetricLogger) Flush() { + if flusher, ok := l.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + func singleJoiningSlash(a, b string) string { aslash := strings.HasSuffix(a, "/") bslash := strings.HasPrefix(b, "/") @@ -157,7 +201,7 @@ func NewReverseProxy(name string, target types.URL, transport http.RoundTripper) TargetName: name, TargetURL: target, } - rp.ServeHTTP = rp.serveHTTP + rp.HandlerFunc = rp.handler return rp } @@ -225,9 +269,32 @@ func (p *ReverseProxy) modifyResponse(rw http.ResponseWriter, res *http.Response return true } -func (p *ReverseProxy) serveHTTP(rw http.ResponseWriter, req *http.Request) { - if _, ok := rw.(DummyResponseWriter); ok { - return +func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + p.HandlerFunc(rw, req) +} + +func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) { + if common.PrometheusEnabled { + t := time.Now() + visitor, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + visitor = req.RemoteAddr + } + lbls := metrics.HTTPRouteMetricLabels{ + Service: p.TargetName, + Method: req.Method, + Host: req.Host, + Visitor: visitor, + Path: req.URL.Path, + } + rw = &httpMetricLogger{ + ResponseWriter: rw, + labels: lbls, + } + defer func() { + duration := time.Since(t) + metrics.GetRouteMetrics().HTTPReqElapsed.With(lbls).Set(float64(duration.Milliseconds())) + }() } transport := p.Transport diff --git a/internal/route/http.go b/internal/route/http.go index 06dad14..2966c0a 100755 --- a/internal/route/http.go +++ b/internal/route/http.go @@ -38,10 +38,6 @@ type ( } SubdomainKey = PT.Alias - - ReverseProxyHandler struct { - *gphttp.ReverseProxy - } ) var ( @@ -52,10 +48,6 @@ var ( // globalMux = http.NewServeMux() // TODO: support regex subdomain matching. ) -func (rp ReverseProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - rp.ReverseProxy.ServeHTTP(w, r) -} - func GetReverseProxies() F.Map[string, *HTTPRoute] { return httpRoutes } @@ -77,10 +69,11 @@ func NewHTTPRoute(entry *entry.ReverseProxyEntry) (impl, E.Error) { trans = gphttp.DefaultTransport.Clone() } - rp := gphttp.NewReverseProxy(string(entry.Alias), entry.URL, trans) + service := string(entry.Alias) + rp := gphttp.NewReverseProxy(service, entry.URL, trans) if len(entry.Middlewares) > 0 { - err := middleware.PatchReverseProxy(string(entry.Alias), rp, entry.Middlewares) + err := middleware.PatchReverseProxy(service, rp, entry.Middlewares) if err != nil { return nil, err } @@ -136,11 +129,11 @@ func (r *HTTPRoute) Start(providerSubtask task.Task) E.Error { if r.handler == nil { switch { case len(r.PathPatterns) == 1 && r.PathPatterns[0] == "/": - r.handler = ReverseProxyHandler{r.rp} + r.handler = r.rp default: mux := http.NewServeMux() for _, p := range r.PathPatterns { - mux.HandleFunc(string(p), r.rp.ServeHTTP) + mux.HandleFunc(string(p), r.rp.HandlerFunc) } r.handler = mux } diff --git a/internal/route/metrics.go b/internal/route/metrics.go new file mode 100644 index 0000000..93ac630 --- /dev/null +++ b/internal/route/metrics.go @@ -0,0 +1,7 @@ +package route + +import "github.com/yusing/go-proxy/internal/metrics" + +func init() { + metrics.InitRouterMetrics(httpRoutes.Size, streamRoutes.Size) +} diff --git a/internal/server/instance.go b/internal/server/instance.go deleted file mode 100644 index 768f5b3..0000000 --- a/internal/server/instance.go +++ /dev/null @@ -1,25 +0,0 @@ -package server - -var proxyServer, apiServer *Server - -func InitProxyServer(opt Options) *Server { - if proxyServer == nil { - proxyServer = NewServer(opt) - } - return proxyServer -} - -func InitAPIServer(opt Options) *Server { - if apiServer == nil { - apiServer = NewServer(opt) - } - return apiServer -} - -func GetProxyServer() *Server { - return proxyServer -} - -func GetAPIServer() *Server { - return apiServer -} diff --git a/internal/server/server.go b/internal/server/server.go index a6caae4..cd563c8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -38,6 +38,12 @@ type Options struct { Handler http.Handler } +func StartServer(opt Options) (s *Server) { + s = NewServer(opt) + s.Start() + return s +} + func NewServer(opt Options) (s *Server) { var httpSer, httpsSer *http.Server var httpHandler http.Handler diff --git a/internal/watcher/events/event_queue.go b/internal/watcher/events/event_queue.go index d2926f6..f899d12 100644 --- a/internal/watcher/events/event_queue.go +++ b/internal/watcher/events/event_queue.go @@ -50,8 +50,25 @@ func NewEventQueue(parent task.Task, flushInterval time.Duration, onFlush OnFlus } func (e *EventQueue) Start(eventCh <-chan Event, errCh <-chan E.Error) { + if common.IsProduction { + origOnFlush := e.onFlush + // recover panic in onFlush when in production mode + e.onFlush = func(flushTask task.Task, events []Event) { + defer func() { + if err := recover(); err != nil { + e.onError(E.New("recovered panic in onFlush"). + Withf("%v", err). + Subject(e.task.Parent().String())) + } + }() + origOnFlush(flushTask, events) + } + } + go func() { defer e.ticker.Stop() + defer e.task.Finish(nil) + for { select { case <-e.task.Context().Done(): @@ -61,18 +78,7 @@ func (e *EventQueue) Start(eventCh <-chan Event, errCh <-chan E.Error) { flushTask := e.task.Subtask("flush events") queue := e.queue e.queue = make([]Event, 0, eventQueueCapacity) - if !common.IsDebug { - go func() { - defer func() { - if err := recover(); err != nil { - e.onError(E.Errorf("recovered panic in onFlush: %v", err).Subject(e.task.Parent().String())) - } - }() - e.onFlush(flushTask, queue) - }() - } else { - go e.onFlush(flushTask, queue) - } + go e.onFlush(flushTask, queue) flushTask.Wait() } e.ticker.Reset(e.flushInterval)