From 80bc018a7fb4d69404b32ba7ef312bba5d7e1a2d Mon Sep 17 00:00:00 2001 From: Yuzerion Date: Wed, 16 Apr 2025 15:02:11 +0800 Subject: [PATCH] feat: custom json marshaling implementation, replace json and yaml library (#89) * chore: replace gopkg.in/yaml.v3 vs goccy/go-yaml; replace encoding/json with bytedance/sonic * fix: yaml unmarshal panic * feat: custom json marshaler implementation * chore: fix import and err marshal handling --------- Co-authored-by: yusing --- agent/pkg/handler/check_health_test.go | 3 +- go.mod | 9 +- go.sum | 24 + internal/api/v1/auth/oidc.go | 3 +- internal/api/v1/auth/oidc_test.go | 3 +- internal/api/v1/auth/userpass.go | 3 +- internal/api/v1/auth/userpass_test.go | 3 +- internal/api/v1/certapi/cert_info.go | 3 +- internal/api/v1/debug/handler.go | 2 +- internal/api/v1/dockerapi/info.go | 9 +- internal/api/v1/dockerapi/utils.go | 3 +- internal/api/v1/homepage_overrides.go | 2 +- internal/api/v1/new_agent.go | 3 +- internal/api/v1/query/query.go | 3 +- internal/autocert/provider_test/ovh_test.go | 4 +- internal/config/config_test.go | 2 +- internal/gperr/base.go | 6 +- internal/gperr/subject.go | 7 +- internal/gperr/utils.go | 4 +- internal/homepage/homepage.go | 9 +- internal/homepage/icon_cache.go | 3 +- internal/homepage/list-icons.go | 3 +- internal/metrics/period/entries.go | 9 +- internal/metrics/period/poller.go | 2 +- internal/metrics/period/tests.go | 71 +++ internal/metrics/systeminfo/system_info.go | 64 +-- .../metrics/systeminfo/system_info_test.go | 33 +- internal/metrics/uptime/status.go | 22 + internal/metrics/uptime/uptime.go | 18 +- internal/metrics/uptime/uptime_test.go | 10 + .../net/gphttp/accesslog/access_logger.go | 2 +- .../gphttp/accesslog/access_logger_test.go | 3 +- internal/net/gphttp/accesslog/formatter.go | 3 +- internal/net/gphttp/body.go | 3 +- internal/net/gphttp/error.go | 3 +- internal/net/gphttp/middleware/middleware.go | 9 +- .../gphttp/middleware/middleware_builder.go | 2 +- .../middleware/middleware_builder_test.go | 5 +- internal/net/gphttp/middleware/test_utils.go | 2 +- internal/net/types/stream.go | 2 +- internal/notif/format.go | 3 +- internal/notif/gotify.go | 3 +- internal/notif/webhook.go | 17 +- internal/route/provider/docker.go | 2 +- internal/route/provider/docker_labels_test.go | 2 +- internal/route/routes/routequery/query.go | 4 +- internal/route/rules/do.go | 4 - internal/route/rules/on.go | 4 - internal/route/rules/rules.go | 11 +- internal/utils/atomic/value.go | 7 +- internal/utils/functional/map.go | 2 +- internal/utils/serialization.go | 29 +- internal/utils/serialization_test.go | 2 +- .../watcher/health/monitor/agent_proxied.go | 3 +- internal/watcher/health/status.go | 8 +- pkg/json/check_empty.go | 55 ++ pkg/json/encoder.go | 17 + pkg/json/json.go | 70 +++ pkg/json/map.go | 24 + pkg/json/map_slice.go | 18 + pkg/json/marshal.go | 269 +++++++++ pkg/json/marshal_test.go | 529 ++++++++++++++++++ pkg/json/special.go | 60 ++ pkg/json/string.go | 334 +++++++++++ pkg/json/struct.go | 103 ++++ 65 files changed, 1749 insertions(+), 205 deletions(-) create mode 100644 internal/metrics/period/tests.go create mode 100644 internal/metrics/uptime/status.go create mode 100644 internal/metrics/uptime/uptime_test.go create mode 100644 pkg/json/check_empty.go create mode 100644 pkg/json/encoder.go create mode 100644 pkg/json/json.go create mode 100644 pkg/json/map.go create mode 100644 pkg/json/map_slice.go create mode 100644 pkg/json/marshal.go create mode 100644 pkg/json/marshal_test.go create mode 100644 pkg/json/special.go create mode 100644 pkg/json/string.go create mode 100644 pkg/json/struct.go diff --git a/agent/pkg/handler/check_health_test.go b/agent/pkg/handler/check_health_test.go index 4633ed6..d90d98e 100644 --- a/agent/pkg/handler/check_health_test.go +++ b/agent/pkg/handler/check_health_test.go @@ -1,7 +1,6 @@ package handler_test import ( - "encoding/json" "net" "net/http" "net/http/httptest" @@ -9,6 +8,8 @@ import ( "strconv" "testing" + "github.com/yusing/go-proxy/pkg/json" + "github.com/stretchr/testify/require" "github.com/yusing/go-proxy/agent/pkg/agent" "github.com/yusing/go-proxy/agent/pkg/handler" diff --git a/go.mod b/go.mod index 77bdc37..e68cf9a 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,17 @@ go 1.24.2 // misc require ( + github.com/bytedance/sonic v1.13.2 // faster json unmarshal (for marshal it's using custom implementation) github.com/fsnotify/fsnotify v1.9.0 // file watcher github.com/go-acme/lego/v4 v4.22.2 // acme client github.com/go-playground/validator/v10 v10.26.0 // validator github.com/gobwas/glob v0.2.3 // glob matcher for route rules + github.com/goccy/go-yaml v1.17.1 // yaml parsing for different config files github.com/gotify/server/v2 v2.6.1 // reference the Message struct for json response github.com/lithammer/fuzzysearch v1.1.8 // fuzzy search for searching icons and filtering metrics github.com/puzpuzpuz/xsync/v3 v3.5.1 // lock free map for concurrent operations golang.org/x/text v0.24.0 // string utilities golang.org/x/time v0.11.0 // time utilities - gopkg.in/yaml.v3 v3.0.1 // yaml parsing for different config files ) // http @@ -70,9 +71,11 @@ require ( github.com/andybalholm/cascadia v1.3.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/goterm v1.0.4 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/cloudflare-go v0.115.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/diskfs/go-diskfs v1.6.0 // indirect @@ -95,6 +98,7 @@ require ( github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/jinzhu/copier v0.4.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/magefile/mage v1.15.0 // indirect @@ -122,6 +126,7 @@ require ( github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect @@ -131,6 +136,7 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/mock v0.5.1 // indirect + golang.org/x/arch v0.16.0 // indirect golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/sync v0.13.0 // indirect @@ -138,5 +144,6 @@ require ( golang.org/x/tools v0.32.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect ) diff --git a/go.sum b/go.sum index 3bee87a..ec5111d 100644 --- a/go.sum +++ b/go.sum @@ -10,12 +10,20 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/cloudflare-go v0.115.0 h1:84/dxeeXweCc0PN5Cto44iTA8AkG1fyT11yPO5ZB7sM= github.com/cloudflare/cloudflare-go v0.115.0/go.mod h1:Ds6urDwn/TF2uIU24mu7H91xkKP8gSAHxQ44DSZgVmU= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -79,6 +87,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -112,6 +122,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -202,13 +216,20 @@ github.com/shirou/gopsutil/v4 v4.25.3/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5Bdj github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= @@ -242,6 +263,8 @@ go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/mock v0.5.1 h1:ASgazW/qBmR+A32MYFDB6E2POoTgOwT509VP0CT/fjs= go.uber.org/mock v0.5.1/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U= +golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -368,3 +391,4 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/internal/api/v1/auth/oidc.go b/internal/api/v1/auth/oidc.go index 2643a99..95ae628 100644 --- a/internal/api/v1/auth/oidc.go +++ b/internal/api/v1/auth/oidc.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rand" "encoding/base64" - "encoding/json" "errors" "fmt" "io" @@ -15,6 +14,8 @@ import ( "strings" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/coreos/go-oidc/v3/oidc" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/net/gphttp" diff --git a/internal/api/v1/auth/oidc_test.go b/internal/api/v1/auth/oidc_test.go index 0ed759f..ad2a5e2 100644 --- a/internal/api/v1/auth/oidc_test.go +++ b/internal/api/v1/auth/oidc_test.go @@ -5,12 +5,13 @@ import ( "crypto/rand" "crypto/rsa" "encoding/base64" - "encoding/json" "net/http" "net/http/httptest" "testing" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/coreos/go-oidc/v3/oidc" "github.com/golang-jwt/jwt/v5" "github.com/yusing/go-proxy/internal/common" diff --git a/internal/api/v1/auth/userpass.go b/internal/api/v1/auth/userpass.go index 239a4cc..39dae3f 100644 --- a/internal/api/v1/auth/userpass.go +++ b/internal/api/v1/auth/userpass.go @@ -1,11 +1,12 @@ package auth import ( - "encoding/json" "fmt" "net/http" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/golang-jwt/jwt/v5" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/gperr" diff --git a/internal/api/v1/auth/userpass_test.go b/internal/api/v1/auth/userpass_test.go index 9a9fbc4..cd958ed 100644 --- a/internal/api/v1/auth/userpass_test.go +++ b/internal/api/v1/auth/userpass_test.go @@ -2,13 +2,14 @@ package auth import ( "bytes" - "encoding/json" "io" "net/http" "net/http/httptest" "testing" "time" + "github.com/yusing/go-proxy/pkg/json" + . "github.com/yusing/go-proxy/internal/utils/testing" "golang.org/x/crypto/bcrypt" ) diff --git a/internal/api/v1/certapi/cert_info.go b/internal/api/v1/certapi/cert_info.go index 07edfd9..c99586c 100644 --- a/internal/api/v1/certapi/cert_info.go +++ b/internal/api/v1/certapi/cert_info.go @@ -1,9 +1,10 @@ package certapi import ( - "encoding/json" "net/http" + "github.com/yusing/go-proxy/pkg/json" + config "github.com/yusing/go-proxy/internal/config/types" ) diff --git a/internal/api/v1/debug/handler.go b/internal/api/v1/debug/handler.go index 271ab09..c128b9d 100644 --- a/internal/api/v1/debug/handler.go +++ b/internal/api/v1/debug/handler.go @@ -50,7 +50,7 @@ func jsonHandler[T debuggable](getData iter.Seq2[string, T]) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { gpwebsocket.DynamicJSONHandler(w, r, func() []map[string]any { return toSortedSlice(getData) - }, 1*time.Second) + }, 200*time.Millisecond) } } diff --git a/internal/api/v1/dockerapi/info.go b/internal/api/v1/dockerapi/info.go index a8bd08c..6aeb4de 100644 --- a/internal/api/v1/dockerapi/info.go +++ b/internal/api/v1/dockerapi/info.go @@ -2,10 +2,11 @@ package dockerapi import ( "context" - "encoding/json" "net/http" "sort" + "github.com/yusing/go-proxy/pkg/json" + dockerSystem "github.com/docker/docker/api/types/system" "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/utils/strutils" @@ -13,8 +14,8 @@ import ( type dockerInfo dockerSystem.Info -func (d *dockerInfo) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]any{ +func (d *dockerInfo) MarshalJSONTo(buf []byte) []byte { + return json.MarshalTo(map[string]any{ "name": d.Name, "version": d.ServerVersion, "containers": map[string]int{ @@ -26,7 +27,7 @@ func (d *dockerInfo) MarshalJSON() ([]byte, error) { "images": d.Images, "n_cpu": d.NCPU, "memory": strutils.FormatByteSize(d.MemTotal), - }) + }, buf) } func DockerInfo(w http.ResponseWriter, r *http.Request) { diff --git a/internal/api/v1/dockerapi/utils.go b/internal/api/v1/dockerapi/utils.go index b25b59a..51f23cf 100644 --- a/internal/api/v1/dockerapi/utils.go +++ b/internal/api/v1/dockerapi/utils.go @@ -2,10 +2,11 @@ package dockerapi import ( "context" - "encoding/json" "net/http" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "github.com/yusing/go-proxy/agent/pkg/agent" diff --git a/internal/api/v1/homepage_overrides.go b/internal/api/v1/homepage_overrides.go index 6c3d303..3cd46af 100644 --- a/internal/api/v1/homepage_overrides.go +++ b/internal/api/v1/homepage_overrides.go @@ -1,12 +1,12 @@ package v1 import ( - "encoding/json" "io" "net/http" "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/net/gphttp" + "github.com/yusing/go-proxy/pkg/json" ) const ( diff --git a/internal/api/v1/new_agent.go b/internal/api/v1/new_agent.go index 9abc627..6b6daa5 100644 --- a/internal/api/v1/new_agent.go +++ b/internal/api/v1/new_agent.go @@ -1,13 +1,14 @@ package v1 import ( - "encoding/json" "fmt" "io" "net/http" "os" "strconv" + "github.com/yusing/go-proxy/pkg/json" + _ "embed" "github.com/yusing/go-proxy/agent/pkg/agent" diff --git a/internal/api/v1/query/query.go b/internal/api/v1/query/query.go index 4463834..df59adc 100644 --- a/internal/api/v1/query/query.go +++ b/internal/api/v1/query/query.go @@ -1,11 +1,12 @@ package query import ( - "encoding/json" "fmt" "io" "net/http" + "github.com/yusing/go-proxy/pkg/json" + v1 "github.com/yusing/go-proxy/internal/api/v1" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/gperr" diff --git a/internal/autocert/provider_test/ovh_test.go b/internal/autocert/provider_test/ovh_test.go index 4d65212..b59ee29 100644 --- a/internal/autocert/provider_test/ovh_test.go +++ b/internal/autocert/provider_test/ovh_test.go @@ -4,9 +4,9 @@ import ( "testing" "github.com/go-acme/lego/v4/providers/dns/ovh" + "github.com/goccy/go-yaml" "github.com/yusing/go-proxy/internal/utils" . "github.com/yusing/go-proxy/internal/utils/testing" - "gopkg.in/yaml.v3" ) // type Config struct { @@ -44,7 +44,7 @@ oauth2_config: } testYaml = testYaml[1:] // remove first \n opt := make(map[string]any) - ExpectNoError(t, yaml.Unmarshal([]byte(testYaml), opt)) + ExpectNoError(t, yaml.Unmarshal([]byte(testYaml), &opt)) ExpectNoError(t, utils.MapUnmarshalValidate(opt, cfg)) ExpectEqual(t, cfg, cfgExpected) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7070ec5..75f040f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,6 +5,7 @@ import ( "path" "testing" + "github.com/goccy/go-yaml" "github.com/stretchr/testify/assert" "github.com/yusing/go-proxy/agent/pkg/agent" "github.com/yusing/go-proxy/internal/common" @@ -12,7 +13,6 @@ import ( "github.com/yusing/go-proxy/internal/route/provider" "github.com/yusing/go-proxy/internal/utils" . "github.com/yusing/go-proxy/internal/utils/testing" - "gopkg.in/yaml.v3" ) func TestFileProviderValidate(t *testing.T) { diff --git a/internal/gperr/base.go b/internal/gperr/base.go index bc30107..65c5860 100644 --- a/internal/gperr/base.go +++ b/internal/gperr/base.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "encoding/json" + "github.com/yusing/go-proxy/pkg/json" ) // baseError is an immutable wrapper around an error. @@ -49,6 +49,6 @@ func (err *baseError) Error() string { return err.Err.Error() } -func (err *baseError) MarshalJSON() ([]byte, error) { - return json.Marshal(err.Err) +func (err *baseError) MarshalJSONTo(buf []byte) []byte { + return json.MarshalTo(err.Err, buf) } diff --git a/internal/gperr/subject.go b/internal/gperr/subject.go index 4439951..6167cca 100644 --- a/internal/gperr/subject.go +++ b/internal/gperr/subject.go @@ -5,8 +5,7 @@ import ( "slices" "strings" - "encoding/json" - + "github.com/yusing/go-proxy/pkg/json" "github.com/yusing/go-proxy/internal/utils/strutils/ansi" ) @@ -94,7 +93,7 @@ func (err *withSubject) Error() string { return sb.String() } -func (err *withSubject) MarshalJSON() ([]byte, error) { +func (err *withSubject) MarshalJSONTo(buf []byte) []byte { subjects := slices.Clone(err.Subjects) slices.Reverse(subjects) @@ -102,5 +101,5 @@ func (err *withSubject) MarshalJSON() ([]byte, error) { "subjects": subjects, "err": err.Err, } - return json.Marshal(reversed) + return json.MarshalTo(reversed, buf) } diff --git a/internal/gperr/utils.go b/internal/gperr/utils.go index b26b683..f64559c 100644 --- a/internal/gperr/utils.go +++ b/internal/gperr/utils.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "encoding/json" + stdJSON "encoding/json" ) func newError(message string) error { @@ -73,7 +73,7 @@ func IsJSONMarshallable(err error) bool { case *baseError: return IsJSONMarshallable(err.Err) default: - var v json.Marshaler + var v stdJSON.Marshaler return errors.As(err, &v) } } diff --git a/internal/homepage/homepage.go b/internal/homepage/homepage.go index 9ad1d81..77dc620 100644 --- a/internal/homepage/homepage.go +++ b/internal/homepage/homepage.go @@ -1,9 +1,10 @@ package homepage import ( - "encoding/json" "strings" + "github.com/yusing/go-proxy/pkg/json" + config "github.com/yusing/go-proxy/internal/config/types" "github.com/yusing/go-proxy/internal/utils" ) @@ -42,7 +43,7 @@ func (cfg *ItemConfig) GetOverride(alias string) *ItemConfig { return overrideConfigInstance.GetOverride(alias, cfg) } -func (item *Item) MarshalJSON() ([]byte, error) { +func (item *Item) MarshalJSONTo(buf []byte) []byte { var url *string if !strings.ContainsRune(item.Alias, '.') { godoxyCfg := config.GetInstance().Value() @@ -55,7 +56,7 @@ func (item *Item) MarshalJSON() ([]byte, error) { } else { url = &item.Alias } - return json.Marshal(map[string]any{ + return json.MarshalTo(map[string]any{ "show": item.Show, "alias": item.Alias, "provider": item.Provider, @@ -66,7 +67,7 @@ func (item *Item) MarshalJSON() ([]byte, error) { "description": item.Description, "sort_order": item.SortOrder, "widget_config": item.WidgetConfig, - }) + }, buf) } func (c Homepage) Add(item *Item) { diff --git a/internal/homepage/icon_cache.go b/internal/homepage/icon_cache.go index b7f68ea..ac35db3 100644 --- a/internal/homepage/icon_cache.go +++ b/internal/homepage/icon_cache.go @@ -1,10 +1,11 @@ package homepage import ( - "encoding/json" "sync" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/task" diff --git a/internal/homepage/list-icons.go b/internal/homepage/list-icons.go index a76a87e..9979fe9 100644 --- a/internal/homepage/list-icons.go +++ b/internal/homepage/list-icons.go @@ -1,12 +1,13 @@ package homepage import ( - "encoding/json" "io" "net/http" "sync" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/lithammer/fuzzysearch/fuzzy" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/logging" diff --git a/internal/metrics/period/entries.go b/internal/metrics/period/entries.go index d9a5554..db61431 100644 --- a/internal/metrics/period/entries.go +++ b/internal/metrics/period/entries.go @@ -1,8 +1,9 @@ package period import ( - "encoding/json" "time" + + "github.com/yusing/go-proxy/pkg/json" ) type Entries[T any] struct { @@ -48,11 +49,11 @@ func (e *Entries[T]) Get() []*T { return res } -func (e *Entries[T]) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]any{ +func (e *Entries[T]) MarshalJSONTo(buf []byte) []byte { + return json.MarshalTo(map[string]any{ "entries": e.Get(), "interval": e.interval, - }) + }, buf) } func (e *Entries[T]) UnmarshalJSON(data []byte) error { diff --git a/internal/metrics/period/poller.go b/internal/metrics/period/poller.go index f60014d..0e36e0d 100644 --- a/internal/metrics/period/poller.go +++ b/internal/metrics/period/poller.go @@ -2,7 +2,6 @@ package period import ( "context" - "encoding/json" "fmt" "net/url" "os" @@ -15,6 +14,7 @@ import ( "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/utils/atomic" + "github.com/yusing/go-proxy/pkg/json" ) type ( diff --git a/internal/metrics/period/tests.go b/internal/metrics/period/tests.go new file mode 100644 index 0000000..94abac4 --- /dev/null +++ b/internal/metrics/period/tests.go @@ -0,0 +1,71 @@ +package period + +import ( + "context" + "net/http" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yusing/go-proxy/pkg/json" +) + +func (p *Poller[T, AggregateT]) Test(t *testing.T, query url.Values) { + t.Helper() + for range 3 { + require.NoError(t, p.testPoll()) + } + t.Run("periods", func(t *testing.T) { + assert.NoError(t, p.testMarshalPeriods(query)) + }) + t.Run("no period", func(t *testing.T) { + assert.NoError(t, p.testMarshalNoPeriod()) + }) +} + +func (p *Poller[T, AggregateT]) testPeriod(period string, query url.Values) (any, error) { + query.Set("period", period) + return p.getRespData(&http.Request{URL: &url.URL{RawQuery: query.Encode()}}) +} + +func (p *Poller[T, AggregateT]) testPoll() error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + data, err := p.poll(ctx, p.lastResult.Load()) + if err != nil { + return err + } + for _, period := range p.period.Entries { + period.Add(time.Now(), data) + } + p.lastResult.Store(data) + return nil +} + +func (p *Poller[T, AggregateT]) testMarshalPeriods(query url.Values) error { + for period := range p.period.Entries { + data, err := p.testPeriod(string(period), query) + if err != nil { + return err + } + _, err = json.Marshal(data) + if err != nil { + return err + } + } + return nil +} + +func (p *Poller[T, AggregateT]) testMarshalNoPeriod() error { + data, err := p.getRespData(&http.Request{URL: &url.URL{}}) + if err != nil { + return err + } + _, err = json.Marshal(data) + if err != nil { + return err + } + return nil +} diff --git a/internal/metrics/systeminfo/system_info.go b/internal/metrics/systeminfo/system_info.go index abe37b9..1d44c50 100644 --- a/internal/metrics/systeminfo/system_info.go +++ b/internal/metrics/systeminfo/system_info.go @@ -3,7 +3,6 @@ package systeminfo import ( "bytes" "context" - "encoding/json" "errors" "fmt" "net/url" @@ -20,6 +19,7 @@ import ( "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/metrics/period" + "github.com/yusing/go-proxy/pkg/json" ) // json tags are left for tests @@ -55,7 +55,7 @@ type ( DownloadSpeed float64 `json:"download_speed"` } Sensors []sensors.TemperatureStat - Aggregated []map[string]any + Aggregated = json.MapSlice[any] ) type SystemInfo struct { @@ -295,8 +295,8 @@ func (s *SystemInfo) collectSensorsInfo(ctx context.Context) error { } // explicitly implement MarshalJSON to avoid reflection -func (s *SystemInfo) MarshalJSON() ([]byte, error) { - b := bytes.NewBuffer(make([]byte, 0, 1024)) +func (s *SystemInfo) MarshalJSONTo(buf []byte) []byte { + b := bytes.NewBuffer(buf) b.WriteRune('{') @@ -315,7 +315,7 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { // memory b.WriteString(`,"memory":`) if s.Memory != nil { - b.WriteString(fmt.Sprintf( + b.Write(fmt.Appendf(nil, `{"total":%d,"available":%d,"used":%d,"used_percent":%s}`, s.Memory.Total, s.Memory.Available, @@ -329,13 +329,13 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { // disk b.WriteString(`,"disks":`) if len(s.Disks) > 0 { - b.WriteString("{") + b.WriteRune('{') first := true for device, disk := range s.Disks { if !first { b.WriteRune(',') } - b.WriteString(fmt.Sprintf( + b.Write(fmt.Appendf(nil, `"%s":{"device":%q,"path":%q,"fstype":%q,"total":%d,"free":%d,"used":%d,"used_percent":%s}`, device, device, @@ -362,7 +362,7 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { if !first { b.WriteRune(',') } - b.WriteString(fmt.Sprintf( + b.Write(fmt.Appendf(nil, `"%s":{"name":%q,"read_bytes":%d,"write_bytes":%d,"read_speed":%s,"write_speed":%s,"iops":%d}`, name, name, @@ -382,7 +382,7 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { // network b.WriteString(`,"network":`) if s.Network != nil { - b.WriteString(fmt.Sprintf( + b.Write(fmt.Appendf(nil, `{"bytes_sent":%d,"bytes_recv":%d,"upload_speed":%s,"download_speed":%s}`, s.Network.BytesSent, s.Network.BytesRecv, @@ -396,13 +396,13 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { // sensors b.WriteString(`,"sensors":`) if len(s.Sensors) > 0 { - b.WriteString("{") + b.WriteRune('{') first := true for _, sensor := range s.Sensors { if !first { b.WriteRune(',') } - b.WriteString(fmt.Sprintf( + b.Write(fmt.Appendf(nil, `%q:{"name":%q,"temperature":%s,"high":%s,"critical":%s}`, sensor.SensorKey, sensor.SensorKey, @@ -418,7 +418,7 @@ func (s *SystemInfo) MarshalJSON() ([]byte, error) { } b.WriteRune('}') - return []byte(b.String()), nil + return b.Bytes() } func (s *Sensors) UnmarshalJSON(data []byte) error { @@ -560,43 +560,3 @@ func aggregate(entries []*SystemInfo, query url.Values) (total int, result Aggre } return len(aggregated), aggregated } - -func (result Aggregated) MarshalJSON() ([]byte, error) { - buf := bytes.NewBuffer(make([]byte, 0, 1024)) - - buf.WriteByte('[') - i := 0 - n := len(result) - for _, entry := range result { - buf.WriteRune('{') - j := 0 - m := len(entry) - for k, v := range entry { - buf.WriteByte('"') - buf.WriteString(k) - buf.WriteByte('"') - buf.WriteByte(':') - switch v := v.(type) { - case float64: - buf.WriteString(strconv.FormatFloat(v, 'f', 2, 64)) - case uint64: - buf.WriteString(strconv.FormatUint(v, 10)) - case int64: - buf.WriteString(strconv.FormatInt(v, 10)) - default: - panic(fmt.Sprintf("unexpected type: %T", v)) - } - if j != m-1 { - buf.WriteByte(',') - } - j++ - } - buf.WriteByte('}') - if i != n-1 { - buf.WriteByte(',') - } - i++ - } - buf.WriteByte(']') - return buf.Bytes(), nil -} diff --git a/internal/metrics/systeminfo/system_info_test.go b/internal/metrics/systeminfo/system_info_test.go index 6579069..25740a9 100644 --- a/internal/metrics/systeminfo/system_info_test.go +++ b/internal/metrics/systeminfo/system_info_test.go @@ -1,13 +1,13 @@ package systeminfo import ( - "encoding/json" "net/url" "reflect" "testing" "github.com/shirou/gopsutil/v4/sensors" . "github.com/yusing/go-proxy/internal/utils/testing" + "github.com/yusing/go-proxy/pkg/json" ) func TestExcludeDisks(t *testing.T) { @@ -191,8 +191,7 @@ func TestSerialize(t *testing.T) { for _, query := range allQueries { t.Run(query, func(t *testing.T) { _, result := aggregate(entries, url.Values{"aggregate": []string{query}}) - s, err := result.MarshalJSON() - ExpectNoError(t, err) + s := result.MarshalJSONTo(nil) var v []map[string]any ExpectNoError(t, json.Unmarshal(s, &v)) ExpectEqual(t, len(v), len(result)) @@ -206,31 +205,3 @@ func TestSerialize(t *testing.T) { }) } } - -func BenchmarkJSONMarshal(b *testing.B) { - entries := make([]*SystemInfo, b.N) - for i := range b.N { - entries[i] = testInfo - } - queries := map[string]Aggregated{} - for _, query := range allQueries { - _, result := aggregate(entries, url.Values{"aggregate": []string{query}}) - queries[query] = result - } - b.ReportAllocs() - b.ResetTimer() - b.Run("optimized", func(b *testing.B) { - for b.Loop() { - for _, query := range allQueries { - _, _ = queries[query].MarshalJSON() - } - } - }) - b.Run("json", func(b *testing.B) { - for b.Loop() { - for _, query := range allQueries { - _, _ = json.Marshal([]map[string]any(queries[query])) - } - } - }) -} diff --git a/internal/metrics/uptime/status.go b/internal/metrics/uptime/status.go new file mode 100644 index 0000000..f101ae1 --- /dev/null +++ b/internal/metrics/uptime/status.go @@ -0,0 +1,22 @@ +package uptime + +import ( + "fmt" + + "github.com/yusing/go-proxy/internal/watcher/health" +) + +type Status struct { + Status health.Status + Latency int64 + Timestamp int64 +} + +type RouteStatuses map[string][]*Status + +func (s *Status) MarshalJSONTo(buf []byte) []byte { + return fmt.Appendf(buf, + `{"status":"%s","latency":"%d","timestamp":"%d"}`, + s.Status, s.Latency, s.Timestamp, + ) +} diff --git a/internal/metrics/uptime/uptime.go b/internal/metrics/uptime/uptime.go index 3162e3d..41b0732 100644 --- a/internal/metrics/uptime/uptime.go +++ b/internal/metrics/uptime/uptime.go @@ -2,7 +2,6 @@ package uptime import ( "context" - "encoding/json" "net/url" "sort" "time" @@ -13,20 +12,15 @@ import ( "github.com/yusing/go-proxy/internal/route/routes" "github.com/yusing/go-proxy/internal/route/routes/routequery" "github.com/yusing/go-proxy/internal/watcher/health" + "github.com/yusing/go-proxy/pkg/json" ) type ( StatusByAlias struct { - Map map[string]*routequery.HealthInfoRaw `json:"statuses"` - Timestamp int64 `json:"timestamp"` + Map json.Map[*routequery.HealthInfoRaw] `json:"statuses"` + Timestamp int64 `json:"timestamp"` } - Status struct { - Status health.Status `json:"status"` - Latency int64 `json:"latency"` - Timestamp int64 `json:"timestamp"` - } - RouteStatuses map[string][]*Status - Aggregated []map[string]any + Aggregated = json.MapSlice[any] ) var Poller = period.NewPoller("uptime", getStatuses, aggregateStatuses) @@ -124,7 +118,3 @@ func (rs RouteStatuses) aggregate(limit int, offset int) Aggregated { } return result } - -func (result Aggregated) MarshalJSON() ([]byte, error) { - return json.Marshal([]map[string]any(result)) -} diff --git a/internal/metrics/uptime/uptime_test.go b/internal/metrics/uptime/uptime_test.go new file mode 100644 index 0000000..8182aad --- /dev/null +++ b/internal/metrics/uptime/uptime_test.go @@ -0,0 +1,10 @@ +package uptime + +import ( + "net/url" + "testing" +) + +func TestPoller(t *testing.T) { + Poller.Test(t, url.Values{"limit": []string{"1"}}) +} diff --git a/internal/net/gphttp/accesslog/access_logger.go b/internal/net/gphttp/accesslog/access_logger.go index 1637377..3f56fa2 100644 --- a/internal/net/gphttp/accesslog/access_logger.go +++ b/internal/net/gphttp/accesslog/access_logger.go @@ -194,6 +194,6 @@ func (l *AccessLogger) write(data []byte) { if err != nil { l.handleErr(err) } else { - logging.Debug().Msg("access log flushed to " + l.io.Name()) + logging.Trace().Msg("access log flushed to " + l.io.Name()) } } diff --git a/internal/net/gphttp/accesslog/access_logger_test.go b/internal/net/gphttp/accesslog/access_logger_test.go index 7e7af45..ee903c4 100644 --- a/internal/net/gphttp/accesslog/access_logger_test.go +++ b/internal/net/gphttp/accesslog/access_logger_test.go @@ -2,13 +2,14 @@ package accesslog_test import ( "bytes" - "encoding/json" "fmt" "net/http" "net/url" "testing" "time" + "github.com/yusing/go-proxy/pkg/json" + . "github.com/yusing/go-proxy/internal/net/gphttp/accesslog" "github.com/yusing/go-proxy/internal/task" . "github.com/yusing/go-proxy/internal/utils/testing" diff --git a/internal/net/gphttp/accesslog/formatter.go b/internal/net/gphttp/accesslog/formatter.go index d0fcd68..deca013 100644 --- a/internal/net/gphttp/accesslog/formatter.go +++ b/internal/net/gphttp/accesslog/formatter.go @@ -2,13 +2,14 @@ package accesslog import ( "bytes" - "encoding/json" "net" "net/http" "net/url" "strconv" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/logging" ) diff --git a/internal/net/gphttp/body.go b/internal/net/gphttp/body.go index 3a94023..45d78e7 100644 --- a/internal/net/gphttp/body.go +++ b/internal/net/gphttp/body.go @@ -2,11 +2,12 @@ package gphttp import ( "context" - "encoding/json" "errors" "fmt" "net/http" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/logging" ) diff --git a/internal/net/gphttp/error.go b/internal/net/gphttp/error.go index f269e3f..fc37077 100644 --- a/internal/net/gphttp/error.go +++ b/internal/net/gphttp/error.go @@ -2,11 +2,12 @@ package gphttp import ( "context" - "encoding/json" "errors" "net/http" "syscall" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/net/gphttp/httpheaders" ) diff --git a/internal/net/gphttp/middleware/middleware.go b/internal/net/gphttp/middleware/middleware.go index c6b5fd8..cc12aa9 100644 --- a/internal/net/gphttp/middleware/middleware.go +++ b/internal/net/gphttp/middleware/middleware.go @@ -1,12 +1,13 @@ package middleware import ( - "encoding/json" "net/http" "reflect" "sort" "strings" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/logging" gphttp "github.com/yusing/go-proxy/internal/net/gphttp" @@ -158,12 +159,12 @@ func (m *Middleware) String() string { return m.name } -func (m *Middleware) MarshalJSON() ([]byte, error) { - return json.MarshalIndent(map[string]any{ +func (m *Middleware) MarshalJSONTo(buf []byte) []byte { + return json.MarshalTo(map[string]any{ "name": m.name, "options": m.impl, "priority": m.priority, - }, "", " ") + }, buf) } func (m *Middleware) ModifyRequest(next http.HandlerFunc, w http.ResponseWriter, r *http.Request) { diff --git a/internal/net/gphttp/middleware/middleware_builder.go b/internal/net/gphttp/middleware/middleware_builder.go index c7be596..2401334 100644 --- a/internal/net/gphttp/middleware/middleware_builder.go +++ b/internal/net/gphttp/middleware/middleware_builder.go @@ -6,8 +6,8 @@ import ( "path" "sort" + "github.com/goccy/go-yaml" "github.com/yusing/go-proxy/internal/gperr" - "gopkg.in/yaml.v3" ) var ErrMissingMiddlewareUse = gperr.New("missing middleware 'use' field") diff --git a/internal/net/gphttp/middleware/middleware_builder_test.go b/internal/net/gphttp/middleware/middleware_builder_test.go index 08e8402..3036ca6 100644 --- a/internal/net/gphttp/middleware/middleware_builder_test.go +++ b/internal/net/gphttp/middleware/middleware_builder_test.go @@ -2,9 +2,10 @@ package middleware import ( _ "embed" - "encoding/json" "testing" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/gperr" . "github.com/yusing/go-proxy/internal/utils/testing" ) @@ -16,7 +17,7 @@ func TestBuild(t *testing.T) { errs := gperr.NewBuilder("") middlewares := BuildMiddlewaresFromYAML("", testMiddlewareCompose, errs) ExpectNoError(t, errs.Error()) - Must(json.MarshalIndent(middlewares, "", " ")) + json.Marshal(middlewares) // t.Log(string(data)) // TODO: test } diff --git a/internal/net/gphttp/middleware/test_utils.go b/internal/net/gphttp/middleware/test_utils.go index edb5e24..3448734 100644 --- a/internal/net/gphttp/middleware/test_utils.go +++ b/internal/net/gphttp/middleware/test_utils.go @@ -3,12 +3,12 @@ package middleware import ( "bytes" _ "embed" - "encoding/json" "io" "net/http" "net/http/httptest" "net/url" + "github.com/yusing/go-proxy/pkg/json" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy" diff --git a/internal/net/types/stream.go b/internal/net/types/stream.go index 2892a5f..371621d 100644 --- a/internal/net/types/stream.go +++ b/internal/net/types/stream.go @@ -1,4 +1,4 @@ -package types +package gpnet import ( "fmt" diff --git a/internal/notif/format.go b/internal/notif/format.go index f54d897..da51763 100644 --- a/internal/notif/format.go +++ b/internal/notif/format.go @@ -2,7 +2,8 @@ package notif import ( "bytes" - "encoding/json" + + "github.com/yusing/go-proxy/pkg/json" ) func formatMarkdown(extras LogFields) string { diff --git a/internal/notif/gotify.go b/internal/notif/gotify.go index bb2d951..390d4af 100644 --- a/internal/notif/gotify.go +++ b/internal/notif/gotify.go @@ -2,11 +2,12 @@ package notif import ( "bytes" - "encoding/json" "fmt" "io" "net/http" + "github.com/yusing/go-proxy/pkg/json" + "github.com/gotify/server/v2/model" "github.com/rs/zerolog" ) diff --git a/internal/notif/webhook.go b/internal/notif/webhook.go index 13fd23c..bccd5fc 100644 --- a/internal/notif/webhook.go +++ b/internal/notif/webhook.go @@ -2,12 +2,13 @@ package notif import ( _ "embed" - "encoding/json" "fmt" "io" "net/http" "strings" + "github.com/yusing/go-proxy/pkg/json" + "github.com/yusing/go-proxy/internal/gperr" ) @@ -101,10 +102,7 @@ func (webhook *Webhook) makeRespError(resp *http.Response) error { } func (webhook *Webhook) MakeBody(logMsg *LogMessage) (io.Reader, error) { - title, err := json.Marshal(logMsg.Title) - if err != nil { - return nil, err - } + title := json.String(logMsg.Title) fields, err := formatDiscord(logMsg.Extras) if err != nil { return nil, err @@ -115,13 +113,10 @@ func (webhook *Webhook) MakeBody(logMsg *LogMessage) (io.Reader, error) { } else { color = logMsg.Color.DecString() } - message, err := json.Marshal(formatMarkdown(logMsg.Extras)) - if err != nil { - return nil, err - } + message := json.String(formatMarkdown(logMsg.Extras)) plTempl := strings.NewReplacer( - "$title", string(title), - "$message", string(message), + "$title", title, + "$message", message, "$fields", fields, "$color", color, ) diff --git a/internal/route/provider/docker.go b/internal/route/provider/docker.go index 592b3e5..2658593 100755 --- a/internal/route/provider/docker.go +++ b/internal/route/provider/docker.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/docker/docker/client" + "github.com/goccy/go-yaml" "github.com/rs/zerolog" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/docker" @@ -14,7 +15,6 @@ import ( U "github.com/yusing/go-proxy/internal/utils" "github.com/yusing/go-proxy/internal/utils/strutils" "github.com/yusing/go-proxy/internal/watcher" - "gopkg.in/yaml.v3" ) type DockerProvider struct { diff --git a/internal/route/provider/docker_labels_test.go b/internal/route/provider/docker_labels_test.go index d9e400b..4d5b221 100644 --- a/internal/route/provider/docker_labels_test.go +++ b/internal/route/provider/docker_labels_test.go @@ -4,9 +4,9 @@ import ( "testing" "github.com/docker/docker/api/types" + "github.com/goccy/go-yaml" "github.com/yusing/go-proxy/internal/docker" . "github.com/yusing/go-proxy/internal/utils/testing" - "gopkg.in/yaml.v3" _ "embed" ) diff --git a/internal/route/routes/routequery/query.go b/internal/route/routes/routequery/query.go index f1b3037..d1bcf7f 100644 --- a/internal/route/routes/routequery/query.go +++ b/internal/route/routes/routequery/query.go @@ -26,8 +26,8 @@ func getHealthInfo(r route.Route) map[string]string { } type HealthInfoRaw struct { - Status health.Status - Latency time.Duration + Status health.Status `json:"status,string"` + Latency time.Duration `json:"latency"` } func getHealthInfoRaw(r route.Route) *HealthInfoRaw { diff --git a/internal/route/rules/do.go b/internal/route/rules/do.go index f113a4c..336a90b 100644 --- a/internal/route/rules/do.go +++ b/internal/route/rules/do.go @@ -300,7 +300,3 @@ func (cmd *Command) isBypass() bool { func (cmd *Command) String() string { return cmd.raw } - -func (cmd *Command) MarshalText() ([]byte, error) { - return []byte(cmd.String()), nil -} diff --git a/internal/route/rules/on.go b/internal/route/rules/on.go index f69e7a8..da3e453 100644 --- a/internal/route/rules/on.go +++ b/internal/route/rules/on.go @@ -261,10 +261,6 @@ func (on *RuleOn) String() string { return on.raw } -func (on *RuleOn) MarshalText() ([]byte, error) { - return []byte(on.String()), nil -} - func parseOn(line string) (Checker, gperr.Error) { ors := strutils.SplitRune(line, '|') diff --git a/internal/route/rules/rules.go b/internal/route/rules/rules.go index 4d1dbcc..032c523 100644 --- a/internal/route/rules/rules.go +++ b/internal/route/rules/rules.go @@ -1,8 +1,9 @@ package rules import ( - "encoding/json" "net/http" + + "github.com/yusing/go-proxy/pkg/json" ) type ( @@ -40,8 +41,8 @@ type ( */ Rule struct { Name string `json:"name"` - On RuleOn `json:"on"` - Do Command `json:"do"` + On RuleOn `json:"on,string"` + Do Command `json:"do,string"` } ) @@ -102,12 +103,12 @@ func (rules Rules) BuildHandler(caller string, up http.Handler) http.HandlerFunc } } -func (rules Rules) MarshalJSON() ([]byte, error) { +func (rules Rules) MarshalJSONTo(buf []byte) []byte { names := make([]string, len(rules)) for i, rule := range rules { names[i] = rule.Name } - return json.Marshal(names) + return json.MarshalTo(names, buf) } func (rule *Rule) String() string { diff --git a/internal/utils/atomic/value.go b/internal/utils/atomic/value.go index 65140fe..d05f80b 100644 --- a/internal/utils/atomic/value.go +++ b/internal/utils/atomic/value.go @@ -1,8 +1,9 @@ package atomic import ( - "encoding/json" "sync/atomic" + + "github.com/yusing/go-proxy/pkg/json" ) type Value[T any] struct { @@ -29,6 +30,6 @@ func (a *Value[T]) Swap(v T) T { return zero } -func (a *Value[T]) MarshalJSON() ([]byte, error) { - return json.Marshal(a.Load()) +func (a *Value[T]) MarshalJSONTo(buf []byte) []byte { + return json.MarshalTo(a.Load(), buf) } diff --git a/internal/utils/functional/map.go b/internal/utils/functional/map.go index 1efc9ab..3516e03 100644 --- a/internal/utils/functional/map.go +++ b/internal/utils/functional/map.go @@ -3,8 +3,8 @@ package functional import ( "sync" + "github.com/goccy/go-yaml" "github.com/puzpuzpuz/xsync/v3" - "gopkg.in/yaml.v3" ) type Map[KT comparable, VT any] struct { diff --git a/internal/utils/serialization.go b/internal/utils/serialization.go index 5120e29..c150a84 100644 --- a/internal/utils/serialization.go +++ b/internal/utils/serialization.go @@ -1,7 +1,7 @@ package utils import ( - "encoding/json" + "bytes" "errors" "net" "net/url" @@ -11,11 +11,13 @@ import ( "strings" "time" + "github.com/yusing/go-proxy/pkg/json" + "github.com/go-playground/validator/v10" + "github.com/goccy/go-yaml" "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/utils/functional" "github.com/yusing/go-proxy/internal/utils/strutils" - "gopkg.in/yaml.v3" ) type SerializedObject = map[string]any @@ -24,9 +26,12 @@ type ( MapMarshaler interface { MarshalMap() map[string]any } - MapUnmarshaller interface { + MapUnmarshaler interface { UnmarshalMap(m map[string]any) gperr.Error } + Marshaler interface { + MarshalJSONTo(buf *bytes.Buffer) error + } ) var ( @@ -50,12 +55,8 @@ var ( typeURL = reflect.TypeFor[url.URL]() typeCIDR = reflect.TypeFor[net.IPNet]() - typeMapMarshaller = reflect.TypeFor[MapMarshaler]() - typeMapUnmarshaler = reflect.TypeFor[MapUnmarshaller]() - typeJSONMarshaller = reflect.TypeFor[json.Marshaler]() + typeMapUnmarshaler = reflect.TypeFor[MapUnmarshaler]() typeStrParser = reflect.TypeFor[strutils.Parser]() - - typeAny = reflect.TypeOf((*any)(nil)).Elem() ) var defaultValues = functional.NewMapOf[reflect.Type, func() any]() @@ -92,14 +93,14 @@ func extractFields(t reflect.Type) (all, anonymous []reflect.StructField) { if !field.IsExported() { continue } + // not checking tagJSON because json:"-" is for skipping json.Marshal if field.Tag.Get(tagDeserialize) == "-" { continue } if field.Anonymous { - f1, f2 := extractFields(field.Type) - fields = append(fields, f1...) + nested, _ := extractFields(field.Type) + fields = append(fields, nested...) anonymous = append(anonymous, field) - anonymous = append(anonymous, f2...) } else { fields = append(fields, field) } @@ -215,7 +216,7 @@ func MapUnmarshalValidate(src SerializedObject, dst any) (err gperr.Error) { if err != nil { return err } - return dstV.Addr().Interface().(MapUnmarshaller).UnmarshalMap(src) + return dstV.Addr().Interface().(MapUnmarshaler).UnmarshalMap(src) } dstV, dstT, err = dive(dstV) @@ -560,7 +561,7 @@ func ConvertString(src string, dst reflect.Value) (convertible bool, convErr gpe func UnmarshalValidateYAML[T any](data []byte, target *T) gperr.Error { m := make(map[string]any) - if err := yaml.Unmarshal(data, m); err != nil { + if err := yaml.Unmarshal(data, &m); err != nil { return gperr.Wrap(err) } return MapUnmarshalValidate(m, target) @@ -568,7 +569,7 @@ func UnmarshalValidateYAML[T any](data []byte, target *T) gperr.Error { func UnmarshalValidateYAMLMap[V any](data []byte) (_ functional.Map[string, V], err gperr.Error) { m := make(map[string]any) - if err = gperr.Wrap(yaml.Unmarshal(data, m)); err != nil { + if err = gperr.Wrap(yaml.Unmarshal(data, &m)); err != nil { return } m2 := make(map[string]V, len(m)) diff --git a/internal/utils/serialization_test.go b/internal/utils/serialization_test.go index 4c67c75..56907b8 100644 --- a/internal/utils/serialization_test.go +++ b/internal/utils/serialization_test.go @@ -8,8 +8,8 @@ import ( "strconv" "testing" + "github.com/goccy/go-yaml" . "github.com/yusing/go-proxy/internal/utils/testing" - "gopkg.in/yaml.v3" ) func TestUnmarshal(t *testing.T) { diff --git a/internal/watcher/health/monitor/agent_proxied.go b/internal/watcher/health/monitor/agent_proxied.go index 67db85c..b8aa102 100644 --- a/internal/watcher/health/monitor/agent_proxied.go +++ b/internal/watcher/health/monitor/agent_proxied.go @@ -1,11 +1,12 @@ package monitor import ( - "encoding/json" "errors" "net/http" "net/url" + "github.com/yusing/go-proxy/pkg/json" + agentPkg "github.com/yusing/go-proxy/agent/pkg/agent" "github.com/yusing/go-proxy/internal/watcher/health" ) diff --git a/internal/watcher/health/status.go b/internal/watcher/health/status.go index 355dd70..f58022c 100644 --- a/internal/watcher/health/status.go +++ b/internal/watcher/health/status.go @@ -1,6 +1,8 @@ package health -import "encoding/json" +import ( + "github.com/yusing/go-proxy/pkg/json" +) type Status uint8 @@ -35,10 +37,6 @@ func (s Status) String() string { } } -func (s Status) MarshalJSON() ([]byte, error) { - return []byte(`"` + s.String() + `"`), nil -} - func (s *Status) UnmarshalJSON(data []byte) error { var str string if err := json.Unmarshal(data, &str); err != nil { diff --git a/pkg/json/check_empty.go b/pkg/json/check_empty.go new file mode 100644 index 0000000..7f0b1b5 --- /dev/null +++ b/pkg/json/check_empty.go @@ -0,0 +1,55 @@ +package json + +import "reflect" + +type checkEmptyFunc func(v reflect.Value) bool + +var checkEmptyFuncs = map[reflect.Kind]checkEmptyFunc{ + reflect.String: checkStringEmpty, + reflect.Int: checkIntEmpty, + reflect.Int8: checkIntEmpty, + reflect.Int16: checkIntEmpty, + reflect.Int32: checkIntEmpty, + reflect.Int64: checkIntEmpty, + reflect.Uint: checkUintEmpty, + reflect.Uint8: checkUintEmpty, + reflect.Uint16: checkUintEmpty, + reflect.Uint32: checkUintEmpty, + reflect.Uint64: checkUintEmpty, + reflect.Float32: checkFloatEmpty, + reflect.Float64: checkFloatEmpty, + reflect.Bool: checkBoolEmpty, + reflect.Slice: checkLenEmpty, + reflect.Map: checkLenEmpty, + reflect.Array: checkLenEmpty, + reflect.Chan: reflect.Value.IsNil, + reflect.Func: reflect.Value.IsNil, + reflect.Interface: reflect.Value.IsNil, + reflect.Pointer: reflect.Value.IsNil, + reflect.Struct: reflect.Value.IsZero, + reflect.UnsafePointer: reflect.Value.IsNil, +} + +func checkStringEmpty(v reflect.Value) bool { + return v.String() == "" +} + +func checkIntEmpty(v reflect.Value) bool { + return v.Int() == 0 +} + +func checkUintEmpty(v reflect.Value) bool { + return v.Uint() == 0 +} + +func checkFloatEmpty(v reflect.Value) bool { + return v.Float() == 0 +} + +func checkBoolEmpty(v reflect.Value) bool { + return !v.Bool() +} + +func checkLenEmpty(v reflect.Value) bool { + return v.Len() == 0 +} diff --git a/pkg/json/encoder.go b/pkg/json/encoder.go new file mode 100644 index 0000000..a8f2a49 --- /dev/null +++ b/pkg/json/encoder.go @@ -0,0 +1,17 @@ +package json + +import "io" + +type Encoder struct { + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(v any) error { + data, _ := Marshal(v) + _, err := e.w.Write(data) + return err +} diff --git a/pkg/json/json.go b/pkg/json/json.go new file mode 100644 index 0000000..470c65b --- /dev/null +++ b/pkg/json/json.go @@ -0,0 +1,70 @@ +package json + +import ( + "reflect" + "sync" + + "github.com/bytedance/sonic" +) + +type Marshaler interface { + MarshalJSONTo(buf []byte) []byte +} + +var ( + Unmarshal = sonic.Unmarshal + Valid = sonic.Valid + NewDecoder = sonic.ConfigDefault.NewDecoder +) + +// Marshal returns the JSON encoding of v. +// +// It's like json.Marshal, but with some differences: +// +// - It's ~4-5x faster in most cases. +// +// - It also supports custom Marshaler interface (MarshalJSONTo(buf []byte) []byte) +// to allow further optimizations. +// +// - It leverages the strutils library. +// +// - It drops the need to implement Marshaler or json.Marshaler by supports extra field tags: +// +// `byte_size` to format the field to human readable size. +// +// `unix_time` to format the uint64 field to string date-time without specifying MarshalJSONTo. +// +// `use_marshaler` to force using the custom marshaler for primitive types declaration (e.g. `type Status int`). +// +// - It correct the behavior of *url.URL and time.Duration. +// +// - It does not support maps other than string-keyed maps. +func Marshal(v any) ([]byte, error) { + buf := newBytes() + defer putBytes(buf) + return cloneBytes(appendMarshal(reflect.ValueOf(v), buf)), nil +} + +func MarshalTo(v any, buf []byte) []byte { + return appendMarshal(reflect.ValueOf(v), buf) +} + +const bufSize = 1024 + +var bytesPool = sync.Pool{ + New: func() any { + return make([]byte, 0, bufSize) + }, +} + +func newBytes() []byte { + return bytesPool.Get().([]byte) +} + +func putBytes(buf []byte) { + bytesPool.Put(buf[:0]) +} + +func cloneBytes(buf []byte) (res []byte) { + return append(res, buf...) +} diff --git a/pkg/json/map.go b/pkg/json/map.go new file mode 100644 index 0000000..86eff00 --- /dev/null +++ b/pkg/json/map.go @@ -0,0 +1,24 @@ +package json + +import ( + "reflect" +) + +type Map[V any] map[string]V + +func (m Map[V]) MarshalJSONTo(buf []byte) []byte { + buf = append(buf, '{') + i := 0 + n := len(m) + for k, v := range m { + buf = AppendString(buf, k) + buf = append(buf, ':') + buf = appendMarshal(reflect.ValueOf(v), buf) + if i != n-1 { + buf = append(buf, ',') + } + i++ + } + buf = append(buf, '}') + return buf +} diff --git a/pkg/json/map_slice.go b/pkg/json/map_slice.go new file mode 100644 index 0000000..c1878ce --- /dev/null +++ b/pkg/json/map_slice.go @@ -0,0 +1,18 @@ +package json + +type MapSlice[V any] []Map[V] + +func (s MapSlice[V]) MarshalJSONTo(buf []byte) []byte { + buf = append(buf, '[') + i := 0 + n := len(s) + for _, entry := range s { + buf = entry.MarshalJSONTo(buf) + if i != n-1 { + buf = append(buf, ',') + } + i++ + } + buf = append(buf, ']') + return buf +} diff --git a/pkg/json/marshal.go b/pkg/json/marshal.go new file mode 100644 index 0000000..a22f9fd --- /dev/null +++ b/pkg/json/marshal.go @@ -0,0 +1,269 @@ +package json + +import ( + "encoding" + stdJSON "encoding/json" + + "fmt" + "net" + "net/url" + "reflect" + "strconv" + "time" + + "github.com/puzpuzpuz/xsync/v3" +) + +type marshalFunc func(v reflect.Value, buf []byte) []byte + +var ( + marshalFuncByKind map[reflect.Kind]marshalFunc + + marshalFuncsByType = newCacheMap[reflect.Type, marshalFunc]() + flattenFieldsCache = newCacheMap[reflect.Type, []*field]() + + nilValue = reflect.ValueOf(nil) +) + +func init() { + marshalFuncByKind = map[reflect.Kind]marshalFunc{ + reflect.String: appendString, + reflect.Bool: appendBool, + reflect.Int: appendInt, + reflect.Int8: appendInt, + reflect.Int16: appendInt, + reflect.Int32: appendInt, + reflect.Int64: appendInt, + reflect.Uint: appendUint, + reflect.Uint8: appendUint, + reflect.Uint16: appendUint, + reflect.Uint32: appendUint, + reflect.Uint64: appendUint, + reflect.Float32: appendFloat, + reflect.Float64: appendFloat, + reflect.Map: appendMap, + reflect.Slice: appendArray, + reflect.Array: appendArray, + reflect.Pointer: appendPtrInterface, + reflect.Interface: appendPtrInterface, + } + // pre-caching some frequently used types + marshalFuncsByType.Store(reflect.TypeFor[*url.URL](), appendStringer) + marshalFuncsByType.Store(reflect.TypeFor[net.IP](), appendStringer) + marshalFuncsByType.Store(reflect.TypeFor[*net.IPNet](), appendStringer) + marshalFuncsByType.Store(reflect.TypeFor[time.Time](), appendTime) + marshalFuncsByType.Store(reflect.TypeFor[time.Duration](), appendDuration) +} + +func newCacheMap[K comparable, V any]() *xsync.MapOf[K, V] { + return xsync.NewMapOf[K, V]( + xsync.WithGrowOnly(), + xsync.WithPresize(50), + ) +} + +func must(buf []byte, err error) []byte { + if err != nil { + panic(fmt.Errorf("custom json marshal error: %w", err)) + } + return buf +} + +func appendMarshal(v reflect.Value, buf []byte) []byte { + if v == nilValue { + return append(buf, "null"...) + } + kind := v.Kind() + if kind == reflect.Struct { + if res, ok := appendWithCachedFunc(v, buf); ok { + return res + } + return appendStruct(v, buf) + } + marshalFunc, ok := marshalFuncByKind[kind] + if !ok { + panic(fmt.Errorf("unsupported type: %s", v.Type())) + } + return marshalFunc(v, buf) +} + +func appendWithCachedFunc(v reflect.Value, buf []byte) (res []byte, ok bool) { + marshalFunc, ok := marshalFuncsByType.Load(v.Type()) + if ok { + return marshalFunc(v, buf), true + } + return nil, false +} + +func appendBool(v reflect.Value, buf []byte) []byte { + return strconv.AppendBool(buf, v.Bool()) +} + +func appendInt(v reflect.Value, buf []byte) []byte { + return strconv.AppendInt(buf, v.Int(), 10) +} + +func appendUint(v reflect.Value, buf []byte) []byte { + return strconv.AppendUint(buf, v.Uint(), 10) +} + +func appendFloat(v reflect.Value, buf []byte) []byte { + return strconv.AppendFloat(buf, v.Float(), 'f', 2, 64) +} + +func appendWithCustomMarshaler(v reflect.Value, buf []byte) (res []byte, ok bool) { + switch vv := v.Interface().(type) { + case Marshaler: + cacheMarshalFunc(v.Type(), appendWithMarshalTo) + return vv.MarshalJSONTo(buf), true + case fmt.Stringer: + cacheMarshalFunc(v.Type(), appendStringer) + return AppendString(buf, vv.String()), true + case stdJSON.Marshaler: + cacheMarshalFunc(v.Type(), appendStdJSONMarshaler) + return append(buf, must(vv.MarshalJSON())...), true + case encoding.BinaryAppender: + cacheMarshalFunc(v.Type(), appendBinaryAppender) + //FIXME: append escaped + return must(vv.AppendBinary(buf)), true + case encoding.TextAppender: + cacheMarshalFunc(v.Type(), appendTextAppender) + //FIXME: append escaped + return must(vv.AppendText(buf)), true + case encoding.TextMarshaler: + cacheMarshalFunc(v.Type(), appendTestMarshaler) + return AppendString(buf, must(vv.MarshalText())), true + case encoding.BinaryMarshaler: + cacheMarshalFunc(v.Type(), appendBinaryMarshaler) + return AppendString(buf, must(vv.MarshalBinary())), true + } + return nil, false +} + +func mustAppendWithCustomMarshaler(v reflect.Value, buf []byte) []byte { + res, ok := appendWithCustomMarshaler(v, buf) + if !ok { + panic(fmt.Errorf("tag %q used but no marshaler implemented: %s", tagUseMarshaler, v.Type())) + } + return res +} + +func appendKV(k reflect.Value, v reflect.Value, buf []byte) []byte { + buf = AppendString(buf, k.String()) + buf = append(buf, ':') + return appendMarshal(v, buf) +} + +func appendStruct(v reflect.Value, buf []byte) []byte { + if res, ok := appendWithCustomMarshaler(v, buf); ok { + return res + } + buf = append(buf, '{') + oldN := len(buf) + fields := flattenFields(v.Type()) + + for _, f := range fields { + cur := v.Field(f.index) + if f.omitEmpty && f.checkEmpty(cur) { + continue + } + if !f.hasInner { + buf = f.appendKV(cur, buf) + buf = append(buf, ',') + } else { + if f.isPtr { + cur = cur.Elem() + } + for _, inner := range f.inner { + buf = inner.appendKV(cur.Field(inner.index), buf) + buf = append(buf, ',') + } + } + } + + n := len(buf) + if oldN != n { + buf = buf[:n-1] + } + return append(buf, '}') +} + +func appendMap(v reflect.Value, buf []byte) []byte { + if v.Type().Key().Kind() != reflect.String { + panic(fmt.Errorf("map key must be string: %s", v.Type())) + } + buf = append(buf, '{') + i := 0 + oldN := len(buf) + iter := v.MapRange() + for iter.Next() { + k := iter.Key() + v := iter.Value() + buf = appendKV(k, v, buf) + buf = append(buf, ',') + i++ + } + n := len(buf) + if oldN != n { + buf = buf[:n-1] + } + return append(buf, '}') +} + +func appendArray(v reflect.Value, buf []byte) []byte { + switch v.Type().Elem().Kind() { + case reflect.String: + return appendStringSlice(v, buf) + case reflect.Uint8: // byte + return appendBytesAsBase64(v, buf) + } + buf = append(buf, '[') + oldN := len(buf) + for i := range v.Len() { + buf = appendMarshal(v.Index(i), buf) + buf = append(buf, ',') + } + n := len(buf) + if oldN != n { + buf = buf[:n-1] + } + return append(buf, ']') +} + +func cacheMarshalFunc(t reflect.Type, marshalFunc marshalFunc) { + marshalFuncsByType.Store(t, marshalFunc) +} + +func appendPtrInterface(v reflect.Value, buf []byte) []byte { + return appendMarshal(v.Elem(), buf) +} + +func appendWithMarshalTo(v reflect.Value, buf []byte) []byte { + return v.Interface().(Marshaler).MarshalJSONTo(buf) +} + +func appendStringer(v reflect.Value, buf []byte) []byte { + return AppendString(buf, v.Interface().(fmt.Stringer).String()) +} + +func appendStdJSONMarshaler(v reflect.Value, buf []byte) []byte { + return append(buf, must(v.Interface().(stdJSON.Marshaler).MarshalJSON())...) +} + +func appendBinaryAppender(v reflect.Value, buf []byte) []byte { + //FIXME: append escaped + return must(v.Interface().(encoding.BinaryAppender).AppendBinary(buf)) +} + +func appendTextAppender(v reflect.Value, buf []byte) []byte { + //FIXME: append escaped + return must(v.Interface().(encoding.TextAppender).AppendText(buf)) +} + +func appendTestMarshaler(v reflect.Value, buf []byte) []byte { + return AppendString(buf, must(v.Interface().(encoding.TextMarshaler).MarshalText())) +} + +func appendBinaryMarshaler(v reflect.Value, buf []byte) []byte { + return AppendString(buf, must(v.Interface().(encoding.BinaryMarshaler).MarshalBinary())) +} diff --git a/pkg/json/marshal_test.go b/pkg/json/marshal_test.go new file mode 100644 index 0000000..02d1599 --- /dev/null +++ b/pkg/json/marshal_test.go @@ -0,0 +1,529 @@ +package json_test + +import ( + stdJSON "encoding/json" + "fmt" + "maps" + "reflect" + "runtime/debug" + "strconv" + "testing" + + "github.com/bytedance/sonic" + "github.com/stretchr/testify/require" + "github.com/yusing/go-proxy/internal/utils/strutils" + . "github.com/yusing/go-proxy/pkg/json" +) + +func init() { + debug.SetMemoryLimit(1024 * 1024) + debug.SetMaxStack(1024 * 1024) +} + +type testStruct struct { + Name string `json:"name"` + Age int `json:"age"` + Score float64 `json:"score"` + Empty *struct { + Value string `json:"value,omitempty"` + } `json:"empty,omitempty"` +} + +type stringer struct { + testStruct +} + +func (s stringer) String() string { + return s.Name +} + +type customMarshaler struct { + Value string +} + +func (cm customMarshaler) MarshalJSONTo(buf []byte) []byte { + return append(buf, []byte(`{"custom":"`+cm.Value+`"}`)...) +} + +type jsonMarshaler struct { + Value string +} + +func (jm jsonMarshaler) MarshalJSON() ([]byte, error) { + return []byte(`{"json_marshaler":"` + jm.Value + `"}`), nil +} + +type withJSONTag struct { + Value string `json:"value"` +} + +type withJSONOmitEmpty struct { + Value string `json:"value,omitempty"` +} + +type withJSONStringTag struct { + Value int64 `json:"value,string"` +} + +type withJSONOmit struct { + Value string `json:"-"` +} + +type withJSONByteSize struct { + Value uint64 `json:"value,byte_size"` +} + +type withJSONUnixTime struct { + Value int64 `json:"value,unix_time"` +} + +type primitiveWithMarshaler int + +func (p primitiveWithMarshaler) MarshalJSONTo(buf []byte) []byte { + return fmt.Appendf(buf, `%q`, strconv.Itoa(int(p))) +} + +type withTagUseMarshaler struct { + Value primitiveWithMarshaler `json:"value,use_marshaler"` +} + +type Anonymous struct { + Value string `json:"value"` + Value2 int `json:"value2"` +} + +type withAnonymous struct { + Anonymous +} + +type withPointerAnonymous struct { + *Anonymous +} + +type selfReferencing struct { + Self *selfReferencing `json:"self"` +} + +var testData = map[string]any{ + "string": "test string", + "number": 42, + "float": 3.14159, + "bool": true, + "null_value": nil, + "array": []any{1, "2", 3.3, true, false, nil}, + "object": map[string]any{ + "nested": "value", + "count": 10, + }, +} + +func TestMarshal(t *testing.T) { + tests := []struct { + name string + input any + expected string + }{ + { + name: "string", + input: "test", + expected: `"test"`, + }, + { + name: "bool_true", + input: true, + expected: `true`, + }, + { + name: "bool_false", + input: false, + expected: `false`, + }, + { + name: "int", + input: 42, + expected: `42`, + }, + { + name: "uint", + input: uint(42), + expected: `42`, + }, + { + name: "float", + input: 3.14, + expected: `3.14`, + }, + { + name: "slice", + input: []int{1, 2, 3}, + expected: `[1,2,3]`, + }, + { + name: "array", + input: [3]int{4, 5, 6}, + expected: `[4,5,6]`, + }, + { + name: "slice_of_struct", + input: []testStruct{{Name: "John", Age: 30, Score: 8.5}, {Name: "Jane", Age: 25, Score: 9.5}}, + expected: `[{"name":"John","age":30,"score":8.50},{"name":"Jane","age":25,"score":9.50}]`, + }, + { + name: "slice_of_struct_pointer", + input: []*testStruct{{Name: "John", Age: 30, Score: 8.5}, {Name: "Jane", Age: 25, Score: 9.5}}, + expected: `[{"name":"John","age":30,"score":8.50},{"name":"Jane","age":25,"score":9.50}]`, + }, + { + name: "slice_of_map", + input: []map[string]any{{"key1": "value1"}, {"key2": "value2"}}, + expected: `[{"key1":"value1"},{"key2":"value2"}]`, + }, + { + name: "struct", + input: testStruct{Name: "John", Age: 30, Score: 8.5}, + expected: `{"name":"John","age":30,"score":8.50}`, + }, + { + name: "struct_pointer", + input: &testStruct{Name: "Jane", Age: 25, Score: 9.5}, + expected: `{"name":"Jane","age":25,"score":9.50}`, + }, + { + name: "byte_slice", + input: []byte("test"), + expected: `"dGVzdA=="`, + }, + { + name: "custom_marshaler", + input: customMarshaler{Value: "test"}, + expected: `{"custom":"test"}`, + }, + { + name: "custom_marshaler_pointer", + input: &customMarshaler{Value: "test"}, + expected: `{"custom":"test"}`, + }, + { + name: "json_marshaler", + input: jsonMarshaler{Value: "test"}, + expected: `{"json_marshaler":"test"}`, + }, + { + name: "json_marshaler_pointer", + input: &jsonMarshaler{Value: "test"}, + expected: `{"json_marshaler":"test"}`, + }, + { + name: "stringer", + input: stringer{testStruct: testStruct{Name: "Bob", Age: 20, Score: 9.5}}, + expected: `"Bob"`, + }, + { + name: "stringer_pointer", + input: &stringer{testStruct: testStruct{Name: "Bob", Age: 20, Score: 9.5}}, + expected: `"Bob"`, + }, + { + name: "with_json_tag", + input: withJSONTag{Value: "test"}, + expected: `{"value":"test"}`, + }, + { + name: "with_json_tag_pointer", + input: &withJSONTag{Value: "test"}, + expected: `{"value":"test"}`, + }, + { + name: "with_json_omit_empty", + input: withJSONOmitEmpty{Value: "test"}, + expected: `{"value":"test"}`, + }, + { + name: "with_json_omit_empty_pointer", + input: &withJSONOmitEmpty{Value: "test"}, + expected: `{"value":"test"}`, + }, + { + name: "with_json_omit_empty_empty", + input: withJSONOmitEmpty{}, + expected: `{}`, + }, + { + name: "with_json_omit_empty_pointer_empty", + input: &withJSONOmitEmpty{}, + expected: `{}`, + }, + { + name: "with_json_omit", + input: withJSONOmit{Value: "test"}, + expected: `{}`, + }, + { + name: "with_json_omit_pointer", + input: &withJSONOmit{Value: "test"}, + expected: `{}`, + }, + { + name: "with_json_string_tag", + input: withJSONStringTag{Value: 1234567890}, + expected: `{"value":"1234567890"}`, + }, + { + name: "with_json_string_tag_pointer", + input: &withJSONStringTag{Value: 1234567890}, + expected: `{"value":"1234567890"}`, + }, + { + name: "with_json_byte_size", + input: withJSONByteSize{Value: 1024}, + expected: fmt.Sprintf(`{"value":"%s"}`, strutils.FormatByteSize(1024)), + }, + { + name: "with_json_byte_size_pointer", + input: &withJSONByteSize{Value: 1024}, + expected: fmt.Sprintf(`{"value":"%s"}`, strutils.FormatByteSize(1024)), + }, + { + name: "with_json_unix_time", + input: withJSONUnixTime{Value: 1713033600}, + expected: fmt.Sprintf(`{"value":"%s"}`, strutils.FormatUnixTime(1713033600)), + }, + { + name: "with_json_unix_time_pointer", + input: &withJSONUnixTime{Value: 1713033600}, + expected: fmt.Sprintf(`{"value":"%s"}`, strutils.FormatUnixTime(1713033600)), + }, + { + name: "with_tag_use_marshaler", + input: withTagUseMarshaler{Value: primitiveWithMarshaler(42)}, + expected: `{"value":"42"}`, + }, + { + name: "with_tag_use_marshaler_pointer", + input: &withTagUseMarshaler{Value: primitiveWithMarshaler(42)}, + expected: `{"value":"42"}`, + }, + { + name: "with_anonymous", + input: withAnonymous{Anonymous: Anonymous{Value: "test", Value2: 1}}, + expected: `{"value":"test","value2":1}`, + }, + { + name: "with_anonymous_pointer", + input: &withAnonymous{Anonymous: Anonymous{Value: "test", Value2: 1}}, + expected: `{"value":"test","value2":1}`, + }, + { + name: "with_pointer_anonymous", + input: &withPointerAnonymous{Anonymous: &Anonymous{Value: "test", Value2: 1}}, + expected: `{"value":"test","value2":1}`, + }, + { + name: "with_pointer_anonymous_nil", + input: &withPointerAnonymous{Anonymous: nil}, + expected: `{}`, + }, + { + // NOTE: not fixing this until needed + // GoDoxy does not have any type with exported self-referencing fields + name: "self_referencing", + input: func() *selfReferencing { + s := &selfReferencing{} + s.Self = s + return s + }(), + expected: `{"self":{"self":{"self":{"self":null}}}}`, + }, + { + name: "nil", + input: nil, + expected: `null`, + }, + { + name: "nil_pointer", + input: (*int)(nil), + expected: `null`, + }, + { + name: "nil_slice", + input: []int(nil), + expected: `[]`, + }, + { + name: "nil_map", + input: map[string]int(nil), + expected: `{}`, + }, + { + name: "nil_map_pointer", + input: (*map[string]int)(nil), + expected: `null`, + }, + { + name: "nil_slice_pointer", + input: (*[]int)(nil), + expected: `null`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, _ := Marshal(tt.input) + require.Equal(t, tt.expected, string(result)) + }) + } + + mapTests := []struct { + name string + input any + }{ + { + name: "map", + input: map[string]int{"one": 1, "two": 2}, + }, + { + name: "map_of_struct", + input: map[string]testStruct{"one": {Name: "John", Age: 30, Score: 8.5}, "two": {Name: "Jane", Age: 25, Score: 9.5}}, + }, + { + name: "complex_map", + input: testData, + }, + } + + for _, tt := range mapTests { + t.Run(tt.name, func(t *testing.T) { + result, _ := Marshal(tt.input) + verify := reflect.MakeMap(reflect.TypeOf(tt.input)) + if err := stdJSON.Unmarshal(result, &verify); err != nil { + t.Fatalf("Unmarshal(%v) error: %v", result, err) + } + iter := verify.MapRange() + for iter.Next() { + k := iter.Key() + v := iter.Value() + vv := reflect.ValueOf(tt.input).MapIndex(k).Interface() + if !v.Equal(reflect.ValueOf(vv)) { + t.Errorf("Marshal([%s]) = %v, want %v", k, v, vv) + } + } + }) + } +} + +func TestMapAndMapSlice(t *testing.T) { + tests := []struct { + name string + input any + expected string + }{ + { + name: "Map", + input: Map[string]{"key1": "value1", "key2": "value2"}, + expected: `{"key1":"value1","key2":"value2"}`, + }, + { + name: "MapSlice", + input: MapSlice[string]{{"key1": "value1"}, {"key2": "value2"}}, + expected: `[{"key1":"value1"},{"key2":"value2"}]`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, _ := Marshal(tt.input) + if string(result) != tt.expected { + t.Errorf("Marshal(%v) = %s, want %s", tt.input, string(result), tt.expected) + } + }) + } +} + +func TestMarshalSyntacticEquivalence(t *testing.T) { + testData := []any{ + "test\r\nstring", + 42, + 3.14, + true, + nil, + []int{1, 2, 3, 4, 5}, + map[string]any{ + "nested": "value", + "count": 10, + "bytes": []byte("test"), + "a": "a\x1b[31m", + }, + testStruct{Name: "Test", Age: 30, Score: 9.8}, + } + + for i, data := range testData { + custom, _ := Marshal(data) + stdlib, err := stdJSON.Marshal(data) + if err != nil { + t.Fatalf("Test %d: Standard Marshal error: %v", i, err) + } + + t.Logf("custom: %s\n", custom) + t.Logf("stdlib: %s\n", stdlib) + + // Unmarshal both into maps to compare structure equivalence + var customMap, stdlibMap any + if err := stdJSON.Unmarshal(custom, &customMap); err != nil { + t.Fatalf("Test %d: Unmarshal custom error: %v", i, err) + } + if err := stdJSON.Unmarshal(stdlib, &stdlibMap); err != nil { + t.Fatalf("Test %d: Unmarshal stdlib error: %v", i, err) + } + + if !reflect.DeepEqual(customMap, stdlibMap) { + t.Errorf("Test %d: Marshal output not equivalent.\nCustom: %s\nStdLib: %s", + i, string(custom), string(stdlib)) + } + } +} + +func BenchmarkMarshalNoStructStdLib(b *testing.B) { + b.Run("StdLib", func(b *testing.B) { + for b.Loop() { + _, _ = stdJSON.Marshal(testData) + } + }) + + b.Run("Sonic", func(b *testing.B) { + for b.Loop() { + _, _ = sonic.Marshal(testData) + } + }) + + b.Run("Custom", func(b *testing.B) { + for b.Loop() { + _, _ = Marshal(testData) + } + }) +} + +func BenchmarkMarshalStruct(b *testing.B) { + withStruct := maps.Clone(testData) + withStruct["struct1"] = withAnonymous{Anonymous: Anonymous{Value: "one", Value2: 1}} + withStruct["struct2"] = &withPointerAnonymous{Anonymous: &Anonymous{Value: "two", Value2: 2}} + withStruct["struct3"] = &testStruct{Name: "three", Age: 30, Score: 9.8} + b.ResetTimer() + + b.Run("StdLib", func(b *testing.B) { + for b.Loop() { + _, _ = stdJSON.Marshal(withStruct) + } + }) + + b.Run("Sonic", func(b *testing.B) { + for b.Loop() { + _, _ = sonic.Marshal(withStruct) + } + }) + + b.Run("Custom", func(b *testing.B) { + for b.Loop() { + _, _ = Marshal(withStruct) + } + }) +} diff --git a/pkg/json/special.go b/pkg/json/special.go new file mode 100644 index 0000000..acf43cd --- /dev/null +++ b/pkg/json/special.go @@ -0,0 +1,60 @@ +package json + +import ( + "encoding" + "fmt" + "reflect" + "time" + + "github.com/yusing/go-proxy/internal/utils/strutils" +) + +func isIntFloat(t reflect.Kind) bool { + return t >= reflect.Bool && t <= reflect.Float64 +} + +func appendStringRepr(v reflect.Value, buf []byte) []byte { // for json tag `string` + kind := v.Kind() + if isIntFloat(kind) { + marshalFunc, _ := marshalFuncByKind[kind] + buf = append(buf, '"') + buf = marshalFunc(v, buf) + buf = append(buf, '"') + return buf + } + switch vv := v.Interface().(type) { + case fmt.Stringer: + buf = AppendString(buf, vv.String()) + case encoding.TextMarshaler: + buf = append(buf, must(vv.MarshalText())...) + case encoding.TextAppender: + buf = must(vv.AppendText(buf)) + default: + panic(fmt.Errorf("tag %q used but type is non-stringable: %s", tagString, v.Type())) + } + return buf +} + +func appendTime(v reflect.Value, buf []byte) []byte { + buf = append(buf, '"') + buf = strutils.AppendTime(v.Interface().(time.Time), buf) + return append(buf, '"') +} + +func appendDuration(v reflect.Value, buf []byte) []byte { + buf = append(buf, '"') + buf = strutils.AppendDuration(v.Interface().(time.Duration), buf) + return append(buf, '"') +} + +func appendByteSize(v reflect.Value, buf []byte) []byte { + buf = append(buf, '"') + buf = strutils.AppendByteSize(v.Interface().(uint64), buf) + return append(buf, '"') +} + +func appendUnixTime(v reflect.Value, buf []byte) []byte { + buf = append(buf, '"') + buf = strutils.AppendTime(time.Unix(v.Interface().(int64), 0), buf) + return append(buf, '"') +} diff --git a/pkg/json/string.go b/pkg/json/string.go new file mode 100644 index 0000000..30d6b30 --- /dev/null +++ b/pkg/json/string.go @@ -0,0 +1,334 @@ +package json + +import ( + "encoding/base64" + "reflect" + "unicode/utf8" +) + +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// safeSet, htmlSafeSet, hex and AppendString are copied from encoding/json. + +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML