From 3332ce34c52eb339ea6d1cc0509e6d3402b10e40 Mon Sep 17 00:00:00 2001 From: yusing Date: Tue, 11 Feb 2025 05:05:56 +0800 Subject: [PATCH] simplify setup process --- agent.compose.yml | 7 +- agent/cmd/args.go | 16 ---- agent/cmd/main.go | 104 +++++++---------------- agent/pkg/agent/config.go | 23 +++-- agent/pkg/agent/utils.go | 30 +++++++ agent/pkg/certs/zip.go | 1 + agent/pkg/env/env.go | 56 +++++++++++- agent/pkg/handler/handler.go | 52 +++++++++++- agent/pkg/server/server.go | 54 +++++++++--- cmd/add_agent.go | 65 ++++++++++++++ cmd/main.go | 15 +--- cmd/new_agent.go | 46 ---------- go.mod | 2 +- internal/common/args.go | 4 +- internal/common/env.go | 3 +- internal/config/config.go | 3 + internal/error/utils.go | 2 +- internal/logging/memlogger/mem_logger.go | 7 +- internal/net/http/server/error.go | 18 ++++ internal/net/http/server/server.go | 20 ++--- next-release.md | 64 ++++++++++++-- 21 files changed, 386 insertions(+), 206 deletions(-) delete mode 100644 agent/cmd/args.go create mode 100644 agent/pkg/agent/utils.go create mode 100644 cmd/add_agent.go delete mode 100644 cmd/new_agent.go create mode 100644 internal/net/http/server/error.go diff --git a/agent.compose.yml b/agent.compose.yml index 0a65186..ebd83cb 100644 --- a/agent.compose.yml +++ b/agent.compose.yml @@ -5,8 +5,11 @@ services: restart: always network_mode: host # do not change this environment: - GODOXY_AGENT_NAME: # defaults to hostname - GODOXY_AGENT_PORT: # defaults to 8890 + AGENT_NAME: # defaults to hostname + AGENT_PORT: # defaults to 8890 + # comma separated list of allowed main server IPs or CIDRs + # to register from this agent + REGISTRATION_ALLOWED_HOSTS: volumes: - /var/run/docker.sock:/var/run/docker.sock - ./certs:/app/certs # store Agent CA cert and Agent SSL cert diff --git a/agent/cmd/args.go b/agent/cmd/args.go deleted file mode 100644 index de4293f..0000000 --- a/agent/cmd/args.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -const ( - CommandStart = "" - CommandNewClient = "new-client" -) - -type agentCommandValidator struct{} - -func (v agentCommandValidator) IsCommandValid(cmd string) bool { - switch cmd { - case CommandStart, CommandNewClient: - return true - } - return false -} diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 36bd2b2..389ae9a 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -1,38 +1,29 @@ package main import ( - "crypto/tls" - "encoding/base64" - "encoding/pem" "fmt" - "net" - "os" - "strings" - "github.com/rs/zerolog" + "github.com/yusing/go-proxy/agent/pkg/agent" "github.com/yusing/go-proxy/agent/pkg/certs" "github.com/yusing/go-proxy/agent/pkg/env" "github.com/yusing/go-proxy/agent/pkg/server" E "github.com/yusing/go-proxy/internal/error" "github.com/yusing/go-proxy/internal/logging" - "github.com/yusing/go-proxy/internal/logging/memlogger" "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/utils" "github.com/yusing/go-proxy/pkg" "gopkg.in/yaml.v3" ) -func init() { - logging.InitLogger(zerolog.MultiLevelWriter(os.Stderr, memlogger.GetMemLogger())) -} - -func printNewClientHelp(ca *tls.Certificate) { - crt, key, err := certs.NewClientCert(ca) - if err != nil { - E.LogFatal("init SSL error", err) +func printNewClientHelp() { + ip, ok := agent.MachineIP() + if !ok { + logging.Warn().Msg("No valid network interface found, change to your actual IP") + ip = "" + } else { + logging.Info().Msgf("Detected machine IP: %s, change if needed", ip) } - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Certificate[0]}) - ip := machineIP() + host := fmt.Sprintf("%s:%d", ip, env.AgentPort) cfgYAML, _ := yaml.Marshal(map[string]any{ "providers": map[string]any{ @@ -40,78 +31,43 @@ func printNewClientHelp(ca *tls.Certificate) { }, }) - certsData, err := certs.ZipCert(caPEM, crt, key) - if err != nil { - E.LogFatal("marshal certs error", err) - } - - fmt.Printf("On main server, run:\nnew-agent '%s' '%s'\n", host, base64.StdEncoding.EncodeToString(certsData)) - fmt.Printf("Then add this host (%s) to main server config like below:\n", host) - fmt.Println(string(cfgYAML)) -} - -func machineIP() string { - interfaces, err := net.Interfaces() - if err != nil { - return "" - } - for _, in := range interfaces { - addrs, err := in.Addrs() - if err != nil { - continue - } - if !strings.HasPrefix(in.Name, "eth") && !strings.HasPrefix(in.Name, "en") { - continue - } - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - return ipnet.IP.String() - } - } - } - } - return "" + logging.Info().Msgf("On main server, run:\n\ndocker exec godoxy /app/run add-agent '%s'\n\n", host) + logging.Info().Msgf("Then add this host (%s) to main server config like below:\n\n", host) + logging.Info().Msg(string(cfgYAML)) } func main() { - args := pkg.GetArgs(agentCommandValidator{}) - ca, srv, isNew, err := certs.InitCerts() if err != nil { E.LogFatal("init CA error", err) } - if args.Command == CommandNewClient { - printNewClientHelp(ca) - return - } - logging.Info().Msgf("GoDoxy Agent version %s", pkg.GetVersion()) logging.Info().Msgf("Agent name: %s", env.AgentName) logging.Info().Msgf("Agent port: %d", env.AgentPort) - logging.Info().Msg("\nTips:") - logging.Info().Msg("1. To change the agent name, you can set the AGENT_NAME environment variable.") - logging.Info().Msg("2. To change the agent port, you can set the AGENT_PORT environment variable.") - logging.Info().Msg("3. To skip the version check, you can set the AGENT_SKIP_VERSION_CHECK environment variable.") - logging.Info().Msgf("4. Create shell alias on main server: `alias new-agent='docker run --rm -v ./certs:/app/certs ghcr.io/yusing/godoxy /app/run new-agent'`") - logging.Info().Msgf("5. Create shell alias on agent server: `alias new-client='docker compose exec agent /app/run new-client'`\n") + logging.Info().Msg(` +Tips: +1. To change the agent name, you can set the AGENT_NAME environment variable. +2. To change the agent port, you can set the AGENT_PORT environment variable. +3. To skip the version check, you can set AGENT_SKIP_VERSION_CHECK to true. +4. If anything goes wrong, you can remove the 'certs' directory and start over. +`) - if isNew { - logging.Info().Msg("Initialization complete.") - logging.Info().Msg("New client cert created") - printNewClientHelp(ca) - logging.Info().Msg("Exiting... Clear the screen and start agent again") - logging.Info().Msg("To create more client certs, run `godoxy-agent new-client`") - return - } - - server.StartAgentServer(task.RootTask("agent", false), server.Options{ + t := task.RootTask("agent", false) + opts := server.Options{ CACert: ca, ServerCert: srv, Port: env.AgentPort, - }) + } + + if isNew { + logging.Info().Msg("Initialization complete.") + printNewClientHelp() + server.StartRegistrationServer(t, opts) + } + + server.StartAgentServer(t, opts) utils.WaitExit(3) } diff --git a/agent/pkg/agent/config.go b/agent/pkg/agent/config.go index 54d2952..fc4da26 100644 --- a/agent/pkg/agent/config.go +++ b/agent/pkg/agent/config.go @@ -13,7 +13,6 @@ import ( "github.com/rs/zerolog" "github.com/yusing/go-proxy/agent/pkg/certs" - "github.com/yusing/go-proxy/agent/pkg/env" E "github.com/yusing/go-proxy/internal/error" "github.com/yusing/go-proxy/internal/logging" gphttp "github.com/yusing/go-proxy/internal/net/http" @@ -94,6 +93,14 @@ func (cfg *AgentConfig) errIfNameExists() E.Error { return nil } +func withoutBuildTime(version string) string { + return strings.Split(version, "-")[0] +} + +func checkVersion(a, b string) bool { + return withoutBuildTime(a) == withoutBuildTime(b) +} + func (cfg *AgentConfig) load() E.Error { certData, err := os.ReadFile(certs.AgentCertsFilename(cfg.Addr)) if err != nil { @@ -132,15 +139,13 @@ func (cfg *AgentConfig) load() E.Error { defer cancel() // check agent version - if !env.AgentSkipVersionCheck { - version, _, err := cfg.Fetch(ctx, EndpointVersion) - if err != nil { - return E.Wrap(err) - } + version, _, err := cfg.Fetch(ctx, EndpointVersion) + if err != nil { + return E.Wrap(err) + } - if string(version) != pkg.GetVersion() { - return E.Errorf("agent version mismatch: server: %s, agent: %s", pkg.GetVersion(), string(version)) - } + if !checkVersion(string(version), pkg.GetVersion()) { + return E.Errorf("agent version mismatch: server: %s, agent: %s", pkg.GetVersion(), string(version)) } // get agent name diff --git a/agent/pkg/agent/utils.go b/agent/pkg/agent/utils.go new file mode 100644 index 0000000..d6f0305 --- /dev/null +++ b/agent/pkg/agent/utils.go @@ -0,0 +1,30 @@ +package agent + +import ( + "net" + "strings" +) + +func MachineIP() (string, bool) { + interfaces, err := net.Interfaces() + if err != nil { + interfaces = []net.Interface{} + } + for _, in := range interfaces { + addrs, err := in.Addrs() + if err != nil { + continue + } + if !strings.HasPrefix(in.Name, "eth") && !strings.HasPrefix(in.Name, "en") { + continue + } + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String(), true + } + } + } + } + return "", false +} diff --git a/agent/pkg/certs/zip.go b/agent/pkg/certs/zip.go index 89532ac..51faa2b 100644 --- a/agent/pkg/certs/zip.go +++ b/agent/pkg/certs/zip.go @@ -32,6 +32,7 @@ func readFile(f *zip.File) ([]byte, error) { func ZipCert(ca, crt, key []byte) ([]byte, error) { data := bytes.NewBuffer(nil) + data.Grow(6144) zipWriter := zip.NewWriter(data) defer zipWriter.Close() diff --git a/agent/pkg/env/env.go b/agent/pkg/env/env.go index 72c8863..ad9ef53 100644 --- a/agent/pkg/env/env.go +++ b/agent/pkg/env/env.go @@ -1,7 +1,10 @@ package env import ( + "log" + "net" "os" + "strings" "github.com/yusing/go-proxy/internal/common" ) @@ -15,7 +18,54 @@ func DefaultAgentName() string { } var ( - AgentName = common.GetEnvString("AGENT_NAME", DefaultAgentName()) - AgentPort = common.GetEnvInt("AGENT_PORT", 8890) - AgentSkipVersionCheck = common.GetEnvBool("AGENT_SKIP_VERSION_CHECK", false) + AgentName = common.GetEnvString("AGENT_NAME", DefaultAgentName()) + AgentPort = common.GetEnvInt("AGENT_PORT", 8890) + AgentRegistrationPort = common.GetEnvInt("AGENT_REGISTRATION_PORT", 8891) + AgentSkipClientCertCheck = common.GetEnvBool("AGENT_SKIP_CLIENT_CERT_CHECK", false) + + RegistrationAllowedHosts = common.GetCommaSepEnv("REGISTRATION_ALLOWED_HOSTS", "") + RegistrationAllowedCIDRs []*net.IPNet ) + +func init() { + cidrs, err := toCIDRs(RegistrationAllowedHosts) + if err != nil { + log.Fatalf("failed to parse allowed hosts: %v", err) + } + if len(cidrs) == 0 { + log.Fatal("REGISTRATION_ALLOWED_HOSTS is empty") + } + RegistrationAllowedCIDRs = cidrs +} + +func toCIDRs(hosts []string) ([]*net.IPNet, error) { + var cidrs []*net.IPNet + for _, host := range hosts { + if !strings.Contains(host, "/") { + host += "/32" + } + _, cidr, err := net.ParseCIDR(host) + if err != nil { + return nil, err + } + cidrs = append(cidrs, cidr) + } + return cidrs, nil +} + +func IsAllowedHost(remoteAddr string) bool { + ip, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + ip = remoteAddr + } + netIP := net.ParseIP(ip) + if netIP == nil { + return false + } + for _, cidr := range RegistrationAllowedCIDRs { + if cidr.Contains(netIP) { + return true + } + } + return false +} diff --git a/agent/pkg/handler/handler.go b/agent/pkg/handler/handler.go index c61371b..adbea9c 100644 --- a/agent/pkg/handler/handler.go +++ b/agent/pkg/handler/handler.go @@ -1,14 +1,21 @@ package handler import ( + "crypto/tls" + "encoding/pem" "fmt" "io" "net/http" "github.com/yusing/go-proxy/agent/pkg/agent" + "github.com/yusing/go-proxy/agent/pkg/certs" "github.com/yusing/go-proxy/agent/pkg/env" v1 "github.com/yusing/go-proxy/internal/api/v1" + "github.com/yusing/go-proxy/internal/api/v1/utils" + E "github.com/yusing/go-proxy/internal/error" + "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/logging/memlogger" + "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/utils/strutils" ) @@ -32,7 +39,7 @@ func (NopWriteCloser) Close() error { return nil } -func NewHandler() http.Handler { +func NewAgentHandler() http.Handler { mux := ServeMux{http.NewServeMux()} mux.HandleFunc(agent.EndpointProxyHTTP+"/{path...}", ProxyHTTP) @@ -46,3 +53,46 @@ func NewHandler() http.Handler { mux.ServeMux.HandleFunc("/", DockerSocketHandler()) return mux } + +// NewRegistrationHandler creates a new registration handler +// It checks if the request is coming from an allowed host +// Generates a new client certificate and zips it +// Sends the zipped certificate to the client +// its run only once on agent first start. +func NewRegistrationHandler(task *task.Task, ca *tls.Certificate) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !env.IsAllowedHost(r.RemoteAddr) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + if r.URL.Path == "/done" { + logging.Info().Msg("registration done") + task.Finish(nil) + w.WriteHeader(http.StatusOK) + return + } + + logging.Info().Msgf("received registration request from %s", r.RemoteAddr) + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Certificate[0]}) + + crt, key, err := certs.NewClientCert(ca) + if err != nil { + utils.HandleErr(w, r, E.Wrap(err, "failed to generate client certificate")) + return + } + + zipped, err := certs.ZipCert(caPEM, crt, key) + if err != nil { + utils.HandleErr(w, r, E.Wrap(err, "failed to zip certificate")) + return + } + + w.Header().Set("Content-Type", "application/zip") + if _, err := w.Write(zipped); err != nil { + logging.Error().Err(err).Msg("failed to respond to registration request") + return + } + } +} diff --git a/agent/pkg/server/server.go b/agent/pkg/server/server.go index d67be41..369887f 100644 --- a/agent/pkg/server/server.go +++ b/agent/pkg/server/server.go @@ -11,9 +11,10 @@ import ( "net/http" "time" + "github.com/yusing/go-proxy/agent/pkg/env" "github.com/yusing/go-proxy/agent/pkg/handler" - "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/logging" + "github.com/yusing/go-proxy/internal/net/http/server" "github.com/yusing/go-proxy/internal/task" ) @@ -23,7 +24,7 @@ type Options struct { } func StartAgentServer(parent task.Parent, opt Options) { - t := parent.Subtask("agent server") + t := parent.Subtask("agent_server") caCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: opt.CACert.Certificate[0]}) caCertPool := x509.NewCertPool() @@ -36,23 +37,24 @@ func StartAgentServer(parent task.Parent, opt Options) { ClientAuth: tls.RequireAndVerifyClientCert, } - if common.IsDebug { + if env.AgentSkipClientCertCheck { tlsConfig.ClientAuth = tls.NoClientCert } - l, err := net.Listen("tcp", fmt.Sprintf(":%d", opt.Port)) - if err != nil { - logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to listen on port") - return - } - server := &http.Server{ - Handler: handler.NewHandler(), + agentServer := &http.Server{ + Handler: handler.NewAgentHandler(), TLSConfig: tlsConfig, ErrorLog: log.New(logging.GetLogger(), "", 0), } + go func() { + l, err := net.Listen("tcp", fmt.Sprintf(":%d", opt.Port)) + if err != nil { + logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to listen on port") + return + } defer l.Close() - if err := server.Serve(tls.NewListener(l, tlsConfig)); err != nil { + if err := agentServer.Serve(tls.NewListener(l, tlsConfig)); err != nil { logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to serve") } }() @@ -66,10 +68,38 @@ func StartAgentServer(parent task.Parent, opt Options) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - err := server.Shutdown(ctx) + err := agentServer.Shutdown(ctx) if err != nil { logging.Error().Err(err).Int("port", opt.Port).Msg("failed to shutdown agent server") } logging.Info().Int("port", opt.Port).Msg("agent server stopped") }() } + +func StartRegistrationServer(parent task.Parent, opt Options) { + t := parent.Subtask("registration_server") + + registrationServer := &http.Server{ + Addr: fmt.Sprintf(":%d", opt.Port), + Handler: handler.NewRegistrationHandler(t, opt.CACert), + ErrorLog: log.New(logging.GetLogger(), "", 0), + } + + go func() { + err := registrationServer.ListenAndServe() + server.HandleError(logging.GetLogger(), err) + }() + + logging.Info().Int("port", opt.Port).Msg("registration server started") + + defer t.Finish(nil) + <-t.Context().Done() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + err := registrationServer.Shutdown(ctx) + server.HandleError(logging.GetLogger(), err) + + logging.Info().Int("port", opt.Port).Msg("registration server stopped") +} diff --git a/cmd/add_agent.go b/cmd/add_agent.go new file mode 100644 index 0000000..5668025 --- /dev/null +++ b/cmd/add_agent.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "io" + "net/http" + "os" + "time" + + "github.com/yusing/go-proxy/agent/pkg/certs" + "github.com/yusing/go-proxy/internal/logging" +) + +func AddAgent(args []string) { + if len(args) != 1 { + logging.Fatal().Msgf("invalid arguments: %v, expect host", args) + } + host := args[0] + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", "http://"+host, nil) + if err != nil { + logging.Fatal().Err(err).Msg("failed to create request") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + logging.Fatal().Err(err).Msg("failed to send request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logging.Fatal().Int("status", resp.StatusCode).Msg("failed to add agent") + } + + zip, err := io.ReadAll(resp.Body) + if err != nil { + logging.Fatal().Err(err).Msg("failed to read response body") + } + + f, err := os.OpenFile(certs.AgentCertsFilename(host), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + logging.Fatal().Err(err).Msg("failed to create client certs file") + } + defer f.Close() + + if _, err := f.Write(zip); err != nil { + logging.Fatal().Err(err).Msg("failed to save client certs") + } + + logging.Info().Msgf("agent %s added, certs saved to %s", host, certs.AgentCertsFilename(host)) + + req, err = http.NewRequestWithContext(ctx, "GET", "http://"+host+"/done", nil) + if err != nil { + logging.Fatal().Err(err).Msg("failed to create done request") + } + + resp, err = http.DefaultClient.Do(req) + if err != nil { + logging.Fatal().Err(err).Msg("failed to send done request") + } + defer resp.Body.Close() +} diff --git a/cmd/main.go b/cmd/main.go index 3ed9ca8..8d1cf42 100755 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,11 +2,9 @@ package main import ( "encoding/json" - "io" "log" "os" - "github.com/rs/zerolog" "github.com/yusing/go-proxy/internal" "github.com/yusing/go-proxy/internal/api/v1/auth" "github.com/yusing/go-proxy/internal/api/v1/favicon" @@ -16,7 +14,6 @@ import ( E "github.com/yusing/go-proxy/internal/error" "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/logging" - "github.com/yusing/go-proxy/internal/logging/memlogger" "github.com/yusing/go-proxy/internal/net/http/middleware" "github.com/yusing/go-proxy/internal/route/routes/routequery" "github.com/yusing/go-proxy/internal/utils" @@ -25,14 +22,6 @@ import ( var rawLogger = log.New(os.Stdout, "", 0) -func init() { - var out io.Writer = os.Stderr - if common.EnableLogStreaming { - out = zerolog.MultiLevelWriter(out, memlogger.GetMemLogger()) - } - logging.InitLogger(out) -} - func main() { initProfiling() args := pkg.GetArgs(common.MainServerCommandValidator{}) @@ -41,8 +30,8 @@ func main() { case common.CommandSetup: Setup() return - case common.CommandNewAgent: - NewAgent(args.Args) + case common.CommandAddAgent: + AddAgent(args.Args) return case common.CommandReload: if err := query.ReloadServer(); err != nil { diff --git a/cmd/new_agent.go b/cmd/new_agent.go deleted file mode 100644 index 05c9afe..0000000 --- a/cmd/new_agent.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "encoding/base64" - "log" - "net" - "os" - - "github.com/yusing/go-proxy/agent/pkg/certs" -) - -func NewAgent(args []string) { - if len(args) != 2 { - log.Fatalf("invalid arguments: %v", args) - } - host := args[0] - certDataBase64 := args[1] - - ip, _, err := net.SplitHostPort(host) - if err != nil { - log.Fatalf("invalid host: %v", err) - } - - _, err = net.ResolveIPAddr("ip", ip) - if err != nil { - log.Fatalf("invalid host: %v", err) - } - - certData, err := base64.StdEncoding.DecodeString(certDataBase64) - if err != nil { - log.Fatalf("invalid cert data: %v", err) - } - - f, err := os.OpenFile(certs.AgentCertsFilename(host), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - log.Fatalf("failed to create file: %v", err) - } - defer f.Close() - - _, err = f.Write(certData) - if err != nil { - log.Fatalf("failed to write cert data: %v", err) - } - - log.Printf("agent cert created: %s", certs.AgentCertsFilename(host)) -} diff --git a/go.mod b/go.mod index 3a56ea0..12a7e67 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/puzpuzpuz/xsync/v3 v3.5.0 github.com/rs/zerolog v1.33.0 + github.com/shirou/gopsutil/v4 v4.25.1 github.com/vincent-petithory/dataurl v1.0.0 golang.org/x/crypto v0.33.0 golang.org/x/net v0.35.0 @@ -68,7 +69,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect diff --git a/internal/common/args.go b/internal/common/args.go index cd825fe..824118b 100644 --- a/internal/common/args.go +++ b/internal/common/args.go @@ -3,7 +3,7 @@ package common const ( CommandStart = "" CommandSetup = "setup" - CommandNewAgent = "new-agent" + CommandAddAgent = "add-agent" CommandValidate = "validate" CommandListConfigs = "ls-config" CommandListRoutes = "ls-routes" @@ -20,7 +20,7 @@ func (v MainServerCommandValidator) IsCommandValid(cmd string) bool { switch cmd { case CommandStart, CommandSetup, - CommandNewAgent, + CommandAddAgent, CommandValidate, CommandListConfigs, CommandListRoutes, diff --git a/internal/common/env.go b/internal/common/env.go index a62f861..a3f8bba 100644 --- a/internal/common/env.go +++ b/internal/common/env.go @@ -20,8 +20,7 @@ var ( IsTrace = GetEnvBool("TRACE", false) && IsDebug IsProduction = !IsTest && !IsDebug - EnableLogStreaming = GetEnvBool("LOG_STREAMING", true) - DebugMemLogger = GetEnvBool("DEBUG_MEM_LOGGER", false) && EnableLogStreaming + DebugMemLogger = GetEnvBool("DEBUG_MEM_LOGGER", false) ProxyHTTPAddr, ProxyHTTPHost, diff --git a/internal/config/config.go b/internal/config/config.go index bfc72d1..7ae78c4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -290,6 +290,9 @@ func (cfg *Config) loadRouteProviders(providers *types.Providers) E.Error { for _, agent := range providers.Agents { cfg.providers.Store(agent.Name(), proxy.NewAgentProvider(&agent)) } + if cfg.providers.Size() == 0 { + return nil + } cfg.providers.RangeAllParallel(func(_ string, p *proxy.Provider) { if err := p.LoadRoutes(); err != nil { errs.Add(err.Subject(p.String())) diff --git a/internal/error/utils.go b/internal/error/utils.go index db346cb..8e36f83 100644 --- a/internal/error/utils.go +++ b/internal/error/utils.go @@ -23,7 +23,7 @@ func Wrap(err error, message ...string) Error { if len(message) == 0 || message[0] == "" { return From(err) } - return Errorf("%w: %s", err, message[0]) + return Errorf("%s: %w", message[0], err) } func From(err error) Error { diff --git a/internal/logging/memlogger/mem_logger.go b/internal/logging/memlogger/mem_logger.go index 8647e62..eeaacca 100644 --- a/internal/logging/memlogger/mem_logger.go +++ b/internal/logging/memlogger/mem_logger.go @@ -5,10 +5,12 @@ import ( "context" "io" "net/http" + "os" "sync" "time" "github.com/coder/websocket" + "github.com/rs/zerolog" "github.com/yusing/go-proxy/internal/api/v1/utils" "github.com/yusing/go-proxy/internal/common" config "github.com/yusing/go-proxy/internal/config/types" @@ -55,9 +57,6 @@ var memLoggerInstance = &memLogger{ } func init() { - if !common.EnableLogStreaming { - return - } memLoggerInstance.Grow(maxMemLogSize) if common.DebugMemLogger { @@ -78,6 +77,8 @@ func init() { } }() } + + logging.InitLogger(zerolog.MultiLevelWriter(os.Stderr, memLoggerInstance)) } func LogsWS(config config.ConfigInstance) http.HandlerFunc { diff --git a/internal/net/http/server/error.go b/internal/net/http/server/error.go new file mode 100644 index 0000000..3b43061 --- /dev/null +++ b/internal/net/http/server/error.go @@ -0,0 +1,18 @@ +package server + +import ( + "context" + "errors" + "net/http" + + "github.com/rs/zerolog" +) + +func HandleError(logger *zerolog.Logger, err error) { + switch { + case err == nil, errors.Is(err, http.ErrServerClosed), errors.Is(err, context.Canceled): + return + default: + logger.Fatal().Err(err).Msg("server error") + } +} diff --git a/internal/net/http/server/server.go b/internal/net/http/server/server.go index d0e48ab..fee526e 100644 --- a/internal/net/http/server/server.go +++ b/internal/net/http/server/server.go @@ -3,7 +3,6 @@ package server import ( "context" "crypto/tls" - "errors" "io" "log" "net" @@ -100,7 +99,7 @@ func (s *Server) Start(parent task.Parent) { s.startTime = time.Now() if s.http != nil { go func() { - s.handleErr("http", s.http.ListenAndServe()) + s.handleErr(s.http.ListenAndServe()) }() s.httpStarted = true s.l.Info().Str("addr", s.http.Addr).Msg("server started") @@ -110,11 +109,11 @@ func (s *Server) Start(parent task.Parent) { go func() { l, err := net.Listen("tcp", s.https.Addr) if err != nil { - s.handleErr("https", err) + s.handleErr(err) return } defer l.Close() - s.handleErr("https", s.https.Serve(tls.NewListener(l, s.https.TLSConfig))) + s.handleErr(s.https.Serve(tls.NewListener(l, s.https.TLSConfig))) }() s.httpsStarted = true s.l.Info().Str("addr", s.https.Addr).Msgf("server started") @@ -132,13 +131,13 @@ func (s *Server) stop() { defer cancel() if s.http != nil && s.httpStarted { - s.handleErr("http", s.http.Shutdown(ctx)) + s.handleErr(s.http.Shutdown(ctx)) s.httpStarted = false s.l.Info().Str("addr", s.http.Addr).Msgf("server stopped") } if s.https != nil && s.httpsStarted { - s.handleErr("https", s.https.Shutdown(ctx)) + s.handleErr(s.https.Shutdown(ctx)) s.httpsStarted = false s.l.Info().Str("addr", s.https.Addr).Msgf("server stopped") } @@ -148,11 +147,6 @@ func (s *Server) Uptime() time.Duration { return time.Since(s.startTime) } -func (s *Server) handleErr(scheme string, err error) { - switch { - case err == nil, errors.Is(err, http.ErrServerClosed), errors.Is(err, context.Canceled): - return - default: - s.l.Fatal().Err(err).Str("scheme", scheme).Msg("server error") - } +func (s *Server) handleErr(err error) { + HandleError(&s.l, err) } diff --git a/next-release.md b/next-release.md index 7d48601..4c8569e 100644 --- a/next-release.md +++ b/next-release.md @@ -1,8 +1,8 @@ ## GoDoxy v0.10.0 -### GoDoxy-Agent +### GoDoxy Agent -listen only on Agent API server, authenticate and encrypt connection with mTLS. Maintain secure connection between GoDoxy main and GoDoxy agent server +Maintain secure connection between main server and agent server by authenticating and encrypting connection with mTLS. Main benefits: @@ -20,9 +20,16 @@ Main benefits: #### How to setup -1. Agent server generates CA cert, SSL certificate and Client certificate on first run. -2. Follow the output on screen to run `godoxy new-agent : ...` on GoDoxy main server to store generated certs -3. Add config output to GoDoxy main server in `config.yml` under `providers.agents` +Prerequisites: + +- GoDoxy main server must be running + +1. Create a directory for agent server, cd into it +2. Copy `agent.compose.yml` into the directory +3. Modify `agent.compose.yml` to set `REGISTRATION_ALLOWED_HOSTS` +4. Run `docker-compose up -d` to start agent +5. Follow instructions on screen to run command on GoDoxy main server +6. Add config output to GoDoxy main server in `config.yml` under `providers.agents` ```yaml providers: agents: @@ -31,6 +38,47 @@ Main benefits: ### How does it work -1. Main server and agent server negotiate mTLS -2. Agent server verify main server's client cert and check if server version matches agent version -3. Agent server now acts as a http proxy and docker socket proxy +Setup flow: + +```mermaid +flowchart TD + subgraph Agent Server + A[Create a directory] --> + B[Setup agent.compose.yml] --> + C[Set REGISTRATION_ALLOWED_HOSTS] --> + D[Run agent] --> + E[Wait for main server to register] + + F[Respond to main server] + G[Agent now run in agent mode] + end + subgraph Main Server + E --> + H[Run register command] --> + I[Send registration request] --> F --> + J[Store client certs] --> + K[Send done request] --> G --> + L[Add agent to config.yml] + end +``` + +Run flow: + +```mermaid +flowchart TD + subgraph Agent HTTPS Server + aa[Load CA and SSL certs] --> + ab[Start HTTPS server] --> + + ac[Receive request] --> + ad[Verify client cert] --> + ae[Handle request] --> ac + end + subgraph Main Server + ma[Load client certs] --> + mb[Query agent version] --> ac + mb --> mc[Check if agent version matches] --> + md[Query agent info] --> ac + md --> ae --> me[Store agent info] + end +```