simplify setup process

agent.compose.yml

@@ -5,8 +5,11 @@ services:
     restart: always
     network_mode: host # do not change this
     environment:
-      GODOXY_AGENT_NAME: # defaults to hostname
+      AGENT_NAME: # defaults to hostname
-      GODOXY_AGENT_PORT: # defaults to 8890
+      AGENT_PORT: # defaults to 8890
+      # comma separated list of allowed main server IPs or CIDRs
+      # to register from this agent
+      REGISTRATION_ALLOWED_HOSTS:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./certs:/app/certs # store Agent CA cert and Agent SSL cert

agent/cmd/args.go

@@ -1,16 +0,0 @@
-package main
-
-const (
-	CommandStart     = ""
-	CommandNewClient = "new-client"
-)
-
-type agentCommandValidator struct{}
-
-func (v agentCommandValidator) IsCommandValid(cmd string) bool {
-	switch cmd {
-	case CommandStart, CommandNewClient:
-		return true
-	}
-	return false
-}

agent/cmd/main.go

@@ -1,38 +1,29 @@
 package main
 
 import (
-	"crypto/tls"
-	"encoding/base64"
-	"encoding/pem"
 	"fmt"
-	"net"
-	"os"
-	"strings"
 
-	"github.com/rs/zerolog"
+	"github.com/yusing/go-proxy/agent/pkg/agent"
 	"github.com/yusing/go-proxy/agent/pkg/certs"
 	"github.com/yusing/go-proxy/agent/pkg/env"
 	"github.com/yusing/go-proxy/agent/pkg/server"
 	E "github.com/yusing/go-proxy/internal/error"
 	"github.com/yusing/go-proxy/internal/logging"
-	"github.com/yusing/go-proxy/internal/logging/memlogger"
 	"github.com/yusing/go-proxy/internal/task"
 	"github.com/yusing/go-proxy/internal/utils"
 	"github.com/yusing/go-proxy/pkg"
 	"gopkg.in/yaml.v3"
 )
 
-func init() {
+func printNewClientHelp() {
-	logging.InitLogger(zerolog.MultiLevelWriter(os.Stderr, memlogger.GetMemLogger()))
+	ip, ok := agent.MachineIP()
-}
+	if !ok {
-
+		logging.Warn().Msg("No valid network interface found, change <machine-ip> to your actual IP")
-func printNewClientHelp(ca *tls.Certificate) {
+		ip = "<machine-ip>"
-	crt, key, err := certs.NewClientCert(ca)
+	} else {
-	if err != nil {
+		logging.Info().Msgf("Detected machine IP: %s, change if needed", ip)
-		E.LogFatal("init SSL error", err)
 	}
-	caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Certificate[0]})
+
-	ip := machineIP()
 	host := fmt.Sprintf("%s:%d", ip, env.AgentPort)
 	cfgYAML, _ := yaml.Marshal(map[string]any{
 		"providers": map[string]any{
@@ -40,78 +31,43 @@ func printNewClientHelp(ca *tls.Certificate) {
 		},
 	})
 
-	certsData, err := certs.ZipCert(caPEM, crt, key)
+	logging.Info().Msgf("On main server, run:\n\ndocker exec godoxy /app/run add-agent '%s'\n\n", host)
-	if err != nil {
+	logging.Info().Msgf("Then add this host (%s) to main server config like below:\n\n", host)
-		E.LogFatal("marshal certs error", err)
+	logging.Info().Msg(string(cfgYAML))
-	}
-
-	fmt.Printf("On main server, run:\nnew-agent '%s' '%s'\n", host, base64.StdEncoding.EncodeToString(certsData))
-	fmt.Printf("Then add this host (%s) to main server config like below:\n", host)
-	fmt.Println(string(cfgYAML))
-}
-
-func machineIP() string {
-	interfaces, err := net.Interfaces()
-	if err != nil {
-		return "<machine-ip>"
-	}
-	for _, in := range interfaces {
-		addrs, err := in.Addrs()
-		if err != nil {
-			continue
-		}
-		if !strings.HasPrefix(in.Name, "eth") && !strings.HasPrefix(in.Name, "en") {
-			continue
-		}
-		for _, addr := range addrs {
-			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-				if ipnet.IP.To4() != nil {
-					return ipnet.IP.String()
-				}
-			}
-		}
-	}
-	return "<machine-ip>"
 }
 
 func main() {
-	args := pkg.GetArgs(agentCommandValidator{})
-
 	ca, srv, isNew, err := certs.InitCerts()
 	if err != nil {
 		E.LogFatal("init CA error", err)
 	}
 
-	if args.Command == CommandNewClient {
-		printNewClientHelp(ca)
-		return
-	}
-
 	logging.Info().Msgf("GoDoxy Agent version %s", pkg.GetVersion())
 	logging.Info().Msgf("Agent name: %s", env.AgentName)
 	logging.Info().Msgf("Agent port: %d", env.AgentPort)
 
-	logging.Info().Msg("\nTips:")
+	logging.Info().Msg(`
-	logging.Info().Msg("1. To change the agent name, you can set the AGENT_NAME environment variable.")
+Tips:
-	logging.Info().Msg("2. To change the agent port, you can set the AGENT_PORT environment variable.")
+1. To change the agent name, you can set the AGENT_NAME environment variable.
-	logging.Info().Msg("3. To skip the version check, you can set the AGENT_SKIP_VERSION_CHECK environment variable.")
+2. To change the agent port, you can set the AGENT_PORT environment variable.
-	logging.Info().Msgf("4. Create shell alias on main server: `alias new-agent='docker run --rm -v ./certs:/app/certs ghcr.io/yusing/godoxy /app/run new-agent'`")
+3. To skip the version check, you can set AGENT_SKIP_VERSION_CHECK to true.
-	logging.Info().Msgf("5. Create shell alias on agent server: `alias new-client='docker compose exec agent /app/run new-client'`\n")
+4. If anything goes wrong, you can remove the 'certs' directory and start over.
+`)
 
-	if isNew {
+	t := task.RootTask("agent", false)
-		logging.Info().Msg("Initialization complete.")
+	opts := server.Options{
-		logging.Info().Msg("New client cert created")
-		printNewClientHelp(ca)
-		logging.Info().Msg("Exiting... Clear the screen and start agent again")
-		logging.Info().Msg("To create more client certs, run `godoxy-agent new-client`")
-		return
-	}
-
-	server.StartAgentServer(task.RootTask("agent", false), server.Options{
 		CACert:     ca,
 		ServerCert: srv,
 		Port:       env.AgentPort,
-	})
+	}
+
+	if isNew {
+		logging.Info().Msg("Initialization complete.")
+		printNewClientHelp()
+		server.StartRegistrationServer(t, opts)
+	}
+
+	server.StartAgentServer(t, opts)
 
 	utils.WaitExit(3)
 }

agent/pkg/agent/config.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/rs/zerolog"
 	"github.com/yusing/go-proxy/agent/pkg/certs"
-	"github.com/yusing/go-proxy/agent/pkg/env"
 	E "github.com/yusing/go-proxy/internal/error"
 	"github.com/yusing/go-proxy/internal/logging"
 	gphttp "github.com/yusing/go-proxy/internal/net/http"
@@ -94,6 +93,14 @@ func (cfg *AgentConfig) errIfNameExists() E.Error {
 	return nil
 }
 
+func withoutBuildTime(version string) string {
+	return strings.Split(version, "-")[0]
+}
+
+func checkVersion(a, b string) bool {
+	return withoutBuildTime(a) == withoutBuildTime(b)
+}
+
 func (cfg *AgentConfig) load() E.Error {
 	certData, err := os.ReadFile(certs.AgentCertsFilename(cfg.Addr))
 	if err != nil {
@@ -132,15 +139,13 @@ func (cfg *AgentConfig) load() E.Error {
 	defer cancel()
 
 	// check agent version
-	if !env.AgentSkipVersionCheck {
+	version, _, err := cfg.Fetch(ctx, EndpointVersion)
-		version, _, err := cfg.Fetch(ctx, EndpointVersion)
+	if err != nil {
-		if err != nil {
+		return E.Wrap(err)
-			return E.Wrap(err)
+	}
-		}
 
-		if string(version) != pkg.GetVersion() {
+	if !checkVersion(string(version), pkg.GetVersion()) {
-			return E.Errorf("agent version mismatch: server: %s, agent: %s", pkg.GetVersion(), string(version))
+		return E.Errorf("agent version mismatch: server: %s, agent: %s", pkg.GetVersion(), string(version))
-		}
 	}
 
 	// get agent name

agent/pkg/agent/utils.go

@@ -0,0 +1,30 @@
+package agent
+
+import (
+	"net"
+	"strings"
+)
+
+func MachineIP() (string, bool) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		interfaces = []net.Interface{}
+	}
+	for _, in := range interfaces {
+		addrs, err := in.Addrs()
+		if err != nil {
+			continue
+		}
+		if !strings.HasPrefix(in.Name, "eth") && !strings.HasPrefix(in.Name, "en") {
+			continue
+		}
+		for _, addr := range addrs {
+			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+				if ipnet.IP.To4() != nil {
+					return ipnet.IP.String(), true
+				}
+			}
+		}
+	}
+	return "", false
+}

agent/pkg/certs/zip.go

@@ -32,6 +32,7 @@ func readFile(f *zip.File) ([]byte, error) {
 
 func ZipCert(ca, crt, key []byte) ([]byte, error) {
 	data := bytes.NewBuffer(nil)
+	data.Grow(6144)
 	zipWriter := zip.NewWriter(data)
 	defer zipWriter.Close()
 

agent/pkg/env/env.go

@@ -1,7 +1,10 @@
 package env
 
 import (
+	"log"
+	"net"
 	"os"
+	"strings"
 
 	"github.com/yusing/go-proxy/internal/common"
 )
@@ -15,7 +18,54 @@ func DefaultAgentName() string {
 }
 
 var (
-	AgentName             = common.GetEnvString("AGENT_NAME", DefaultAgentName())
+	AgentName                = common.GetEnvString("AGENT_NAME", DefaultAgentName())
-	AgentPort             = common.GetEnvInt("AGENT_PORT", 8890)
+	AgentPort                = common.GetEnvInt("AGENT_PORT", 8890)
-	AgentSkipVersionCheck = common.GetEnvBool("AGENT_SKIP_VERSION_CHECK", false)
+	AgentRegistrationPort    = common.GetEnvInt("AGENT_REGISTRATION_PORT", 8891)
+	AgentSkipClientCertCheck = common.GetEnvBool("AGENT_SKIP_CLIENT_CERT_CHECK", false)
+
+	RegistrationAllowedHosts = common.GetCommaSepEnv("REGISTRATION_ALLOWED_HOSTS", "")
+	RegistrationAllowedCIDRs []*net.IPNet
 )
+
+func init() {
+	cidrs, err := toCIDRs(RegistrationAllowedHosts)
+	if err != nil {
+		log.Fatalf("failed to parse allowed hosts: %v", err)
+	}
+	if len(cidrs) == 0 {
+		log.Fatal("REGISTRATION_ALLOWED_HOSTS is empty")
+	}
+	RegistrationAllowedCIDRs = cidrs
+}
+
+func toCIDRs(hosts []string) ([]*net.IPNet, error) {
+	var cidrs []*net.IPNet
+	for _, host := range hosts {
+		if !strings.Contains(host, "/") {
+			host += "/32"
+		}
+		_, cidr, err := net.ParseCIDR(host)
+		if err != nil {
+			return nil, err
+		}
+		cidrs = append(cidrs, cidr)
+	}
+	return cidrs, nil
+}
+
+func IsAllowedHost(remoteAddr string) bool {
+	ip, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		ip = remoteAddr
+	}
+	netIP := net.ParseIP(ip)
+	if netIP == nil {
+		return false
+	}
+	for _, cidr := range RegistrationAllowedCIDRs {
+		if cidr.Contains(netIP) {
+			return true
+		}
+	}
+	return false
+}

agent/pkg/handler/handler.go

@@ -1,14 +1,21 @@
 package handler
 
 import (
+	"crypto/tls"
+	"encoding/pem"
 	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/yusing/go-proxy/agent/pkg/agent"
+	"github.com/yusing/go-proxy/agent/pkg/certs"
 	"github.com/yusing/go-proxy/agent/pkg/env"
 	v1 "github.com/yusing/go-proxy/internal/api/v1"
+	"github.com/yusing/go-proxy/internal/api/v1/utils"
+	E "github.com/yusing/go-proxy/internal/error"
+	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/logging/memlogger"
+	"github.com/yusing/go-proxy/internal/task"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
 )
 
@@ -32,7 +39,7 @@ func (NopWriteCloser) Close() error {
 	return nil
 }
 
-func NewHandler() http.Handler {
+func NewAgentHandler() http.Handler {
 	mux := ServeMux{http.NewServeMux()}
 
 	mux.HandleFunc(agent.EndpointProxyHTTP+"/{path...}", ProxyHTTP)
@@ -46,3 +53,46 @@ func NewHandler() http.Handler {
 	mux.ServeMux.HandleFunc("/", DockerSocketHandler())
 	return mux
 }
+
+// NewRegistrationHandler creates a new registration handler
+// It checks if the request is coming from an allowed host
+// Generates a new client certificate and zips it
+// Sends the zipped certificate to the client
+// its run only once on agent first start.
+func NewRegistrationHandler(task *task.Task, ca *tls.Certificate) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !env.IsAllowedHost(r.RemoteAddr) {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		if r.URL.Path == "/done" {
+			logging.Info().Msg("registration done")
+			task.Finish(nil)
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		logging.Info().Msgf("received registration request from %s", r.RemoteAddr)
+
+		caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Certificate[0]})
+
+		crt, key, err := certs.NewClientCert(ca)
+		if err != nil {
+			utils.HandleErr(w, r, E.Wrap(err, "failed to generate client certificate"))
+			return
+		}
+
+		zipped, err := certs.ZipCert(caPEM, crt, key)
+		if err != nil {
+			utils.HandleErr(w, r, E.Wrap(err, "failed to zip certificate"))
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/zip")
+		if _, err := w.Write(zipped); err != nil {
+			logging.Error().Err(err).Msg("failed to respond to registration request")
+			return
+		}
+	}
+}

agent/pkg/server/server.go

@@ -11,9 +11,10 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/yusing/go-proxy/agent/pkg/env"
 	"github.com/yusing/go-proxy/agent/pkg/handler"
-	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/yusing/go-proxy/internal/net/http/server"
 	"github.com/yusing/go-proxy/internal/task"
 )
 
@@ -23,7 +24,7 @@ type Options struct {
 }
 
 func StartAgentServer(parent task.Parent, opt Options) {
-	t := parent.Subtask("agent server")
+	t := parent.Subtask("agent_server")
 
 	caCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: opt.CACert.Certificate[0]})
 	caCertPool := x509.NewCertPool()
@@ -36,23 +37,24 @@ func StartAgentServer(parent task.Parent, opt Options) {
 		ClientAuth:   tls.RequireAndVerifyClientCert,
 	}
 
-	if common.IsDebug {
+	if env.AgentSkipClientCertCheck {
 		tlsConfig.ClientAuth = tls.NoClientCert
 	}
-	l, err := net.Listen("tcp", fmt.Sprintf(":%d", opt.Port))
-	if err != nil {
-		logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to listen on port")
-		return
-	}
 
-	server := &http.Server{
+	agentServer := &http.Server{
-		Handler:   handler.NewHandler(),
+		Handler:   handler.NewAgentHandler(),
 		TLSConfig: tlsConfig,
 		ErrorLog:  log.New(logging.GetLogger(), "", 0),
 	}
+
 	go func() {
+		l, err := net.Listen("tcp", fmt.Sprintf(":%d", opt.Port))
+		if err != nil {
+			logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to listen on port")
+			return
+		}
 		defer l.Close()
-		if err := server.Serve(tls.NewListener(l, tlsConfig)); err != nil {
+		if err := agentServer.Serve(tls.NewListener(l, tlsConfig)); err != nil {
 			logging.Fatal().Err(err).Int("port", opt.Port).Msg("failed to serve")
 		}
 	}()
@@ -66,10 +68,38 @@ func StartAgentServer(parent task.Parent, opt Options) {
 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 		defer cancel()
 
-		err := server.Shutdown(ctx)
+		err := agentServer.Shutdown(ctx)
 		if err != nil {
 			logging.Error().Err(err).Int("port", opt.Port).Msg("failed to shutdown agent server")
 		}
 		logging.Info().Int("port", opt.Port).Msg("agent server stopped")
 	}()
 }
+
+func StartRegistrationServer(parent task.Parent, opt Options) {
+	t := parent.Subtask("registration_server")
+
+	registrationServer := &http.Server{
+		Addr:     fmt.Sprintf(":%d", opt.Port),
+		Handler:  handler.NewRegistrationHandler(t, opt.CACert),
+		ErrorLog: log.New(logging.GetLogger(), "", 0),
+	}
+
+	go func() {
+		err := registrationServer.ListenAndServe()
+		server.HandleError(logging.GetLogger(), err)
+	}()
+
+	logging.Info().Int("port", opt.Port).Msg("registration server started")
+
+	defer t.Finish(nil)
+	<-t.Context().Done()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	err := registrationServer.Shutdown(ctx)
+	server.HandleError(logging.GetLogger(), err)
+
+	logging.Info().Int("port", opt.Port).Msg("registration server stopped")
+}

cmd/add_agent.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/yusing/go-proxy/agent/pkg/certs"
+	"github.com/yusing/go-proxy/internal/logging"
+)
+
+func AddAgent(args []string) {
+	if len(args) != 1 {
+		logging.Fatal().Msgf("invalid arguments: %v, expect host", args)
+	}
+	host := args[0]
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", "http://"+host, nil)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to create request")
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to send request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		logging.Fatal().Int("status", resp.StatusCode).Msg("failed to add agent")
+	}
+
+	zip, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to read response body")
+	}
+
+	f, err := os.OpenFile(certs.AgentCertsFilename(host), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to create client certs file")
+	}
+	defer f.Close()
+
+	if _, err := f.Write(zip); err != nil {
+		logging.Fatal().Err(err).Msg("failed to save client certs")
+	}
+
+	logging.Info().Msgf("agent %s added, certs saved to %s", host, certs.AgentCertsFilename(host))
+
+	req, err = http.NewRequestWithContext(ctx, "GET", "http://"+host+"/done", nil)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to create done request")
+	}
+
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		logging.Fatal().Err(err).Msg("failed to send done request")
+	}
+	defer resp.Body.Close()
+}

cmd/main.go

@@ -2,11 +2,9 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"log"
 	"os"
 
-	"github.com/rs/zerolog"
 	"github.com/yusing/go-proxy/internal"
 	"github.com/yusing/go-proxy/internal/api/v1/auth"
 	"github.com/yusing/go-proxy/internal/api/v1/favicon"
@@ -16,7 +14,6 @@ import (
 	E "github.com/yusing/go-proxy/internal/error"
 	"github.com/yusing/go-proxy/internal/homepage"
 	"github.com/yusing/go-proxy/internal/logging"
-	"github.com/yusing/go-proxy/internal/logging/memlogger"
 	"github.com/yusing/go-proxy/internal/net/http/middleware"
 	"github.com/yusing/go-proxy/internal/route/routes/routequery"
 	"github.com/yusing/go-proxy/internal/utils"
@@ -25,14 +22,6 @@ import (
 
 var rawLogger = log.New(os.Stdout, "", 0)
 
-func init() {
-	var out io.Writer = os.Stderr
-	if common.EnableLogStreaming {
-		out = zerolog.MultiLevelWriter(out, memlogger.GetMemLogger())
-	}
-	logging.InitLogger(out)
-}
-
 func main() {
 	initProfiling()
 	args := pkg.GetArgs(common.MainServerCommandValidator{})
@@ -41,8 +30,8 @@ func main() {
 	case common.CommandSetup:
 		Setup()
 		return
-	case common.CommandNewAgent:
+	case common.CommandAddAgent:
-		NewAgent(args.Args)
+		AddAgent(args.Args)
 		return
 	case common.CommandReload:
 		if err := query.ReloadServer(); err != nil {

cmd/new_agent.go

@@ -1,46 +0,0 @@
-package main
-
-import (
-	"encoding/base64"
-	"log"
-	"net"
-	"os"
-
-	"github.com/yusing/go-proxy/agent/pkg/certs"
-)
-
-func NewAgent(args []string) {
-	if len(args) != 2 {
-		log.Fatalf("invalid arguments: %v", args)
-	}
-	host := args[0]
-	certDataBase64 := args[1]
-
-	ip, _, err := net.SplitHostPort(host)
-	if err != nil {
-		log.Fatalf("invalid host: %v", err)
-	}
-
-	_, err = net.ResolveIPAddr("ip", ip)
-	if err != nil {
-		log.Fatalf("invalid host: %v", err)
-	}
-
-	certData, err := base64.StdEncoding.DecodeString(certDataBase64)
-	if err != nil {
-		log.Fatalf("invalid cert data: %v", err)
-	}
-
-	f, err := os.OpenFile(certs.AgentCertsFilename(host), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
-	if err != nil {
-		log.Fatalf("failed to create file: %v", err)
-	}
-	defer f.Close()
-
-	_, err = f.Write(certData)
-	if err != nil {
-		log.Fatalf("failed to write cert data: %v", err)
-	}
-
-	log.Printf("agent cert created: %s", certs.AgentCertsFilename(host))
-}

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/puzpuzpuz/xsync/v3 v3.5.0
 	github.com/rs/zerolog v1.33.0
+	github.com/shirou/gopsutil/v4 v4.25.1
 	github.com/vincent-petithory/dataurl v1.0.0
 	golang.org/x/crypto v0.33.0
 	golang.org/x/net v0.35.0
@@ -68,7 +69,6 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect

internal/common/args.go

@@ -3,7 +3,7 @@ package common
 const (
 	CommandStart              = ""
 	CommandSetup              = "setup"
-	CommandNewAgent           = "new-agent"
+	CommandAddAgent           = "add-agent"
 	CommandValidate           = "validate"
 	CommandListConfigs        = "ls-config"
 	CommandListRoutes         = "ls-routes"
@@ -20,7 +20,7 @@ func (v MainServerCommandValidator) IsCommandValid(cmd string) bool {
 	switch cmd {
 	case CommandStart,
 		CommandSetup,
-		CommandNewAgent,
+		CommandAddAgent,
 		CommandValidate,
 		CommandListConfigs,
 		CommandListRoutes,

internal/common/env.go

@@ -20,8 +20,7 @@ var (
 	IsTrace      = GetEnvBool("TRACE", false) && IsDebug
 	IsProduction = !IsTest && !IsDebug
 
-	EnableLogStreaming = GetEnvBool("LOG_STREAMING", true)
+	DebugMemLogger = GetEnvBool("DEBUG_MEM_LOGGER", false)
-	DebugMemLogger     = GetEnvBool("DEBUG_MEM_LOGGER", false) && EnableLogStreaming
 
 	ProxyHTTPAddr,
 	ProxyHTTPHost,

internal/config/config.go

@@ -290,6 +290,9 @@ func (cfg *Config) loadRouteProviders(providers *types.Providers) E.Error {
 	for _, agent := range providers.Agents {
 		cfg.providers.Store(agent.Name(), proxy.NewAgentProvider(&agent))
 	}
+	if cfg.providers.Size() == 0 {
+		return nil
+	}
 	cfg.providers.RangeAllParallel(func(_ string, p *proxy.Provider) {
 		if err := p.LoadRoutes(); err != nil {
 			errs.Add(err.Subject(p.String()))

internal/error/utils.go

@@ -23,7 +23,7 @@ func Wrap(err error, message ...string) Error {
 	if len(message) == 0 || message[0] == "" {
 		return From(err)
 	}
-	return Errorf("%w: %s", err, message[0])
+	return Errorf("%s: %w", message[0], err)
 }
 
 func From(err error) Error {

internal/logging/memlogger/mem_logger.go

@@ -5,10 +5,12 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 
 	"github.com/coder/websocket"
+	"github.com/rs/zerolog"
 	"github.com/yusing/go-proxy/internal/api/v1/utils"
 	"github.com/yusing/go-proxy/internal/common"
 	config "github.com/yusing/go-proxy/internal/config/types"
@@ -55,9 +57,6 @@ var memLoggerInstance = &memLogger{
 }
 
 func init() {
-	if !common.EnableLogStreaming {
-		return
-	}
 	memLoggerInstance.Grow(maxMemLogSize)
 
 	if common.DebugMemLogger {
@@ -78,6 +77,8 @@ func init() {
 			}
 		}()
 	}
+
+	logging.InitLogger(zerolog.MultiLevelWriter(os.Stderr, memLoggerInstance))
 }
 
 func LogsWS(config config.ConfigInstance) http.HandlerFunc {

internal/net/http/server/error.go

@@ -0,0 +1,18 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/rs/zerolog"
+)
+
+func HandleError(logger *zerolog.Logger, err error) {
+	switch {
+	case err == nil, errors.Is(err, http.ErrServerClosed), errors.Is(err, context.Canceled):
+		return
+	default:
+		logger.Fatal().Err(err).Msg("server error")
+	}
+}

internal/net/http/server/server.go

@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"io"
 	"log"
 	"net"
@@ -100,7 +99,7 @@ func (s *Server) Start(parent task.Parent) {
 	s.startTime = time.Now()
 	if s.http != nil {
 		go func() {
-			s.handleErr("http", s.http.ListenAndServe())
+			s.handleErr(s.http.ListenAndServe())
 		}()
 		s.httpStarted = true
 		s.l.Info().Str("addr", s.http.Addr).Msg("server started")
@@ -110,11 +109,11 @@ func (s *Server) Start(parent task.Parent) {
 		go func() {
 			l, err := net.Listen("tcp", s.https.Addr)
 			if err != nil {
-				s.handleErr("https", err)
+				s.handleErr(err)
 				return
 			}
 			defer l.Close()
-			s.handleErr("https", s.https.Serve(tls.NewListener(l, s.https.TLSConfig)))
+			s.handleErr(s.https.Serve(tls.NewListener(l, s.https.TLSConfig)))
 		}()
 		s.httpsStarted = true
 		s.l.Info().Str("addr", s.https.Addr).Msgf("server started")
@@ -132,13 +131,13 @@ func (s *Server) stop() {
 	defer cancel()
 
 	if s.http != nil && s.httpStarted {
-		s.handleErr("http", s.http.Shutdown(ctx))
+		s.handleErr(s.http.Shutdown(ctx))
 		s.httpStarted = false
 		s.l.Info().Str("addr", s.http.Addr).Msgf("server stopped")
 	}
 
 	if s.https != nil && s.httpsStarted {
-		s.handleErr("https", s.https.Shutdown(ctx))
+		s.handleErr(s.https.Shutdown(ctx))
 		s.httpsStarted = false
 		s.l.Info().Str("addr", s.https.Addr).Msgf("server stopped")
 	}
@@ -148,11 +147,6 @@ func (s *Server) Uptime() time.Duration {
 	return time.Since(s.startTime)
 }
 
-func (s *Server) handleErr(scheme string, err error) {
+func (s *Server) handleErr(err error) {
-	switch {
+	HandleError(&s.l, err)
-	case err == nil, errors.Is(err, http.ErrServerClosed), errors.Is(err, context.Canceled):
-		return
-	default:
-		s.l.Fatal().Err(err).Str("scheme", scheme).Msg("server error")
-	}
 }

next-release.md

@@ -1,8 +1,8 @@
 ## GoDoxy v0.10.0
 
-### GoDoxy-Agent
+### GoDoxy Agent
 
-listen only on Agent API server, authenticate and encrypt connection with mTLS. Maintain secure connection between GoDoxy main and GoDoxy agent server
+Maintain secure connection between main server and agent server by authenticating and encrypting connection with mTLS.
 
 Main benefits:
 
@@ -20,9 +20,16 @@ Main benefits:
 
 #### How to setup
 
-1. Agent server generates CA cert, SSL certificate and Client certificate on first run.
+Prerequisites:
-2. Follow the output on screen to run `godoxy new-agent <ip>:<port> ...` on GoDoxy main server to store generated certs
+
-3. Add config output to GoDoxy main server in `config.yml` under `providers.agents`
+- GoDoxy main server must be running
+
+1. Create a directory for agent server, cd into it
+2. Copy `agent.compose.yml` into the directory
+3. Modify `agent.compose.yml` to set `REGISTRATION_ALLOWED_HOSTS`
+4. Run `docker-compose up -d` to start agent
+5. Follow instructions on screen to run command on GoDoxy main server
+6. Add config output to GoDoxy main server in `config.yml` under `providers.agents`
    ```yaml
    providers:
      agents:
@@ -31,6 +38,47 @@ Main benefits:
 
 ### How does it work
 
-1. Main server and agent server negotiate mTLS
+Setup flow:
-2. Agent server verify main server's client cert and check if server version matches agent version
+
-3. Agent server now acts as a http proxy and docker socket proxy
+```mermaid
+flowchart TD
+    subgraph Agent Server
+        A[Create a directory] -->
+        B[Setup agent.compose.yml] -->
+        C[Set REGISTRATION_ALLOWED_HOSTS] -->
+        D[Run agent] -->
+        E[Wait for main server to register]
+
+        F[Respond to main server]
+        G[Agent now run in agent mode]
+    end
+    subgraph Main Server
+      E -->
+      H[Run register command] -->
+      I[Send registration request] --> F -->
+      J[Store client certs] -->
+      K[Send done request] --> G -->
+      L[Add agent to config.yml]
+    end
+```
+
+Run flow:
+
+```mermaid
+flowchart TD
+    subgraph Agent HTTPS Server
+        aa[Load CA and SSL certs] -->
+        ab[Start HTTPS server] -->
+
+        ac[Receive request] -->
+        ad[Verify client cert] -->
+        ae[Handle request] --> ac
+    end
+    subgraph Main Server
+        ma[Load client certs] -->
+        mb[Query agent version] --> ac
+        mb --> mc[Check if agent version matches] -->
+        md[Query agent info] --> ac
+        md --> ae --> me[Store agent info]
+    end
+```