added ping latency to healthcheck result

This commit is contained in:
yusing 2024-11-30 06:43:47 +08:00
parent 497879fb4b
commit fb9de4c4ad
6 changed files with 50 additions and 19 deletions

View file

@ -72,7 +72,7 @@ func (cfg *Config) GetProvider() (*Provider, E.Error) {
var err error var err error
if privKey, err = cfg.loadACMEKey(); err != nil { if privKey, err = cfg.loadACMEKey(); err != nil {
logging.Err(err).Msg("load ACME private key failed, generating one...") logging.Info().Err(err).Msg("load ACME private key failed, generating one...")
privKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) privKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil { if err != nil {
return nil, E.New("generate ACME private key").With(err) return nil, E.New("generate ACME private key").With(err)

View file

@ -132,12 +132,12 @@ func (w *Watcher) getStatusUpdateReady() health.Status {
return health.StatusHealthy return health.StatusHealthy
} }
healthy, _, err := w.hc.CheckHealth() result, err := w.hc.CheckHealth()
switch { switch {
case err != nil: case err != nil:
w.ready.Store(false) w.ready.Store(false)
return health.StatusError return health.StatusError
case healthy: case result.Healthy:
w.ready.Store(true) w.ready.Store(true)
return health.StatusHealthy return health.StatusHealthy
default: default:

View file

@ -10,6 +10,11 @@ import (
) )
type ( type (
HealthCheckResult struct {
Healthy bool
Detail string
Latency time.Duration
}
HealthMonitor interface { HealthMonitor interface {
task.TaskStarter task.TaskStarter
task.TaskFinisher task.TaskFinisher
@ -20,7 +25,7 @@ type (
Name() string Name() string
} }
HealthChecker interface { HealthChecker interface {
CheckHealth() (healthy bool, detail string, err error) CheckHealth() (result *HealthCheckResult, err error)
URL() types.URL URL() types.URL
Config() *HealthCheckConfig Config() *HealthCheckConfig
UpdateURL(url types.URL) UpdateURL(url types.URL)

View file

@ -4,6 +4,7 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"net/http" "net/http"
"time"
"github.com/yusing/go-proxy/internal/net/types" "github.com/yusing/go-proxy/internal/net/types"
"github.com/yusing/go-proxy/internal/watcher/health" "github.com/yusing/go-proxy/internal/watcher/health"
@ -40,7 +41,7 @@ func NewHTTPHealthChecker(url types.URL, config *health.HealthCheckConfig) healt
return NewHTTPHealthMonitor(url, config) return NewHTTPHealthMonitor(url, config)
} }
func (mon *HTTPHealthMonitor) CheckHealth() (healthy bool, detail string, err error) { func (mon *HTTPHealthMonitor) CheckHealth() (result *health.HealthCheckResult, err error) {
ctx, cancel := mon.ContextWithTimeout("ping request timed out") ctx, cancel := mon.ContextWithTimeout("ping request timed out")
defer cancel() defer cancel()
@ -57,24 +58,30 @@ func (mon *HTTPHealthMonitor) CheckHealth() (healthy bool, detail string, err er
req.Header.Set("Connection", "close") req.Header.Set("Connection", "close")
req.Header.Set("User-Agent", "GoDoxy/"+pkg.GetVersion()) req.Header.Set("User-Agent", "GoDoxy/"+pkg.GetVersion())
start := time.Now()
resp, respErr := pinger.Do(req) resp, respErr := pinger.Do(req)
if respErr == nil { if respErr == nil {
resp.Body.Close() resp.Body.Close()
} }
result = &health.HealthCheckResult{
Latency: time.Since(start),
}
switch { switch {
case respErr != nil: case respErr != nil:
// treat tls error as healthy // treat tls error as healthy
var tlsErr *tls.CertificateVerificationError var tlsErr *tls.CertificateVerificationError
if ok := errors.As(respErr, &tlsErr); !ok { if ok := errors.As(respErr, &tlsErr); !ok {
detail = respErr.Error() result.Detail = respErr.Error()
return return
} }
case resp.StatusCode == http.StatusServiceUnavailable: case resp.StatusCode == http.StatusServiceUnavailable:
detail = resp.Status result.Detail = resp.Status
return return
} }
healthy = true result.Healthy = true
return return
} }

View file

@ -20,7 +20,7 @@ import (
) )
type ( type (
HealthCheckFunc func() (healthy bool, detail string, err error) HealthCheckFunc func() (result *health.HealthCheckResult, err error)
monitor struct { monitor struct {
service string service string
config *health.HealthCheckConfig config *health.HealthCheckConfig
@ -161,7 +161,7 @@ func (mon *monitor) MarshalJSON() ([]byte, error) {
func (mon *monitor) checkUpdateHealth() error { func (mon *monitor) checkUpdateHealth() error {
logger := logging.With().Str("name", mon.Name()).Logger() logger := logging.With().Str("name", mon.Name()).Logger()
healthy, detail, err := mon.checkHealth() result, err := mon.checkHealth()
if err != nil { if err != nil {
defer mon.task.Finish(err) defer mon.task.Finish(err)
mon.status.Store(health.StatusError) mon.status.Store(health.StatusError)
@ -171,19 +171,33 @@ func (mon *monitor) checkUpdateHealth() error {
return nil return nil
} }
var status health.Status var status health.Status
if healthy { if result.Healthy {
status = health.StatusHealthy status = health.StatusHealthy
} else { } else {
status = health.StatusUnhealthy status = health.StatusUnhealthy
} }
if healthy != (mon.status.Swap(status) == health.StatusHealthy) { if result.Healthy != (mon.status.Swap(status) == health.StatusHealthy) {
if healthy { extras := map[string]any{
"Service Name": mon.service,
"Service URL": mon.url.Load().String(),
}
if result.Healthy {
logger.Info().Msg("server is up") logger.Info().Msg("server is up")
notif.Notify(mon.service, "server is up") extras["Ping"] = fmt.Sprintf("%d ms", result.Latency.Milliseconds())
notif.Notify(&notif.LogMessage{
Title: "✅ Service is up ✅",
Extras: extras,
Color: notif.Green,
})
} else { } else {
logger.Warn().Msg("server is down") logger.Warn().Msg("server is down")
logger.Debug().Msg(detail) logger.Debug().Msg(result.Detail)
notif.Notify(mon.service, "server is down") extras["Detail"] = result.Detail
notif.Notify(&notif.LogMessage{
Title: "❌ Service went down ❌",
Extras: extras,
Color: notif.Red,
})
} }
} }
if mon.metric != nil { if mon.metric != nil {

View file

@ -2,6 +2,7 @@ package monitor
import ( import (
"net" "net"
"time"
"github.com/yusing/go-proxy/internal/net/types" "github.com/yusing/go-proxy/internal/net/types"
"github.com/yusing/go-proxy/internal/watcher/health" "github.com/yusing/go-proxy/internal/watcher/health"
@ -28,18 +29,22 @@ func NewRawHealthChecker(url types.URL, config *health.HealthCheckConfig) health
return NewRawHealthMonitor(url, config) return NewRawHealthMonitor(url, config)
} }
func (mon *RawHealthMonitor) CheckHealth() (healthy bool, detail string, err error) { func (mon *RawHealthMonitor) CheckHealth() (result *health.HealthCheckResult, err error) {
ctx, cancel := mon.ContextWithTimeout("ping request timed out") ctx, cancel := mon.ContextWithTimeout("ping request timed out")
defer cancel() defer cancel()
url := mon.url.Load() url := mon.url.Load()
start := time.Now()
conn, dialErr := mon.dialer.DialContext(ctx, url.Scheme, url.Host) conn, dialErr := mon.dialer.DialContext(ctx, url.Scheme, url.Host)
result = &health.HealthCheckResult{
Latency: time.Since(start),
}
if dialErr != nil { if dialErr != nil {
detail = dialErr.Error() result.Detail = dialErr.Error()
/* trunk-ignore(golangci-lint/nilerr) */ /* trunk-ignore(golangci-lint/nilerr) */
return return
} }
conn.Close() conn.Close()
healthy = true result.Healthy = true
return return
} }