added ping latency to healthcheck result

This commit is contained in:
yusing 2024-11-30 06:43:47 +08:00
parent 497879fb4b
commit fb9de4c4ad
6 changed files with 50 additions and 19 deletions

View file

@ -72,7 +72,7 @@ func (cfg *Config) GetProvider() (*Provider, E.Error) {
var err error
if privKey, err = cfg.loadACMEKey(); err != nil {
logging.Err(err).Msg("load ACME private key failed, generating one...")
logging.Info().Err(err).Msg("load ACME private key failed, generating one...")
privKey, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, E.New("generate ACME private key").With(err)

View file

@ -132,12 +132,12 @@ func (w *Watcher) getStatusUpdateReady() health.Status {
return health.StatusHealthy
}
healthy, _, err := w.hc.CheckHealth()
result, err := w.hc.CheckHealth()
switch {
case err != nil:
w.ready.Store(false)
return health.StatusError
case healthy:
case result.Healthy:
w.ready.Store(true)
return health.StatusHealthy
default:

View file

@ -10,6 +10,11 @@ import (
)
type (
HealthCheckResult struct {
Healthy bool
Detail string
Latency time.Duration
}
HealthMonitor interface {
task.TaskStarter
task.TaskFinisher
@ -20,7 +25,7 @@ type (
Name() string
}
HealthChecker interface {
CheckHealth() (healthy bool, detail string, err error)
CheckHealth() (result *HealthCheckResult, err error)
URL() types.URL
Config() *HealthCheckConfig
UpdateURL(url types.URL)

View file

@ -4,6 +4,7 @@ import (
"crypto/tls"
"errors"
"net/http"
"time"
"github.com/yusing/go-proxy/internal/net/types"
"github.com/yusing/go-proxy/internal/watcher/health"
@ -40,7 +41,7 @@ func NewHTTPHealthChecker(url types.URL, config *health.HealthCheckConfig) healt
return NewHTTPHealthMonitor(url, config)
}
func (mon *HTTPHealthMonitor) CheckHealth() (healthy bool, detail string, err error) {
func (mon *HTTPHealthMonitor) CheckHealth() (result *health.HealthCheckResult, err error) {
ctx, cancel := mon.ContextWithTimeout("ping request timed out")
defer cancel()
@ -57,24 +58,30 @@ func (mon *HTTPHealthMonitor) CheckHealth() (healthy bool, detail string, err er
req.Header.Set("Connection", "close")
req.Header.Set("User-Agent", "GoDoxy/"+pkg.GetVersion())
start := time.Now()
resp, respErr := pinger.Do(req)
if respErr == nil {
resp.Body.Close()
}
result = &health.HealthCheckResult{
Latency: time.Since(start),
}
switch {
case respErr != nil:
// treat tls error as healthy
var tlsErr *tls.CertificateVerificationError
if ok := errors.As(respErr, &tlsErr); !ok {
detail = respErr.Error()
result.Detail = respErr.Error()
return
}
case resp.StatusCode == http.StatusServiceUnavailable:
detail = resp.Status
result.Detail = resp.Status
return
}
healthy = true
result.Healthy = true
return
}

View file

@ -20,7 +20,7 @@ import (
)
type (
HealthCheckFunc func() (healthy bool, detail string, err error)
HealthCheckFunc func() (result *health.HealthCheckResult, err error)
monitor struct {
service string
config *health.HealthCheckConfig
@ -161,7 +161,7 @@ func (mon *monitor) MarshalJSON() ([]byte, error) {
func (mon *monitor) checkUpdateHealth() error {
logger := logging.With().Str("name", mon.Name()).Logger()
healthy, detail, err := mon.checkHealth()
result, err := mon.checkHealth()
if err != nil {
defer mon.task.Finish(err)
mon.status.Store(health.StatusError)
@ -171,19 +171,33 @@ func (mon *monitor) checkUpdateHealth() error {
return nil
}
var status health.Status
if healthy {
if result.Healthy {
status = health.StatusHealthy
} else {
status = health.StatusUnhealthy
}
if healthy != (mon.status.Swap(status) == health.StatusHealthy) {
if healthy {
if result.Healthy != (mon.status.Swap(status) == health.StatusHealthy) {
extras := map[string]any{
"Service Name": mon.service,
"Service URL": mon.url.Load().String(),
}
if result.Healthy {
logger.Info().Msg("server is up")
notif.Notify(mon.service, "server is up")
extras["Ping"] = fmt.Sprintf("%d ms", result.Latency.Milliseconds())
notif.Notify(&notif.LogMessage{
Title: "✅ Service is up ✅",
Extras: extras,
Color: notif.Green,
})
} else {
logger.Warn().Msg("server is down")
logger.Debug().Msg(detail)
notif.Notify(mon.service, "server is down")
logger.Debug().Msg(result.Detail)
extras["Detail"] = result.Detail
notif.Notify(&notif.LogMessage{
Title: "❌ Service went down ❌",
Extras: extras,
Color: notif.Red,
})
}
}
if mon.metric != nil {

View file

@ -2,6 +2,7 @@ package monitor
import (
"net"
"time"
"github.com/yusing/go-proxy/internal/net/types"
"github.com/yusing/go-proxy/internal/watcher/health"
@ -28,18 +29,22 @@ func NewRawHealthChecker(url types.URL, config *health.HealthCheckConfig) health
return NewRawHealthMonitor(url, config)
}
func (mon *RawHealthMonitor) CheckHealth() (healthy bool, detail string, err error) {
func (mon *RawHealthMonitor) CheckHealth() (result *health.HealthCheckResult, err error) {
ctx, cancel := mon.ContextWithTimeout("ping request timed out")
defer cancel()
url := mon.url.Load()
start := time.Now()
conn, dialErr := mon.dialer.DialContext(ctx, url.Scheme, url.Host)
result = &health.HealthCheckResult{
Latency: time.Since(start),
}
if dialErr != nil {
detail = dialErr.Error()
result.Detail = dialErr.Error()
/* trunk-ignore(golangci-lint/nilerr) */
return
}
conn.Close()
healthy = true
result.Healthy = true
return
}