improve error handling for system info metrics

This commit is contained in:
yusing 2025-02-14 03:49:56 +08:00
parent 267fd403da
commit f0198616ad
3 changed files with 98 additions and 35 deletions

View file

@ -53,6 +53,13 @@ var (
OIDCScopes = GetEnvString("OIDC_SCOPES", "openid, profile, email") OIDCScopes = GetEnvString("OIDC_SCOPES", "openid, profile, email")
OIDCAllowedUsers = GetCommaSepEnv("OIDC_ALLOWED_USERS", "") OIDCAllowedUsers = GetCommaSepEnv("OIDC_ALLOWED_USERS", "")
OIDCAllowedGroups = GetCommaSepEnv("OIDC_ALLOWED_GROUPS", "") OIDCAllowedGroups = GetCommaSepEnv("OIDC_ALLOWED_GROUPS", "")
// metrics configuration
MetricsDisableCPU = GetEnvBool("METRICS_DISABLE_CPU", false)
MetricsDisableMemory = GetEnvBool("METRICS_DISABLE_MEMORY", false)
MetricsDisableDisk = GetEnvBool("METRICS_DISABLE_DISK", false)
MetricsDisableNetwork = GetEnvBool("METRICS_DISABLE_NETWORK", false)
MetricsDisableSensors = GetEnvBool("METRICS_DISABLE_SENSORS", false)
) )
func GetEnv[T any](key string, defaultValue T, parser func(string) (T, error)) T { func GetEnv[T any](key string, defaultValue T, parser func(string) (T, error)) T {

View file

@ -122,3 +122,9 @@ func (b *Builder) AddRange(errs ...error) *Builder {
return b return b
} }
func (b *Builder) ForEach(fn func(error)) {
for _, err := range b.errs {
fn(err)
}
}

View file

@ -3,6 +3,7 @@ package systeminfo
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"time" "time"
"github.com/shirou/gopsutil/v4/cpu" "github.com/shirou/gopsutil/v4/cpu"
@ -10,6 +11,9 @@ import (
"github.com/shirou/gopsutil/v4/mem" "github.com/shirou/gopsutil/v4/mem"
"github.com/shirou/gopsutil/v4/net" "github.com/shirou/gopsutil/v4/net"
"github.com/shirou/gopsutil/v4/sensors" "github.com/shirou/gopsutil/v4/sensors"
"github.com/yusing/go-proxy/internal/common"
E "github.com/yusing/go-proxy/internal/error"
"github.com/yusing/go-proxy/internal/logging"
"github.com/yusing/go-proxy/internal/metrics/period" "github.com/yusing/go-proxy/internal/metrics/period"
"github.com/yusing/go-proxy/internal/utils/strutils" "github.com/yusing/go-proxy/internal/utils/strutils"
) )
@ -31,44 +35,90 @@ func init() {
Poller.Start() Poller.Start()
} }
func getSystemInfo(ctx context.Context, lastResult *SystemInfo) (*SystemInfo, error) { func _() { // check if this behavior is not changed
memoryInfo, err := mem.VirtualMemory() var _ sensors.Warnings = disk.Warnings{}
if err != nil {
return nil, err
} }
func getSystemInfo(ctx context.Context, lastResult *SystemInfo) (*SystemInfo, error) {
errs := E.NewBuilder("failed to get system info")
var systemInfo SystemInfo
if !common.MetricsDisableCPU {
cpuAverage, err := cpu.PercentWithContext(ctx, 150*time.Millisecond, false) cpuAverage, err := cpu.PercentWithContext(ctx, 150*time.Millisecond, false)
if err != nil { if err != nil {
return nil, err errs.Add(err)
} else {
systemInfo.CPUAverage = cpuAverage[0]
} }
}
if !common.MetricsDisableMemory {
memoryInfo, err := mem.VirtualMemory()
if err != nil {
errs.Add(err)
}
systemInfo.Memory = memoryInfo
}
if !common.MetricsDisableDisk {
diskInfo, err := disk.Usage("/") diskInfo, err := disk.Usage("/")
if err != nil { if err != nil {
return nil, err errs.Add(err)
} }
systemInfo.Disk = diskInfo
}
if !common.MetricsDisableNetwork {
networkIO, err := net.IOCounters(false) networkIO, err := net.IOCounters(false)
if err != nil { if err != nil {
return nil, err errs.Add(err)
} } else {
sensors, err := sensors.SensorsTemperatures() networkIO := networkIO[0]
if err != nil { systemInfo.NetworkIO = &networkIO
return nil, err
}
var networkUp, networkDown float64 var networkUp, networkDown float64
if lastResult != nil { if lastResult != nil {
interval := time.Since(lastResult.Timestamp).Seconds() interval := time.Since(lastResult.Timestamp).Seconds()
networkUp = float64(networkIO[0].BytesSent-lastResult.NetworkIO.BytesSent) / interval networkUp = float64(networkIO.BytesSent-lastResult.NetworkIO.BytesSent) / interval
networkDown = float64(networkIO[0].BytesRecv-lastResult.NetworkIO.BytesRecv) / interval networkDown = float64(networkIO.BytesRecv-lastResult.NetworkIO.BytesRecv) / interval
}
systemInfo.NetworkUp = networkUp
systemInfo.NetworkDown = networkDown
}
} }
return &SystemInfo{ if !common.MetricsDisableSensors {
Timestamp: time.Now(), sensorsInfo, err := sensors.SensorsTemperatures()
CPUAverage: cpuAverage[0], if err != nil {
Memory: memoryInfo, errs.Add(err)
Disk: diskInfo, }
NetworkIO: &networkIO[0], systemInfo.Sensors = sensorsInfo
NetworkUp: networkUp, }
NetworkDown: networkDown,
Sensors: sensors, if errs.HasError() {
}, nil allWarnings := E.NewBuilder("")
allErrors := E.NewBuilder("failed to get system info")
errs.ForEach(func(err error) {
// disk.Warnings has the same type
// all Warnings are alias of common.Warnings from "github.com/shirou/gopsutil/v4/internal/common"
// see line 37
var warnings sensors.Warnings
if errors.As(err, &warnings) {
for _, warning := range warnings.List {
allWarnings.Add(warning)
}
} else {
allErrors.Add(err)
}
})
if allWarnings.HasError() {
logging.Warn().Msg(allWarnings.String())
}
if allErrors.HasError() {
return nil, allErrors.Error()
}
}
return &systemInfo, nil
} }
func (s *SystemInfo) MarshalJSON() ([]byte, error) { func (s *SystemInfo) MarshalJSON() ([]byte, error) {