implement icon cache expiry, cleanup code and upgrade deps

go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/PuerkitoBio/goquery v1.10.1
 	github.com/coder/websocket v1.8.12
 	github.com/coreos/go-oidc/v3 v3.12.0
-	github.com/docker/cli v27.5.0+incompatible
+	github.com/docker/cli v27.5.1+incompatible
-	github.com/docker/docker v27.5.0+incompatible
+	github.com/docker/docker v27.5.1+incompatible
 	github.com/fsnotify/fsnotify v1.8.0
 	github.com/go-acme/lego/v4 v4.21.0
 	github.com/go-playground/validator/v10 v10.24.0

go.sum

@@ -27,10 +27,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/cli v27.5.0+incompatible h1:aMphQkcGtpHixwwhAXJT1rrK/detk2JIvDaFkLctbGM=
+github.com/docker/cli v27.5.1+incompatible h1:JB9cieUT9YNiMITtIsguaN55PLOHhBSz3LKVc6cqWaY=
-github.com/docker/cli v27.5.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v27.5.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/docker v27.5.0+incompatible h1:um++2NcQtGRTz5eEgO6aJimo6/JxrTXC941hd05JO6U=
+github.com/docker/docker v27.5.1+incompatible h1:4PYU5dnBYqRQi0294d1FBECqT9ECWeQAIfE8q4YnPY8=
-github.com/docker/docker v27.5.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v27.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

internal/api/v1/favicon/cache.go

@@ -0,0 +1,133 @@
+package favicon
+
+import (
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/common"
+	"github.com/yusing/go-proxy/internal/logging"
+	route "github.com/yusing/go-proxy/internal/route/types"
+	"github.com/yusing/go-proxy/internal/task"
+	"github.com/yusing/go-proxy/internal/utils"
+)
+
+type cacheEntry struct {
+	Icon       []byte    `json:"icon"`
+	LastAccess time.Time `json:"lastAccess"`
+}
+
+// cache key can be absolute url or route name.
+var (
+	iconCache   = make(map[string]*cacheEntry)
+	iconCacheMu sync.RWMutex
+)
+
+const (
+	iconCacheTTL    = 24 * time.Hour
+	cleanUpInterval = time.Hour
+)
+
+func InitIconCache() {
+	err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache)
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to load icon cache")
+	} else {
+		logging.Info().Msgf("icon cache loaded (%d icons)", len(iconCache))
+	}
+
+	go func() {
+		cleanupTicker := time.NewTicker(cleanUpInterval)
+		defer cleanupTicker.Stop()
+		for {
+			select {
+			case <-task.RootContextCanceled():
+				return
+			case <-cleanupTicker.C:
+				pruneExpiredIconCache()
+			}
+		}
+	}()
+
+	task.OnProgramExit("save_favicon_cache", func() {
+		iconCacheMu.Lock()
+		defer iconCacheMu.Unlock()
+
+		if len(iconCache) == 0 {
+			return
+		}
+
+		if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil {
+			logging.Error().Err(err).Msg("failed to save icon cache")
+		}
+	})
+}
+
+func pruneExpiredIconCache() {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+
+	nPruned := 0
+	for key, icon := range iconCache {
+		if icon.IsExpired() {
+			delete(iconCache, key)
+			nPruned++
+		}
+	}
+	logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache")
+}
+
+func routeKey(r route.HTTPRoute) string {
+	return r.RawEntry().Provider + ":" + r.TargetName()
+}
+
+func PruneRouteIconCache(route route.HTTPRoute) {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+	delete(iconCache, routeKey(route))
+}
+
+func loadIconCache(key string) *fetchResult {
+	iconCacheMu.RLock()
+	defer iconCacheMu.RUnlock()
+
+	icon, ok := iconCache[key]
+	if ok && icon != nil {
+		logging.Debug().
+			Str("key", key).
+			Msg("icon found in cache")
+		icon.LastAccess = time.Now()
+		return &fetchResult{icon: icon.Icon}
+	}
+	return nil
+}
+
+func storeIconCache(key string, icon []byte) {
+	iconCacheMu.Lock()
+	defer iconCacheMu.Unlock()
+	iconCache[key] = &cacheEntry{Icon: icon, LastAccess: time.Now()}
+}
+
+func (e *cacheEntry) IsExpired() bool {
+	return time.Since(e.LastAccess) > iconCacheTTL
+}
+
+func (e *cacheEntry) UnmarshalJSON(data []byte) error {
+	attempt := struct {
+		Icon       []byte    `json:"icon"`
+		LastAccess time.Time `json:"lastAccess"`
+	}{}
+	err := json.Unmarshal(data, &attempt)
+	if err == nil {
+		e.Icon = attempt.Icon
+		e.LastAccess = attempt.LastAccess
+		return nil
+	}
+	// fallback to bytes
+	err = json.Unmarshal(data, &e.Icon)
+	if err == nil {
+		e.LastAccess = time.Now()
+		return nil
+	}
+	return err
+}

internal/api/v1/favicon/content.go

@@ -0,0 +1,37 @@
+package favicon
+
+import (
+	"bufio"
+	"errors"
+	"net"
+	"net/http"
+)
+
+type content struct {
+	header http.Header
+	data   []byte
+	status int
+}
+
+func newContent() *content {
+	return &content{
+		header: make(http.Header),
+	}
+}
+
+func (c *content) Header() http.Header {
+	return c.header
+}
+
+func (c *content) Write(data []byte) (int, error) {
+	c.data = append(c.data, data...)
+	return len(data), nil
+}
+
+func (c *content) WriteHeader(statusCode int) {
+	c.status = statusCode
+}
+
+func (c *content) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	return nil, nil, errors.New("not supported")
+}

internal/api/v1/favicon/favicon.go

@@ -1,38 +1,26 @@
 package favicon
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"errors"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"path"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/vincent-petithory/dataurl"
 	U "github.com/yusing/go-proxy/internal/api/v1/utils"
-	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/homepage"
 	"github.com/yusing/go-proxy/internal/logging"
 	gphttp "github.com/yusing/go-proxy/internal/net/http"
 	"github.com/yusing/go-proxy/internal/route/routes"
 	route "github.com/yusing/go-proxy/internal/route/types"
-	"github.com/yusing/go-proxy/internal/task"
-	"github.com/yusing/go-proxy/internal/utils"
 )
 
-type content struct {
-	header http.Header
-	data   []byte
-	status int
-}
-
 type fetchResult struct {
 	icon        []byte
 	contentType string
@@ -40,29 +28,6 @@ type fetchResult struct {
 	errMsg      string
 }
 
-func newContent() *content {
-	return &content{
-		header: make(http.Header),
-	}
-}
-
-func (c *content) Header() http.Header {
-	return c.header
-}
-
-func (c *content) Write(data []byte) (int, error) {
-	c.data = append(c.data, data...)
-	return len(data), nil
-}
-
-func (c *content) WriteHeader(statusCode int) {
-	c.status = statusCode
-}
-
-func (c *content) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	return nil, nil, errors.New("not supported")
-}
-
 func (res *fetchResult) OK() bool {
 	return res.icon != nil
 }
@@ -156,60 +121,6 @@ func getFavIconFromURL(iconURL *homepage.IconURL) *fetchResult {
 	return &fetchResult{statusCode: http.StatusBadRequest, errMsg: "invalid icon source"}
 }
 
-// cache key can be absolute url or route name.
-var (
-	iconCache   = make(map[string][]byte)
-	iconCacheMu sync.RWMutex
-)
-
-func InitIconCache() {
-	err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache)
-	if err != nil {
-		logging.Error().Err(err).Msg("failed to load icon cache")
-	} else {
-		logging.Info().Msgf("icon cache loaded (%d icons)", len(iconCache))
-	}
-
-	task.OnProgramExit("save_favicon_cache", func() {
-		iconCacheMu.Lock()
-		defer iconCacheMu.Unlock()
-
-		if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil {
-			logging.Error().Err(err).Msg("failed to save icon cache")
-		}
-	})
-}
-
-func routeKey(r route.HTTPRoute) string {
-	return r.RawEntry().Provider + ":" + r.TargetName()
-}
-
-func ResetIconCache(route route.HTTPRoute) {
-	iconCacheMu.Lock()
-	defer iconCacheMu.Unlock()
-	delete(iconCache, routeKey(route))
-}
-
-func loadIconCache(key string) *fetchResult {
-	iconCacheMu.RLock()
-	defer iconCacheMu.RUnlock()
-	icon, ok := iconCache[key]
-	if ok && icon != nil {
-		logging.Debug().
-			Str("key", key).
-			Msg("icon found in cache")
-
-		return &fetchResult{icon: icon}
-	}
-	return nil
-}
-
-func storeIconCache(key string, icon []byte) {
-	iconCacheMu.Lock()
-	defer iconCacheMu.Unlock()
-	iconCache[key] = icon
-}
-
 func fetchIconAbsolute(url string) *fetchResult {
 	if result := loadIconCache(url); result != nil {
 		return result

internal/homepage/override_config.go

@@ -40,6 +40,9 @@ func InitOverridesConfig() {
 		logging.Info().Msgf("homepage overrides config loaded, %d items", len(overrideConfigInstance.ItemOverrides))
 	}
 	task.OnProgramExit("save_homepage_json_config", func() {
+		if len(overrideConfigInstance.ItemOverrides) == 0 {
+			return
+		}
 		if err := utils.SaveJSON(common.HomepageJSONConfigPath, overrideConfigInstance, 0o644); err != nil {
 			logging.Error().Err(err).Msg("failed to save homepage overrides config")
 		}

internal/route/http.go

@@ -163,7 +163,7 @@ func (r *HTTPRoute) Start(parent task.Parent) E.Error {
 		r.task.OnCancel("metrics_cleanup", r.rp.UnregisterMetrics)
 	}
 
-	r.task.OnCancel("reset_favicon", func() { favicon.ResetIconCache(r) })
+	r.task.OnCancel("reset_favicon", func() { favicon.PruneRouteIconCache(r) })
 	return nil
 }
 

internal/utils/serialization.go

@@ -429,12 +429,12 @@ func DeserializeJSON[T any](data []byte, target T) E.Error {
 	return Deserialize(m, target)
 }
 
-func LoadJSON[T any](path string, dst *T) error {
+func loadSerialized[T any](path string, dst *T, deserialize func(data []byte, dst any) error) error {
 	data, err := os.ReadFile(path)
 	if err != nil {
 		return err
 	}
-	return json.Unmarshal(data, dst)
+	return deserialize(data, dst)
 }
 
 func SaveJSON[T any](path string, src *T, perm os.FileMode) error {
@@ -453,5 +453,5 @@ func LoadJSONIfExist[T any](path string, dst *T) error {
 		}
 		return err
 	}
-	return LoadJSON(path, dst)
+	return loadSerialized(path, dst, json.Unmarshal)
 }