From 8da63daf02029d733d26169696eb56d7bae31545 Mon Sep 17 00:00:00 2001 From: yusing Date: Mon, 28 Apr 2025 11:22:49 +0800 Subject: [PATCH] refactor: simplify and remove duplicated code for icon caching --- cmd/main.go | 6 +- internal/api/v1/list.go | 4 +- internal/common/constants.go | 1 - internal/homepage/icon_cache.go | 83 ++++----- internal/homepage/list-icons.go | 5 + internal/list-icons.go | 297 -------------------------------- internal/route/route.go | 3 +- 7 files changed, 45 insertions(+), 354 deletions(-) delete mode 100644 internal/list-icons.go diff --git a/cmd/main.go b/cmd/main.go index 4c00570..681dbbc 100755 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,13 +6,13 @@ import ( "os" "sync" - "github.com/yusing/go-proxy/internal" "github.com/yusing/go-proxy/internal/api/v1/query" "github.com/yusing/go-proxy/internal/auth" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/config" "github.com/yusing/go-proxy/internal/dnsproviders" "github.com/yusing/go-proxy/internal/gperr" + "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/logging/memlogger" "github.com/yusing/go-proxy/internal/metrics/systeminfo" @@ -50,7 +50,7 @@ func main() { rawLogger.Println("ok") return case common.CommandListIcons: - icons, err := internal.ListAvailableIcons() + icons, err := homepage.ListAvailableIcons() if err != nil { rawLogger.Fatal(err) } @@ -79,7 +79,7 @@ func main() { logging.Info().Msgf("GoDoxy version %s", pkg.GetVersion()) logging.Trace().Msg("trace enabled") parallel( - internal.InitIconListCache, + homepage.InitIconListCache, systeminfo.Poller.Start, ) diff --git a/internal/api/v1/list.go b/internal/api/v1/list.go index 918daea..99cb1b1 100644 --- a/internal/api/v1/list.go +++ b/internal/api/v1/list.go @@ -6,9 +6,9 @@ import ( "strconv" "strings" - "github.com/yusing/go-proxy/internal" "github.com/yusing/go-proxy/internal/common" config "github.com/yusing/go-proxy/internal/config/types" + "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/net/gphttp" "github.com/yusing/go-proxy/internal/net/gphttp/middleware" "github.com/yusing/go-proxy/internal/route/routes" @@ -67,7 +67,7 @@ func List(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) { if err != nil { limit = 0 } - icons, err := internal.SearchIcons(r.FormValue("keyword"), limit) + icons, err := homepage.SearchIcons(r.FormValue("keyword"), limit) if err != nil { gphttp.ClientError(w, err) return diff --git a/internal/common/constants.go b/internal/common/constants.go index 254b1d6..6ddf633 100644 --- a/internal/common/constants.go +++ b/internal/common/constants.go @@ -16,7 +16,6 @@ const ( ConfigPath = ConfigBasePath + "/" + ConfigFileName IconListCachePath = ConfigBasePath + "/.icon_list_cache.json" - IconCachePath = ConfigBasePath + "/.icon_cache.json" NamespaceHomepageOverrides = ".homepage" NamespaceIconCache = ".icon_cache" diff --git a/internal/homepage/icon_cache.go b/internal/homepage/icon_cache.go index d7b2bc4..4f0f658 100644 --- a/internal/homepage/icon_cache.go +++ b/internal/homepage/icon_cache.go @@ -7,6 +7,7 @@ import ( "time" "github.com/yusing/go-proxy/internal/common" + "github.com/yusing/go-proxy/internal/jsonstore" "github.com/yusing/go-proxy/internal/logging" "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/utils" @@ -15,34 +16,24 @@ import ( type cacheEntry struct { Icon []byte `json:"icon"` - ContentType string `json:"content_type"` + ContentType string `json:"content_type,omitempty"` LastAccess atomic.Value[time.Time] `json:"last_access"` } // cache key can be absolute url or route name. var ( - iconCache = make(map[string]*cacheEntry) - iconCacheMu sync.RWMutex + iconCache = jsonstore.Store[*cacheEntry](common.NamespaceIconCache) + iconMu sync.RWMutex ) const ( iconCacheTTL = 3 * 24 * time.Hour cleanUpInterval = time.Minute - maxCacheSize = 1024 * 1024 // 1MB + maxIconSize = 1024 * 1024 // 1MB maxCacheEntries = 100 ) -func InitIconCache() { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - - err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache) - if err != nil { - logging.Error().Err(err).Msg("failed to load icon cache") - } else if len(iconCache) > 0 { - logging.Info().Int("count", len(iconCache)).Msg("icon cache loaded") - } - +func init() { go func() { cleanupTicker := time.NewTicker(cleanUpInterval) defer cleanupTicker.Stop() @@ -55,36 +46,21 @@ func InitIconCache() { } } }() - - task.OnProgramExit("save_favicon_cache", func() { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - - if len(iconCache) == 0 { - return - } - - if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil { - logging.Error().Err(err).Msg("failed to save icon cache") - } - }) } func pruneExpiredIconCache() { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - nPruned := 0 - for key, icon := range iconCache { + for key, icon := range iconCache.Range { if icon.IsExpired() { - delete(iconCache, key) + iconCache.Delete(key) nPruned++ } } - if len(iconCache) > maxCacheEntries { + if iconCache.Size() > maxCacheEntries { + iconCache.Clear() newIconCache := make(map[string]*cacheEntry, maxCacheEntries) i := 0 - for key, icon := range iconCache { + for key, icon := range iconCache.Range { if i == maxCacheEntries { break } @@ -93,7 +69,9 @@ func pruneExpiredIconCache() { i++ } } - iconCache = newIconCache + for key, icon := range newIconCache { + iconCache.Store(key, icon) + } } if nPruned > 0 { logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache") @@ -101,21 +79,18 @@ func pruneExpiredIconCache() { } func PruneRouteIconCache(route route) { - iconCacheMu.Lock() - defer iconCacheMu.Unlock() - delete(iconCache, route.Key()) + iconCache.Delete(route.Key()) } func loadIconCache(key string) *FetchResult { - iconCacheMu.RLock() - defer iconCacheMu.RUnlock() - - icon, ok := iconCache[key] + iconMu.RLock() + defer iconMu.RUnlock() + icon, ok := iconCache.Load(key) if ok && len(icon.Icon) > 0 { logging.Debug(). Str("key", key). Msg("icon found in cache") - icon.LastAccess.Store(time.Now()) + icon.LastAccess.Store(utils.TimeNow()) return &FetchResult{Icon: icon.Icon, contentType: icon.ContentType} } return nil @@ -123,15 +98,17 @@ func loadIconCache(key string) *FetchResult { func storeIconCache(key string, result *FetchResult) { icon := result.Icon - if len(icon) > maxCacheSize { + if len(icon) > maxIconSize { logging.Debug().Int("size", len(icon)).Msg("icon cache size exceeds max cache size") return } - iconCacheMu.Lock() - defer iconCacheMu.Unlock() + + iconMu.Lock() + defer iconMu.Unlock() + entry := &cacheEntry{Icon: icon, ContentType: result.contentType} entry.LastAccess.Store(time.Now()) - iconCache[key] = entry + iconCache.Store(key, entry) logging.Debug().Str("key", key).Int("size", len(icon)).Msg("stored icon cache") } @@ -140,12 +117,20 @@ func (e *cacheEntry) IsExpired() bool { } func (e *cacheEntry) UnmarshalJSON(data []byte) error { + var tmp struct { + Icon []byte `json:"icon"` + ContentType string `json:"content_type,omitempty"` + LastAccess time.Time `json:"last_access"` + } // check if data is json if json.Valid(data) { - err := json.Unmarshal(data, &e) + err := json.Unmarshal(data, &tmp) // return only if unmarshal is successful // otherwise fallback to base64 if err == nil { + e.Icon = tmp.Icon + e.ContentType = tmp.ContentType + e.LastAccess.Store(tmp.LastAccess) return nil } } diff --git a/internal/homepage/list-icons.go b/internal/homepage/list-icons.go index f5ae736..927d694 100644 --- a/internal/homepage/list-icons.go +++ b/internal/homepage/list-icons.go @@ -10,6 +10,7 @@ import ( "github.com/lithammer/fuzzysearch/fuzzy" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/logging" + "github.com/yusing/go-proxy/internal/task" "github.com/yusing/go-proxy/internal/utils" ) @@ -68,6 +69,10 @@ func InitIconListCache() { Int("display_names", len(iconsCache.DisplayNames)). Msg("icon list cache loaded") } + + task.OnProgramExit("save_icon_list_cache", func() { + utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644) + }) } func ListAvailableIcons() (*Cache, error) { diff --git a/internal/list-icons.go b/internal/list-icons.go deleted file mode 100644 index 1f3d2c6..0000000 --- a/internal/list-icons.go +++ /dev/null @@ -1,297 +0,0 @@ -package internal - -import ( - "encoding/json" - "io" - "net/http" - "sync" - "time" - - "github.com/lithammer/fuzzysearch/fuzzy" - "github.com/yusing/go-proxy/internal/common" - "github.com/yusing/go-proxy/internal/logging" - "github.com/yusing/go-proxy/internal/utils" -) - -type GitHubContents struct { //! keep this, may reuse in future - Type string `json:"type"` - Path string `json:"path"` - Name string `json:"name"` - Sha string `json:"sha"` - Size int `json:"size"` -} - -type ( - IconsMap map[string]map[string]struct{} - IconList []string - Cache struct { - WalkxCode, Selfhst IconsMap - DisplayNames ReferenceDisplayNameMap - IconList IconList // combined into a single list - } - ReferenceDisplayNameMap map[string]string -) - -func (icons *Cache) needUpdate() bool { - return len(icons.WalkxCode) == 0 || len(icons.Selfhst) == 0 || len(icons.IconList) == 0 || len(icons.DisplayNames) == 0 -} - -const updateInterval = 2 * time.Hour - -var ( - iconsCache *Cache - iconsCahceMu sync.RWMutex - lastUpdate time.Time -) - -const ( - walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json" - selfhstIcons = "https://cdn.selfh.st/directory/icons.json" -) - -func InitIconListCache() { - iconsCahceMu.Lock() - defer iconsCahceMu.Unlock() - - iconsCache = &Cache{ - WalkxCode: make(IconsMap), - Selfhst: make(IconsMap), - DisplayNames: make(ReferenceDisplayNameMap), - IconList: []string{}, - } - err := utils.LoadJSONIfExist(common.IconListCachePath, iconsCache) - if err != nil { - logging.Error().Err(err).Msg("failed to load icon list cache config") - } else if len(iconsCache.IconList) > 0 { - logging.Info(). - Int("icons", len(iconsCache.IconList)). - Int("display_names", len(iconsCache.DisplayNames)). - Msg("icon list cache loaded") - } -} - -func ListAvailableIcons() (*Cache, error) { - iconsCahceMu.RLock() - if time.Since(lastUpdate) < updateInterval { - if !iconsCache.needUpdate() { - iconsCahceMu.RUnlock() - return iconsCache, nil - } - } - iconsCahceMu.RUnlock() - - iconsCahceMu.Lock() - defer iconsCahceMu.Unlock() - - logging.Info().Msg("updating icon data") - icons, err := fetchIconData() - if err != nil { - return nil, err - } - logging.Info(). - Int("icons", len(icons.IconList)). - Int("display_names", len(icons.DisplayNames)). - Msg("icons list updated") - - iconsCache = icons - lastUpdate = time.Now() - - err = utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644) - if err != nil { - logging.Warn().Err(err).Msg("failed to save icon list cache") - } - return icons, nil -} - -func SearchIcons(keyword string, limit int) ([]string, error) { - icons, err := ListAvailableIcons() - if err != nil { - return nil, err - } - if keyword == "" { - return utils.Slice(icons.IconList, limit), nil - } - return utils.Slice(fuzzy.Find(keyword, icons.IconList), limit), nil -} - -func HasWalkxCodeIcon(name string, filetype string) bool { - icons, err := ListAvailableIcons() - if err != nil { - logging.Error().Err(err).Msg("failed to list icons") - return false - } - if _, ok := icons.WalkxCode[filetype]; !ok { - return false - } - _, ok := icons.WalkxCode[filetype][name+"."+filetype] - return ok -} - -func HasSelfhstIcon(name string, filetype string) bool { - icons, err := ListAvailableIcons() - if err != nil { - logging.Error().Err(err).Msg("failed to list icons") - return false - } - if _, ok := icons.Selfhst[filetype]; !ok { - return false - } - _, ok := icons.Selfhst[filetype][name+"."+filetype] - return ok -} - -func GetDisplayName(reference string) (string, bool) { - icons, err := ListAvailableIcons() - if err != nil { - logging.Error().Err(err).Msg("failed to list icons") - return "", false - } - displayName, ok := icons.DisplayNames[reference] - return displayName, ok -} - -func fetchIconData() (*Cache, error) { - walkxCodeIconMap, walkxCodeIconList, err := fetchWalkxCodeIcons() - if err != nil { - return nil, err - } - - n := 0 - for _, items := range walkxCodeIconMap { - n += len(items) - } - - selfhstIconMap, selfhstIconList, referenceToNames, err := fetchSelfhstIcons() - if err != nil { - return nil, err - } - - return &Cache{ - WalkxCode: walkxCodeIconMap, - Selfhst: selfhstIconMap, - DisplayNames: referenceToNames, - IconList: append(walkxCodeIconList, selfhstIconList...), - }, nil -} - -/* -format: - - { - "png": [ - "*.png", - ], - "svg": [ - "*.svg", - ], - "webp": [ - "*.webp", - ] - } -*/ -func fetchWalkxCodeIcons() (IconsMap, IconList, error) { - req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil) - if err != nil { - return nil, nil, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - data := make(map[string][]string) - err = json.Unmarshal(body, &data) - if err != nil { - return nil, nil, err - } - icons := make(IconsMap, len(data)) - iconList := make(IconList, 0, 2000) - for fileType, files := range data { - icons[fileType] = make(map[string]struct{}, len(files)) - for _, icon := range files { - icons[fileType][icon] = struct{}{} - iconList = append(iconList, "@walkxcode/"+icon) - } - } - return icons, iconList, nil -} - -/* -format: - - { - "Name": "2FAuth", - "Reference": "2fauth", - "SVG": "Yes", - "PNG": "Yes", - "WebP": "Yes", - "Light": "Yes", - "Category": "Self-Hosted", - "CreatedAt": "2024-08-16 00:27:23+00:00" - } -*/ -func fetchSelfhstIcons() (IconsMap, IconList, ReferenceDisplayNameMap, error) { - type SelfhStIcon struct { - Name string `json:"Name"` - Reference string `json:"Reference"` - SVG string `json:"SVG"` - PNG string `json:"PNG"` - WebP string `json:"WebP"` - // Light string - // Category string - // CreatedAt string - } - - req, err := http.NewRequest(http.MethodGet, selfhstIcons, nil) - if err != nil { - return nil, nil, nil, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, nil, nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, nil, err - } - - data := make([]SelfhStIcon, 0, 2000) - err = json.Unmarshal(body, &data) - if err != nil { - return nil, nil, nil, err - } - - iconList := make(IconList, 0, len(data)*3) - icons := make(IconsMap) - icons["svg"] = make(map[string]struct{}, len(data)) - icons["png"] = make(map[string]struct{}, len(data)) - icons["webp"] = make(map[string]struct{}, len(data)) - - referenceToNames := make(ReferenceDisplayNameMap, len(data)) - - for _, item := range data { - if item.SVG == "Yes" { - icons["svg"][item.Reference+".svg"] = struct{}{} - iconList = append(iconList, "@selfhst/"+item.Reference+".svg") - } - if item.PNG == "Yes" { - icons["png"][item.Reference+".png"] = struct{}{} - iconList = append(iconList, "@selfhst/"+item.Reference+".png") - } - if item.WebP == "Yes" { - icons["webp"][item.Reference+".webp"] = struct{}{} - iconList = append(iconList, "@selfhst/"+item.Reference+".webp") - } - referenceToNames[item.Reference] = item.Name - } - - return icons, iconList, referenceToNames, nil -} diff --git a/internal/route/route.go b/internal/route/route.go index 65ee8b7..4834d21 100644 --- a/internal/route/route.go +++ b/internal/route/route.go @@ -8,7 +8,6 @@ import ( "github.com/docker/docker/api/types/container" "github.com/yusing/go-proxy/agent/pkg/agent" - "github.com/yusing/go-proxy/internal" "github.com/yusing/go-proxy/internal/docker" "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/homepage" @@ -484,7 +483,7 @@ func (r *Route) FinalizeHomepageConfig() { } else { key = r.Alias } - displayName, ok := internal.GetDisplayName(key) + displayName, ok := homepage.GetDisplayName(key) if ok { hp.Name = displayName } else {