TyleoDvDelaware3217/GoDoxy
1
0
Fork 0
mirror of https://github.com/yusing/godoxy.git synced 2025-05-19 20:32:35 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: simplify and remove duplicated code for icon caching

Browse source
This commit is contained in:
yusing 2025-04-28 11:22:49 +08:00
parent c5fd21552e
commit 8da63daf02
7 changed files with 45 additions and 354 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/main.go
View file

@ -6,13 +6,13 @@ import (
"os"
"sync"
"github.com/yusing/go-proxy/internal"
"github.com/yusing/go-proxy/internal/api/v1/query"
"github.com/yusing/go-proxy/internal/auth"
"github.com/yusing/go-proxy/internal/common"
"github.com/yusing/go-proxy/internal/config"
"github.com/yusing/go-proxy/internal/dnsproviders"
"github.com/yusing/go-proxy/internal/gperr"
"github.com/yusing/go-proxy/internal/homepage"
"github.com/yusing/go-proxy/internal/logging"
"github.com/yusing/go-proxy/internal/logging/memlogger"
"github.com/yusing/go-proxy/internal/metrics/systeminfo"
@ -50,7 +50,7 @@ func main() {
rawLogger.Println("ok")
return
case common.CommandListIcons:
icons, err := internal.ListAvailableIcons()
icons, err := homepage.ListAvailableIcons()
if err != nil {
rawLogger.Fatal(err)
}
@ -79,7 +79,7 @@ func main() {
logging.Info().Msgf("GoDoxy version %s", pkg.GetVersion())
logging.Trace().Msg("trace enabled")
parallel(
internal.InitIconListCache,
homepage.InitIconListCache,
systeminfo.Poller.Start,
)

4
internal/api/v1/list.go
View file

@ -6,9 +6,9 @@ import (
"strconv"
"strings"
"github.com/yusing/go-proxy/internal"
"github.com/yusing/go-proxy/internal/common"
config "github.com/yusing/go-proxy/internal/config/types"
"github.com/yusing/go-proxy/internal/homepage"
"github.com/yusing/go-proxy/internal/net/gphttp"
"github.com/yusing/go-proxy/internal/net/gphttp/middleware"
"github.com/yusing/go-proxy/internal/route/routes"
@ -67,7 +67,7 @@ func List(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
if err != nil {
limit = 0
}
icons, err := internal.SearchIcons(r.FormValue("keyword"), limit)
icons, err := homepage.SearchIcons(r.FormValue("keyword"), limit)
if err != nil {
gphttp.ClientError(w, err)
return

1
internal/common/constants.go
View file

@ -16,7 +16,6 @@ const (
ConfigPath = ConfigBasePath + "/" + ConfigFileName
IconListCachePath = ConfigBasePath + "/.icon_list_cache.json"
IconCachePath = ConfigBasePath + "/.icon_cache.json"
NamespaceHomepageOverrides = ".homepage"
NamespaceIconCache = ".icon_cache"

83
internal/homepage/icon_cache.go
View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/yusing/go-proxy/internal/common"
"github.com/yusing/go-proxy/internal/jsonstore"
"github.com/yusing/go-proxy/internal/logging"
"github.com/yusing/go-proxy/internal/task"
"github.com/yusing/go-proxy/internal/utils"
@ -15,34 +16,24 @@ import (
type cacheEntry struct {
Icon []byte `json:"icon"`
ContentType string `json:"content_type"`
ContentType string `json:"content_type,omitempty"`
LastAccess atomic.Value[time.Time] `json:"last_access"`
}
// cache key can be absolute url or route name.
var (
iconCache = make(map[string]*cacheEntry)
iconCacheMu sync.RWMutex
iconCache = jsonstore.Store[*cacheEntry](common.NamespaceIconCache)
iconMu sync.RWMutex
)
const (
iconCacheTTL = 3 * 24 * time.Hour
cleanUpInterval = time.Minute
maxCacheSize = 1024 * 1024 // 1MB
maxIconSize = 1024 * 1024 // 1MB
maxCacheEntries = 100
)
func InitIconCache() {
iconCacheMu.Lock()
defer iconCacheMu.Unlock()
err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache)
if err != nil {
logging.Error().Err(err).Msg("failed to load icon cache")
} else if len(iconCache) > 0 {
logging.Info().Int("count", len(iconCache)).Msg("icon cache loaded")
}
func init() {
go func() {
cleanupTicker := time.NewTicker(cleanUpInterval)
defer cleanupTicker.Stop()
@ -55,36 +46,21 @@ func InitIconCache() {
}
}
}()
task.OnProgramExit("save_favicon_cache", func() {
iconCacheMu.Lock()
defer iconCacheMu.Unlock()
if len(iconCache) == 0 {
return
}
if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil {
logging.Error().Err(err).Msg("failed to save icon cache")
}
})
}
func pruneExpiredIconCache() {
iconCacheMu.Lock()
defer iconCacheMu.Unlock()
nPruned := 0
for key, icon := range iconCache {
for key, icon := range iconCache.Range {
if icon.IsExpired() {
delete(iconCache, key)
iconCache.Delete(key)
nPruned++
}
}
if len(iconCache) > maxCacheEntries {
if iconCache.Size() > maxCacheEntries {
iconCache.Clear()
newIconCache := make(map[string]*cacheEntry, maxCacheEntries)
i := 0
for key, icon := range iconCache {
for key, icon := range iconCache.Range {
if i == maxCacheEntries {
break
}
@ -93,7 +69,9 @@ func pruneExpiredIconCache() {
i++
}
}
iconCache = newIconCache
for key, icon := range newIconCache {
iconCache.Store(key, icon)
}
}
if nPruned > 0 {
logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache")
@ -101,21 +79,18 @@ func pruneExpiredIconCache() {
}
func PruneRouteIconCache(route route) {
iconCacheMu.Lock()
defer iconCacheMu.Unlock()
delete(iconCache, route.Key())
iconCache.Delete(route.Key())
}
func loadIconCache(key string) *FetchResult {
iconCacheMu.RLock()
defer iconCacheMu.RUnlock()
icon, ok := iconCache[key]
iconMu.RLock()
defer iconMu.RUnlock()
icon, ok := iconCache.Load(key)
if ok && len(icon.Icon) > 0 {
logging.Debug().
Str("key", key).
Msg("icon found in cache")
icon.LastAccess.Store(time.Now())
icon.LastAccess.Store(utils.TimeNow())
return &FetchResult{Icon: icon.Icon, contentType: icon.ContentType}
}
return nil
@ -123,15 +98,17 @@ func loadIconCache(key string) *FetchResult {
func storeIconCache(key string, result *FetchResult) {
icon := result.Icon
if len(icon) > maxCacheSize {
if len(icon) > maxIconSize {
logging.Debug().Int("size", len(icon)).Msg("icon cache size exceeds max cache size")
return
}
iconCacheMu.Lock()
defer iconCacheMu.Unlock()
iconMu.Lock()
defer iconMu.Unlock()
entry := &cacheEntry{Icon: icon, ContentType: result.contentType}
entry.LastAccess.Store(time.Now())
iconCache[key] = entry
iconCache.Store(key, entry)
logging.Debug().Str("key", key).Int("size", len(icon)).Msg("stored icon cache")
}
@ -140,12 +117,20 @@ func (e *cacheEntry) IsExpired() bool {
}
func (e *cacheEntry) UnmarshalJSON(data []byte) error {
var tmp struct {
Icon []byte `json:"icon"`
ContentType string `json:"content_type,omitempty"`
LastAccess time.Time `json:"last_access"`
}
// check if data is json
if json.Valid(data) {
err := json.Unmarshal(data, &e)
err := json.Unmarshal(data, &tmp)
// return only if unmarshal is successful
// otherwise fallback to base64
if err == nil {
e.Icon = tmp.Icon
e.ContentType = tmp.ContentType
e.LastAccess.Store(tmp.LastAccess)
return nil
}
}

5
internal/homepage/list-icons.go
View file

@ -10,6 +10,7 @@ import (
"github.com/lithammer/fuzzysearch/fuzzy"
"github.com/yusing/go-proxy/internal/common"
"github.com/yusing/go-proxy/internal/logging"
"github.com/yusing/go-proxy/internal/task"
"github.com/yusing/go-proxy/internal/utils"
)
@ -68,6 +69,10 @@ func InitIconListCache() {
Int("display_names", len(iconsCache.DisplayNames)).
Msg("icon list cache loaded")
}
task.OnProgramExit("save_icon_list_cache", func() {
utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644)
})
}
func ListAvailableIcons() (*Cache, error) {

297
internal/list-icons.go
View file

@ -1,297 +0,0 @@
package internal
import (
"encoding/json"
"io"
"net/http"
"sync"
"time"
"github.com/lithammer/fuzzysearch/fuzzy"
"github.com/yusing/go-proxy/internal/common"
"github.com/yusing/go-proxy/internal/logging"
"github.com/yusing/go-proxy/internal/utils"
)
type GitHubContents struct { //! keep this, may reuse in future
Type string `json:"type"`
Path string `json:"path"`
Name string `json:"name"`
Sha string `json:"sha"`
Size int `json:"size"`
}
type (
IconsMap map[string]map[string]struct{}
IconList []string
Cache struct {
WalkxCode, Selfhst IconsMap
DisplayNames ReferenceDisplayNameMap
IconList IconList // combined into a single list
}
ReferenceDisplayNameMap map[string]string
)
func (icons *Cache) needUpdate() bool {
return len(icons.WalkxCode) == 0 || len(icons.Selfhst) == 0 || len(icons.IconList) == 0 || len(icons.DisplayNames) == 0
}
const updateInterval = 2 * time.Hour
var (
iconsCache *Cache
iconsCahceMu sync.RWMutex
lastUpdate time.Time
)
const (
walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json"
selfhstIcons = "https://cdn.selfh.st/directory/icons.json"
)
func InitIconListCache() {
iconsCahceMu.Lock()
defer iconsCahceMu.Unlock()
iconsCache = &Cache{
WalkxCode: make(IconsMap),
Selfhst: make(IconsMap),
DisplayNames: make(ReferenceDisplayNameMap),
IconList: []string{},
}
err := utils.LoadJSONIfExist(common.IconListCachePath, iconsCache)
if err != nil {
logging.Error().Err(err).Msg("failed to load icon list cache config")
} else if len(iconsCache.IconList) > 0 {
logging.Info().
Int("icons", len(iconsCache.IconList)).
Int("display_names", len(iconsCache.DisplayNames)).
Msg("icon list cache loaded")
}
}
func ListAvailableIcons() (*Cache, error) {
iconsCahceMu.RLock()
if time.Since(lastUpdate) < updateInterval {
if !iconsCache.needUpdate() {
iconsCahceMu.RUnlock()
return iconsCache, nil
}
}
iconsCahceMu.RUnlock()
iconsCahceMu.Lock()
defer iconsCahceMu.Unlock()
logging.Info().Msg("updating icon data")
icons, err := fetchIconData()
if err != nil {
return nil, err
}
logging.Info().
Int("icons", len(icons.IconList)).
Int("display_names", len(icons.DisplayNames)).
Msg("icons list updated")
iconsCache = icons
lastUpdate = time.Now()
err = utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644)
if err != nil {
logging.Warn().Err(err).Msg("failed to save icon list cache")
}
return icons, nil
}
func SearchIcons(keyword string, limit int) ([]string, error) {
icons, err := ListAvailableIcons()
if err != nil {
return nil, err
}
if keyword == "" {
return utils.Slice(icons.IconList, limit), nil
}
return utils.Slice(fuzzy.Find(keyword, icons.IconList), limit), nil
}
func HasWalkxCodeIcon(name string, filetype string) bool {
icons, err := ListAvailableIcons()
if err != nil {
logging.Error().Err(err).Msg("failed to list icons")
return false
}
if _, ok := icons.WalkxCode[filetype]; !ok {
return false
}
_, ok := icons.WalkxCode[filetype][name+"."+filetype]
return ok
}
func HasSelfhstIcon(name string, filetype string) bool {
icons, err := ListAvailableIcons()
if err != nil {
logging.Error().Err(err).Msg("failed to list icons")
return false
}
if _, ok := icons.Selfhst[filetype]; !ok {
return false
}
_, ok := icons.Selfhst[filetype][name+"."+filetype]
return ok
}
func GetDisplayName(reference string) (string, bool) {
icons, err := ListAvailableIcons()
if err != nil {
logging.Error().Err(err).Msg("failed to list icons")
return "", false
}
displayName, ok := icons.DisplayNames[reference]
return displayName, ok
}
func fetchIconData() (*Cache, error) {
walkxCodeIconMap, walkxCodeIconList, err := fetchWalkxCodeIcons()
if err != nil {
return nil, err
}
n := 0
for _, items := range walkxCodeIconMap {
n += len(items)
}
selfhstIconMap, selfhstIconList, referenceToNames, err := fetchSelfhstIcons()
if err != nil {
return nil, err
}
return &Cache{
WalkxCode: walkxCodeIconMap,
Selfhst: selfhstIconMap,
DisplayNames: referenceToNames,
IconList: append(walkxCodeIconList, selfhstIconList...),
}, nil
}
/*
format:
{
"png": [
"*.png",
],
"svg": [
"*.svg",
],
"webp": [
"*.webp",
]
}
*/
func fetchWalkxCodeIcons() (IconsMap, IconList, error) {
req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil)
if err != nil {
return nil, nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, err
}
data := make(map[string][]string)
err = json.Unmarshal(body, &data)
if err != nil {
return nil, nil, err
}
icons := make(IconsMap, len(data))
iconList := make(IconList, 0, 2000)
for fileType, files := range data {
icons[fileType] = make(map[string]struct{}, len(files))
for _, icon := range files {
icons[fileType][icon] = struct{}{}
iconList = append(iconList, "@walkxcode/"+icon)
}
}
return icons, iconList, nil
}
/*
format:
{
"Name": "2FAuth",
"Reference": "2fauth",
"SVG": "Yes",
"PNG": "Yes",
"WebP": "Yes",
"Light": "Yes",
"Category": "Self-Hosted",
"CreatedAt": "2024-08-16 00:27:23+00:00"
}
*/
func fetchSelfhstIcons() (IconsMap, IconList, ReferenceDisplayNameMap, error) {
type SelfhStIcon struct {
Name string `json:"Name"`
Reference string `json:"Reference"`
SVG string `json:"SVG"`
PNG string `json:"PNG"`
WebP string `json:"WebP"`
// Light string
// Category string
// CreatedAt string
}
req, err := http.NewRequest(http.MethodGet, selfhstIcons, nil)
if err != nil {
return nil, nil, nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, nil, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, nil, err
}
data := make([]SelfhStIcon, 0, 2000)
err = json.Unmarshal(body, &data)
if err != nil {
return nil, nil, nil, err
}
iconList := make(IconList, 0, len(data)*3)
icons := make(IconsMap)
icons["svg"] = make(map[string]struct{}, len(data))
icons["png"] = make(map[string]struct{}, len(data))
icons["webp"] = make(map[string]struct{}, len(data))
referenceToNames := make(ReferenceDisplayNameMap, len(data))
for _, item := range data {
if item.SVG == "Yes" {
icons["svg"][item.Reference+".svg"] = struct{}{}
iconList = append(iconList, "@selfhst/"+item.Reference+".svg")
}
if item.PNG == "Yes" {
icons["png"][item.Reference+".png"] = struct{}{}
iconList = append(iconList, "@selfhst/"+item.Reference+".png")
}
if item.WebP == "Yes" {
icons["webp"][item.Reference+".webp"] = struct{}{}
iconList = append(iconList, "@selfhst/"+item.Reference+".webp")
}
referenceToNames[item.Reference] = item.Name
}
return icons, iconList, referenceToNames, nil
}

3
internal/route/route.go
View file

@ -8,7 +8,6 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/yusing/go-proxy/agent/pkg/agent"
"github.com/yusing/go-proxy/internal"
"github.com/yusing/go-proxy/internal/docker"
"github.com/yusing/go-proxy/internal/gperr"
"github.com/yusing/go-proxy/internal/homepage"
@ -484,7 +483,7 @@ func (r *Route) FinalizeHomepageConfig() {
} else {
key = r.Alias
}
displayName, ok := internal.GetDisplayName(key)
displayName, ok := homepage.GetDisplayName(key)
if ok {
hp.Name = displayName
} else {