From 5df255377411408328ee8f391d46cb9f2591a1a7 Mon Sep 17 00:00:00 2001 From: yusing Date: Thu, 24 Apr 2025 15:34:47 +0800 Subject: [PATCH] merge: better favicon handling --- internal/api/v1/favicon/cache.go | 101 ------ internal/api/v1/favicon/favicon.go | 232 +------------- .../{api/v1/favicon => homepage}/content.go | 2 +- internal/homepage/favicon.go | 222 +++++++++++++ internal/homepage/icon_cache.go | 160 ++++++++++ internal/homepage/icon_url.go | 5 +- internal/homepage/icon_url_test.go | 8 +- internal/homepage/list-icons.go | 297 ++++++++++++++++++ internal/homepage/route.go | 20 ++ internal/net/gphttp/default_client.go | 1 + internal/route/reverse_proxy.go | 4 +- 11 files changed, 722 insertions(+), 330 deletions(-) delete mode 100644 internal/api/v1/favicon/cache.go rename internal/{api/v1/favicon => homepage}/content.go (97%) create mode 100644 internal/homepage/favicon.go create mode 100644 internal/homepage/icon_cache.go create mode 100644 internal/homepage/list-icons.go create mode 100644 internal/homepage/route.go diff --git a/internal/api/v1/favicon/cache.go b/internal/api/v1/favicon/cache.go deleted file mode 100644 index 515fc94..0000000 --- a/internal/api/v1/favicon/cache.go +++ /dev/null @@ -1,101 +0,0 @@ -package favicon - -import ( - "encoding/json" - "time" - - "github.com/yusing/go-proxy/internal/common" - "github.com/yusing/go-proxy/internal/jsonstore" - "github.com/yusing/go-proxy/internal/logging" - "github.com/yusing/go-proxy/internal/route/routes" - "github.com/yusing/go-proxy/internal/task" -) - -type cacheEntry struct { - Icon []byte `json:"icon"` - LastAccess time.Time `json:"lastAccess"` -} - -// cache key can be absolute url or route name. -var iconCache = jsonstore.Store[*cacheEntry](common.NamespaceIconCache) - -const ( - iconCacheTTL = 3 * 24 * time.Hour - cleanUpInterval = time.Hour -) - -func init() { - go func() { - cleanupTicker := time.NewTicker(cleanUpInterval) - defer cleanupTicker.Stop() - for { - select { - case <-task.RootContextCanceled(): - return - case <-cleanupTicker.C: - pruneExpiredIconCache() - } - } - }() -} - -func pruneExpiredIconCache() { - nPruned := 0 - for key, icon := range iconCache.Range { - if icon.IsExpired() { - iconCache.Delete(key) - nPruned++ - } - } - if nPruned > 0 { - logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache") - } -} - -func routeKey(r routes.HTTPRoute) string { - return r.ProviderName() + ":" + r.Name() -} - -func PruneRouteIconCache(route routes.HTTPRoute) { - iconCache.Delete(routeKey(route)) -} - -func loadIconCache(key string) *fetchResult { - icon, ok := iconCache.Load(key) - if ok && icon != nil { - logging.Debug(). - Str("key", key). - Msg("icon found in cache") - icon.LastAccess = time.Now() - return &fetchResult{icon: icon.Icon} - } - return nil -} - -func storeIconCache(key string, icon []byte) { - iconCache.Store(key, &cacheEntry{Icon: icon, LastAccess: time.Now()}) -} - -func (e *cacheEntry) IsExpired() bool { - return time.Since(e.LastAccess) > iconCacheTTL -} - -func (e *cacheEntry) UnmarshalJSON(data []byte) error { - attempt := struct { - Icon []byte `json:"icon"` - LastAccess time.Time `json:"lastAccess"` - }{} - err := json.Unmarshal(data, &attempt) - if err == nil { - e.Icon = attempt.Icon - e.LastAccess = attempt.LastAccess - return nil - } - // fallback to bytes - err = json.Unmarshal(data, &e.Icon) - if err == nil { - e.LastAccess = time.Now() - return nil - } - return err -} diff --git a/internal/api/v1/favicon/favicon.go b/internal/api/v1/favicon/favicon.go index c3b5b6b..9b9d614 100644 --- a/internal/api/v1/favicon/favicon.go +++ b/internal/api/v1/favicon/favicon.go @@ -1,48 +1,13 @@ package favicon import ( - "bytes" - "context" "errors" - "io" "net/http" - "net/url" - "strings" - "time" - "github.com/PuerkitoBio/goquery" - "github.com/vincent-petithory/dataurl" "github.com/yusing/go-proxy/internal/gperr" "github.com/yusing/go-proxy/internal/homepage" - "github.com/yusing/go-proxy/internal/logging" - gphttp "github.com/yusing/go-proxy/internal/net/gphttp" + "github.com/yusing/go-proxy/internal/net/gphttp" "github.com/yusing/go-proxy/internal/route/routes" - "github.com/yusing/go-proxy/internal/utils/strutils" -) - -type fetchResult struct { - icon []byte - contentType string - statusCode int - errMsg string -} - -func (res *fetchResult) OK() bool { - return res.icon != nil -} - -func (res *fetchResult) ContentType() string { - if res.contentType == "" { - if bytes.HasPrefix(res.icon, []byte(" MaxRedirectDepth { - return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "too many redirects"} - } - loc = strutils.SanitizeURI(loc) - if loc == "/" || loc == newReq.URL.Path { - return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "circular redirect"} - } - return findIconSlow(r, req, loc, depth+1) - } - } - return &fetchResult{statusCode: c.status, errMsg: "upstream error: " + string(c.data)} - } - // return icon data - if !gphttp.GetContentType(c.header).IsHTML() { - return &fetchResult{icon: c.data, contentType: c.header.Get("Content-Type")} - } - // try extract from "link[rel=icon]" from path "/" - doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.data)) - if err != nil { - logging.Error().Err(err). - Str("route", r.Name()). - Msg("failed to parse html") - return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "internal error"} - } - ele := doc.Find("head > link[rel=icon]").First() - if ele.Length() == 0 { - return &fetchResult{statusCode: http.StatusNotFound, errMsg: "icon element not found"} - } - href := ele.AttrOr("href", "") - if href == "" { - return &fetchResult{statusCode: http.StatusNotFound, errMsg: "icon href not found"} - } - // https://en.wikipedia.org/wiki/Data_URI_scheme - if strings.HasPrefix(href, "data:image/") { - dataURI, err := dataurl.DecodeString(href) - if err != nil { - logging.Error().Err(err). - Str("route", r.Name()). - Msg("failed to decode favicon") - return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "internal error"} - } - return &fetchResult{icon: dataURI.Data, contentType: dataURI.ContentType()} - } - switch { - case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"): - return fetchIconAbsolute(href) - default: - return findIconSlow(r, req, href, 0) - } + gphttp.WriteBody(w, result.Icon) } diff --git a/internal/api/v1/favicon/content.go b/internal/homepage/content.go similarity index 97% rename from internal/api/v1/favicon/content.go rename to internal/homepage/content.go index 5d4ba3e..fa0ecdc 100644 --- a/internal/api/v1/favicon/content.go +++ b/internal/homepage/content.go @@ -1,4 +1,4 @@ -package favicon +package homepage import ( "bufio" diff --git a/internal/homepage/favicon.go b/internal/homepage/favicon.go new file mode 100644 index 0000000..e611022 --- /dev/null +++ b/internal/homepage/favicon.go @@ -0,0 +1,222 @@ +package homepage + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/url" + "slices" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/vincent-petithory/dataurl" + gphttp "github.com/yusing/go-proxy/internal/net/gphttp" + "github.com/yusing/go-proxy/internal/utils/strutils" +) + +type FetchResult struct { + Icon []byte + StatusCode int + ErrMsg string + + contentType string +} + +const faviconFetchTimeout = 3 * time.Second + +func (res *FetchResult) OK() bool { + return len(res.Icon) > 0 +} + +func (res *FetchResult) ContentType() string { + if res.contentType == "" { + if bytes.HasPrefix(res.Icon, []byte(" maxRedirectDepth { + return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "too many redirects"} + } + + ctx, cancel := context.WithTimeoutCause(ctx, faviconFetchTimeout, errors.New("favicon request timeout")) + defer cancel() + + newReq, err := http.NewRequestWithContext(ctx, "GET", r.TargetURL().String(), nil) + if err != nil { + return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot create request"} + } + newReq.Header.Set("Accept-Encoding", "identity") // disable compression + + u, err := url.ParseRequestURI(strutils.SanitizeURI(uri)) + if err != nil { + return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot parse uri"} + } + newReq.URL.Path = u.Path + newReq.URL.RawPath = u.RawPath + newReq.URL.RawQuery = u.RawQuery + newReq.RequestURI = u.String() + + c := newContent() + r.ServeHTTP(c, newReq) + if c.status != http.StatusOK { + switch c.status { + case 0: + return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "connection error"} + default: + if loc := c.Header().Get("Location"); loc != "" { + loc = strutils.SanitizeURI(loc) + if loc == "/" || loc == newReq.URL.Path || slices.Contains(stack, loc) { + return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "circular redirect"} + } + // append current path to stack + // handles redirect to the same path with different query + return findIconSlow(ctx, r, loc, append(stack, newReq.URL.Path)) + } + } + return &FetchResult{StatusCode: c.status, ErrMsg: "upstream error: " + string(c.data)} + } + // return icon data + if !gphttp.GetContentType(c.header).IsHTML() { + return &FetchResult{Icon: c.data, contentType: c.header.Get("Content-Type")} + } + // try extract from "link[rel=icon]" from path "/" + doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.data)) + if err != nil { + return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to parse html"} + } + ele := doc.Find("head > link[rel=icon]").First() + if ele.Length() == 0 { + return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon element not found"} + } + href := ele.AttrOr("href", "") + if href == "" { + return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon href not found"} + } + // https://en.wikipedia.org/wiki/Data_URI_scheme + if strings.HasPrefix(href, "data:image/") { + dataURI, err := dataurl.DecodeString(href) + if err != nil { + return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to decode favicon"} + } + return &FetchResult{Icon: dataURI.Data, contentType: dataURI.ContentType()} + } + switch { + case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"): + return fetchIconAbsolute(ctx, href) + default: + return findIconSlow(ctx, r, href, append(stack, newReq.URL.Path)) + } +} diff --git a/internal/homepage/icon_cache.go b/internal/homepage/icon_cache.go new file mode 100644 index 0000000..d7b2bc4 --- /dev/null +++ b/internal/homepage/icon_cache.go @@ -0,0 +1,160 @@ +package homepage + +import ( + "encoding/base64" + "encoding/json" + "sync" + "time" + + "github.com/yusing/go-proxy/internal/common" + "github.com/yusing/go-proxy/internal/logging" + "github.com/yusing/go-proxy/internal/task" + "github.com/yusing/go-proxy/internal/utils" + "github.com/yusing/go-proxy/internal/utils/atomic" +) + +type cacheEntry struct { + Icon []byte `json:"icon"` + ContentType string `json:"content_type"` + LastAccess atomic.Value[time.Time] `json:"last_access"` +} + +// cache key can be absolute url or route name. +var ( + iconCache = make(map[string]*cacheEntry) + iconCacheMu sync.RWMutex +) + +const ( + iconCacheTTL = 3 * 24 * time.Hour + cleanUpInterval = time.Minute + maxCacheSize = 1024 * 1024 // 1MB + maxCacheEntries = 100 +) + +func InitIconCache() { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + + err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache) + if err != nil { + logging.Error().Err(err).Msg("failed to load icon cache") + } else if len(iconCache) > 0 { + logging.Info().Int("count", len(iconCache)).Msg("icon cache loaded") + } + + go func() { + cleanupTicker := time.NewTicker(cleanUpInterval) + defer cleanupTicker.Stop() + for { + select { + case <-task.RootContextCanceled(): + return + case <-cleanupTicker.C: + pruneExpiredIconCache() + } + } + }() + + task.OnProgramExit("save_favicon_cache", func() { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + + if len(iconCache) == 0 { + return + } + + if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil { + logging.Error().Err(err).Msg("failed to save icon cache") + } + }) +} + +func pruneExpiredIconCache() { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + + nPruned := 0 + for key, icon := range iconCache { + if icon.IsExpired() { + delete(iconCache, key) + nPruned++ + } + } + if len(iconCache) > maxCacheEntries { + newIconCache := make(map[string]*cacheEntry, maxCacheEntries) + i := 0 + for key, icon := range iconCache { + if i == maxCacheEntries { + break + } + if !icon.IsExpired() { + newIconCache[key] = icon + i++ + } + } + iconCache = newIconCache + } + if nPruned > 0 { + logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache") + } +} + +func PruneRouteIconCache(route route) { + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + delete(iconCache, route.Key()) +} + +func loadIconCache(key string) *FetchResult { + iconCacheMu.RLock() + defer iconCacheMu.RUnlock() + + icon, ok := iconCache[key] + if ok && len(icon.Icon) > 0 { + logging.Debug(). + Str("key", key). + Msg("icon found in cache") + icon.LastAccess.Store(time.Now()) + return &FetchResult{Icon: icon.Icon, contentType: icon.ContentType} + } + return nil +} + +func storeIconCache(key string, result *FetchResult) { + icon := result.Icon + if len(icon) > maxCacheSize { + logging.Debug().Int("size", len(icon)).Msg("icon cache size exceeds max cache size") + return + } + iconCacheMu.Lock() + defer iconCacheMu.Unlock() + entry := &cacheEntry{Icon: icon, ContentType: result.contentType} + entry.LastAccess.Store(time.Now()) + iconCache[key] = entry + logging.Debug().Str("key", key).Int("size", len(icon)).Msg("stored icon cache") +} + +func (e *cacheEntry) IsExpired() bool { + return time.Since(e.LastAccess.Load()) > iconCacheTTL +} + +func (e *cacheEntry) UnmarshalJSON(data []byte) error { + // check if data is json + if json.Valid(data) { + err := json.Unmarshal(data, &e) + // return only if unmarshal is successful + // otherwise fallback to base64 + if err == nil { + return nil + } + } + // fallback to base64 + icon, err := base64.StdEncoding.DecodeString(string(data)) + if err == nil { + e.Icon = icon + e.LastAccess.Store(time.Now()) + return nil + } + return err +} diff --git a/internal/homepage/icon_url.go b/internal/homepage/icon_url.go index f53060d..b6847aa 100644 --- a/internal/homepage/icon_url.go +++ b/internal/homepage/icon_url.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/yusing/go-proxy/internal" "github.com/yusing/go-proxy/internal/gperr" ) @@ -62,10 +61,10 @@ func NewWalkXCodeIconURL(name, format string) *IconURL { // otherwise returns true. func (u *IconURL) HasIcon() bool { if u.IconSource == IconSourceSelfhSt { - return internal.HasSelfhstIcon(u.Extra.Name, u.Extra.FileType) + return HasSelfhstIcon(u.Extra.Name, u.Extra.FileType) } if u.IconSource == IconSourceWalkXCode { - return internal.HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType) + return HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType) } return true } diff --git a/internal/homepage/icon_url_test.go b/internal/homepage/icon_url_test.go index 580ebaa..1b655fc 100644 --- a/internal/homepage/icon_url_test.go +++ b/internal/homepage/icon_url_test.go @@ -3,7 +3,7 @@ package homepage import ( "testing" - . "github.com/yusing/go-proxy/internal/utils/testing" + expect "github.com/yusing/go-proxy/internal/utils/testing" ) func TestIconURL(t *testing.T) { @@ -114,11 +114,11 @@ func TestIconURL(t *testing.T) { u := &IconURL{} err := u.Parse(tc.input) if tc.wantErr { - ExpectError(t, ErrInvalidIconURL, err) + expect.ErrorIs(t, ErrInvalidIconURL, err) } else { tc.wantValue.FullValue = tc.input - ExpectNoError(t, err) - ExpectEqual(t, u, tc.wantValue) + expect.NoError(t, err) + expect.Equal(t, u, tc.wantValue) } }) } diff --git a/internal/homepage/list-icons.go b/internal/homepage/list-icons.go new file mode 100644 index 0000000..f5ae736 --- /dev/null +++ b/internal/homepage/list-icons.go @@ -0,0 +1,297 @@ +package homepage + +import ( + "encoding/json" + "io" + "net/http" + "sync" + "time" + + "github.com/lithammer/fuzzysearch/fuzzy" + "github.com/yusing/go-proxy/internal/common" + "github.com/yusing/go-proxy/internal/logging" + "github.com/yusing/go-proxy/internal/utils" +) + +type GitHubContents struct { //! keep this, may reuse in future + Type string `json:"type"` + Path string `json:"path"` + Name string `json:"name"` + Sha string `json:"sha"` + Size int `json:"size"` +} + +type ( + IconsMap map[string]map[string]struct{} + IconList []string + Cache struct { + WalkxCode, Selfhst IconsMap + DisplayNames ReferenceDisplayNameMap + IconList IconList // combined into a single list + } + ReferenceDisplayNameMap map[string]string +) + +func (icons *Cache) needUpdate() bool { + return len(icons.WalkxCode) == 0 || len(icons.Selfhst) == 0 || len(icons.IconList) == 0 || len(icons.DisplayNames) == 0 +} + +const updateInterval = 2 * time.Hour + +var ( + iconsCache *Cache + iconsCahceMu sync.RWMutex + lastUpdate time.Time +) + +const ( + walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json" + selfhstIcons = "https://cdn.selfh.st/directory/icons.json" +) + +func InitIconListCache() { + iconsCahceMu.Lock() + defer iconsCahceMu.Unlock() + + iconsCache = &Cache{ + WalkxCode: make(IconsMap), + Selfhst: make(IconsMap), + DisplayNames: make(ReferenceDisplayNameMap), + IconList: []string{}, + } + err := utils.LoadJSONIfExist(common.IconListCachePath, iconsCache) + if err != nil { + logging.Error().Err(err).Msg("failed to load icon list cache config") + } else if len(iconsCache.IconList) > 0 { + logging.Info(). + Int("icons", len(iconsCache.IconList)). + Int("display_names", len(iconsCache.DisplayNames)). + Msg("icon list cache loaded") + } +} + +func ListAvailableIcons() (*Cache, error) { + iconsCahceMu.RLock() + if time.Since(lastUpdate) < updateInterval { + if !iconsCache.needUpdate() { + iconsCahceMu.RUnlock() + return iconsCache, nil + } + } + iconsCahceMu.RUnlock() + + iconsCahceMu.Lock() + defer iconsCahceMu.Unlock() + + logging.Info().Msg("updating icon data") + icons, err := fetchIconData() + if err != nil { + return nil, err + } + logging.Info(). + Int("icons", len(icons.IconList)). + Int("display_names", len(icons.DisplayNames)). + Msg("icons list updated") + + iconsCache = icons + lastUpdate = time.Now() + + err = utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644) + if err != nil { + logging.Warn().Err(err).Msg("failed to save icon list cache") + } + return icons, nil +} + +func SearchIcons(keyword string, limit int) ([]string, error) { + icons, err := ListAvailableIcons() + if err != nil { + return nil, err + } + if keyword == "" { + return utils.Slice(icons.IconList, limit), nil + } + return utils.Slice(fuzzy.Find(keyword, icons.IconList), limit), nil +} + +func HasWalkxCodeIcon(name string, filetype string) bool { + icons, err := ListAvailableIcons() + if err != nil { + logging.Error().Err(err).Msg("failed to list icons") + return false + } + if _, ok := icons.WalkxCode[filetype]; !ok { + return false + } + _, ok := icons.WalkxCode[filetype][name+"."+filetype] + return ok +} + +func HasSelfhstIcon(name string, filetype string) bool { + icons, err := ListAvailableIcons() + if err != nil { + logging.Error().Err(err).Msg("failed to list icons") + return false + } + if _, ok := icons.Selfhst[filetype]; !ok { + return false + } + _, ok := icons.Selfhst[filetype][name+"."+filetype] + return ok +} + +func GetDisplayName(reference string) (string, bool) { + icons, err := ListAvailableIcons() + if err != nil { + logging.Error().Err(err).Msg("failed to list icons") + return "", false + } + displayName, ok := icons.DisplayNames[reference] + return displayName, ok +} + +func fetchIconData() (*Cache, error) { + walkxCodeIconMap, walkxCodeIconList, err := fetchWalkxCodeIcons() + if err != nil { + return nil, err + } + + n := 0 + for _, items := range walkxCodeIconMap { + n += len(items) + } + + selfhstIconMap, selfhstIconList, referenceToNames, err := fetchSelfhstIcons() + if err != nil { + return nil, err + } + + return &Cache{ + WalkxCode: walkxCodeIconMap, + Selfhst: selfhstIconMap, + DisplayNames: referenceToNames, + IconList: append(walkxCodeIconList, selfhstIconList...), + }, nil +} + +/* +format: + + { + "png": [ + "*.png", + ], + "svg": [ + "*.svg", + ], + "webp": [ + "*.webp", + ] + } +*/ +func fetchWalkxCodeIcons() (IconsMap, IconList, error) { + req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil) + if err != nil { + return nil, nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + data := make(map[string][]string) + err = json.Unmarshal(body, &data) + if err != nil { + return nil, nil, err + } + icons := make(IconsMap, len(data)) + iconList := make(IconList, 0, 2000) + for fileType, files := range data { + icons[fileType] = make(map[string]struct{}, len(files)) + for _, icon := range files { + icons[fileType][icon] = struct{}{} + iconList = append(iconList, "@walkxcode/"+icon) + } + } + return icons, iconList, nil +} + +/* +format: + + { + "Name": "2FAuth", + "Reference": "2fauth", + "SVG": "Yes", + "PNG": "Yes", + "WebP": "Yes", + "Light": "Yes", + "Category": "Self-Hosted", + "CreatedAt": "2024-08-16 00:27:23+00:00" + } +*/ +func fetchSelfhstIcons() (IconsMap, IconList, ReferenceDisplayNameMap, error) { + type SelfhStIcon struct { + Name string `json:"Name"` + Reference string `json:"Reference"` + SVG string `json:"SVG"` + PNG string `json:"PNG"` + WebP string `json:"WebP"` + // Light string + // Category string + // CreatedAt string + } + + req, err := http.NewRequest(http.MethodGet, selfhstIcons, nil) + if err != nil { + return nil, nil, nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, nil, nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, nil, err + } + + data := make([]SelfhStIcon, 0, 2000) + err = json.Unmarshal(body, &data) + if err != nil { + return nil, nil, nil, err + } + + iconList := make(IconList, 0, len(data)*3) + icons := make(IconsMap) + icons["svg"] = make(map[string]struct{}, len(data)) + icons["png"] = make(map[string]struct{}, len(data)) + icons["webp"] = make(map[string]struct{}, len(data)) + + referenceToNames := make(ReferenceDisplayNameMap, len(data)) + + for _, item := range data { + if item.SVG == "Yes" { + icons["svg"][item.Reference+".svg"] = struct{}{} + iconList = append(iconList, "@selfhst/"+item.Reference+".svg") + } + if item.PNG == "Yes" { + icons["png"][item.Reference+".png"] = struct{}{} + iconList = append(iconList, "@selfhst/"+item.Reference+".png") + } + if item.WebP == "Yes" { + icons["webp"][item.Reference+".webp"] = struct{}{} + iconList = append(iconList, "@selfhst/"+item.Reference+".webp") + } + referenceToNames[item.Reference] = item.Name + } + + return icons, iconList, referenceToNames, nil +} diff --git a/internal/homepage/route.go b/internal/homepage/route.go new file mode 100644 index 0000000..9c5aa56 --- /dev/null +++ b/internal/homepage/route.go @@ -0,0 +1,20 @@ +package homepage + +import ( + "net/http" + + gpnet "github.com/yusing/go-proxy/internal/net/types" + "github.com/yusing/go-proxy/internal/utils/pool" +) + +type route interface { + pool.Object + ProviderName() string + Reference() string + TargetURL() *gpnet.URL +} + +type httpRoute interface { + route + http.Handler +} diff --git a/internal/net/gphttp/default_client.go b/internal/net/gphttp/default_client.go index dee455a..060af65 100644 --- a/internal/net/gphttp/default_client.go +++ b/internal/net/gphttp/default_client.go @@ -24,4 +24,5 @@ var ( Get = httpClient.Get Post = httpClient.Post Head = httpClient.Head + Do = httpClient.Do ) diff --git a/internal/route/reverse_proxy.go b/internal/route/reverse_proxy.go index 0456848..850afe8 100755 --- a/internal/route/reverse_proxy.go +++ b/internal/route/reverse_proxy.go @@ -6,9 +6,9 @@ import ( "github.com/yusing/go-proxy/agent/pkg/agent" "github.com/yusing/go-proxy/agent/pkg/agentproxy" - "github.com/yusing/go-proxy/internal/api/v1/favicon" "github.com/yusing/go-proxy/internal/common" "github.com/yusing/go-proxy/internal/gperr" + "github.com/yusing/go-proxy/internal/homepage" "github.com/yusing/go-proxy/internal/idlewatcher" gphttp "github.com/yusing/go-proxy/internal/net/gphttp" "github.com/yusing/go-proxy/internal/net/gphttp/accesslog" @@ -143,7 +143,7 @@ func (r *ReveseProxyRoute) Start(parent task.Parent) gperr.Error { }) } - r.task.OnCancel("reset_favicon", func() { favicon.PruneRouteIconCache(r) }) + r.task.OnCancel("reset_favicon", func() { homepage.PruneRouteIconCache(r) }) return nil }