mirror of
https://github.com/yusing/godoxy.git
synced 2025-05-20 04:42:33 +02:00
merge: better favicon handling
This commit is contained in:
yusing
parent
31812430f1
commit
5df2553774
11 changed files with 722 additions and 330 deletions
|
|
@ -1,101 +0,0 @@
|
|||
package favicon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/yusing/go-proxy/internal/common"
|
||||
"github.com/yusing/go-proxy/internal/jsonstore"
|
||||
"github.com/yusing/go-proxy/internal/logging"
|
||||
"github.com/yusing/go-proxy/internal/route/routes"
|
||||
"github.com/yusing/go-proxy/internal/task"
|
||||
)
|
||||
|
||||
type cacheEntry struct {
|
||||
Icon []byte `json:"icon"`
|
||||
LastAccess time.Time `json:"lastAccess"`
|
||||
}
|
||||
|
||||
// cache key can be absolute url or route name.
|
||||
var iconCache = jsonstore.Store[*cacheEntry](common.NamespaceIconCache)
|
||||
|
||||
const (
|
||||
iconCacheTTL = 3 * 24 * time.Hour
|
||||
cleanUpInterval = time.Hour
|
||||
)
|
||||
|
||||
func init() {
|
||||
go func() {
|
||||
cleanupTicker := time.NewTicker(cleanUpInterval)
|
||||
defer cleanupTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-task.RootContextCanceled():
|
||||
return
|
||||
case <-cleanupTicker.C:
|
||||
pruneExpiredIconCache()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func pruneExpiredIconCache() {
|
||||
nPruned := 0
|
||||
for key, icon := range iconCache.Range {
|
||||
if icon.IsExpired() {
|
||||
iconCache.Delete(key)
|
||||
nPruned++
|
||||
}
|
||||
}
|
||||
if nPruned > 0 {
|
||||
logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache")
|
||||
}
|
||||
}
|
||||
|
||||
func routeKey(r routes.HTTPRoute) string {
|
||||
return r.ProviderName() + ":" + r.Name()
|
||||
}
|
||||
|
||||
func PruneRouteIconCache(route routes.HTTPRoute) {
|
||||
iconCache.Delete(routeKey(route))
|
||||
}
|
||||
|
||||
func loadIconCache(key string) *fetchResult {
|
||||
icon, ok := iconCache.Load(key)
|
||||
if ok && icon != nil {
|
||||
logging.Debug().
|
||||
Str("key", key).
|
||||
Msg("icon found in cache")
|
||||
icon.LastAccess = time.Now()
|
||||
return &fetchResult{icon: icon.Icon}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func storeIconCache(key string, icon []byte) {
|
||||
iconCache.Store(key, &cacheEntry{Icon: icon, LastAccess: time.Now()})
|
||||
}
|
||||
|
||||
func (e *cacheEntry) IsExpired() bool {
|
||||
return time.Since(e.LastAccess) > iconCacheTTL
|
||||
}
|
||||
|
||||
func (e *cacheEntry) UnmarshalJSON(data []byte) error {
|
||||
attempt := struct {
|
||||
Icon []byte `json:"icon"`
|
||||
LastAccess time.Time `json:"lastAccess"`
|
||||
}{}
|
||||
err := json.Unmarshal(data, &attempt)
|
||||
if err == nil {
|
||||
e.Icon = attempt.Icon
|
||||
e.LastAccess = attempt.LastAccess
|
||||
return nil
|
||||
}
|
||||
// fallback to bytes
|
||||
err = json.Unmarshal(data, &e.Icon)
|
||||
if err == nil {
|
||||
e.LastAccess = time.Now()
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -1,48 +1,13 @@
|
|||
package favicon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/vincent-petithory/dataurl"
|
||||
"github.com/yusing/go-proxy/internal/gperr"
|
||||
"github.com/yusing/go-proxy/internal/homepage"
|
||||
"github.com/yusing/go-proxy/internal/logging"
|
||||
gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
|
||||
"github.com/yusing/go-proxy/internal/net/gphttp"
|
||||
"github.com/yusing/go-proxy/internal/route/routes"
|
||||
"github.com/yusing/go-proxy/internal/utils/strutils"
|
||||
)
|
||||
|
||||
type fetchResult struct {
|
||||
icon []byte
|
||||
contentType string
|
||||
statusCode int
|
||||
errMsg string
|
||||
}
|
||||
|
||||
func (res *fetchResult) OK() bool {
|
||||
return res.icon != nil
|
||||
}
|
||||
|
||||
func (res *fetchResult) ContentType() string {
|
||||
if res.contentType == "" {
|
||||
if bytes.HasPrefix(res.icon, []byte("<svg")) || bytes.HasPrefix(res.icon, []byte("<?xml")) {
|
||||
return "image/svg+xml"
|
||||
}
|
||||
return "image/x-icon"
|
||||
}
|
||||
return res.contentType
|
||||
}
|
||||
|
||||
const (
|
||||
MaxRedirectDepth = 5
|
||||
)
|
||||
|
||||
// GetFavIcon returns the favicon of the route
|
||||
|
|
@ -71,213 +36,42 @@ func GetFavIcon(w http.ResponseWriter, req *http.Request) {
|
|||
gphttp.ClientError(w, err, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
fetchResult := getFavIconFromURL(&iconURL)
|
||||
fetchResult := homepage.FetchFavIconFromURL(req.Context(), &iconURL)
|
||||
if !fetchResult.OK() {
|
||||
http.Error(w, fetchResult.errMsg, fetchResult.statusCode)
|
||||
http.Error(w, fetchResult.ErrMsg, fetchResult.StatusCode)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", fetchResult.ContentType())
|
||||
gphttp.WriteBody(w, fetchResult.icon)
|
||||
gphttp.WriteBody(w, fetchResult.Icon)
|
||||
return
|
||||
}
|
||||
|
||||
// try with route.Homepage.Icon
|
||||
// try with route.Icon
|
||||
r, ok := routes.HTTP.Get(alias)
|
||||
if !ok {
|
||||
gphttp.ClientError(w, errors.New("no such route"), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
var result *fetchResult
|
||||
var result *homepage.FetchResult
|
||||
hp := r.HomepageItem()
|
||||
if hp.Icon != nil {
|
||||
if hp.Icon.IconSource == homepage.IconSourceRelative {
|
||||
result = findIcon(r, req, hp.Icon.Value)
|
||||
result = homepage.FindIcon(req.Context(), r, hp.Icon.Value)
|
||||
} else {
|
||||
result = getFavIconFromURL(hp.Icon)
|
||||
result = homepage.FetchFavIconFromURL(req.Context(), hp.Icon)
|
||||
}
|
||||
} else {
|
||||
// try extract from "link[rel=icon]"
|
||||
result = findIcon(r, req, "/")
|
||||
result = homepage.FindIcon(req.Context(), r, "/")
|
||||
}
|
||||
if result.statusCode == 0 {
|
||||
result.statusCode = http.StatusOK
|
||||
if result.StatusCode == 0 {
|
||||
result.StatusCode = http.StatusOK
|
||||
}
|
||||
if !result.OK() {
|
||||
http.Error(w, result.errMsg, result.statusCode)
|
||||
http.Error(w, result.ErrMsg, result.StatusCode)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", result.ContentType())
|
||||
gphttp.WriteBody(w, result.icon)
|
||||
}
|
||||
|
||||
func getFavIconFromURL(iconURL *homepage.IconURL) *fetchResult {
|
||||
switch iconURL.IconSource {
|
||||
case homepage.IconSourceAbsolute:
|
||||
return fetchIconAbsolute(iconURL.URL())
|
||||
case homepage.IconSourceRelative:
|
||||
return &fetchResult{statusCode: http.StatusBadRequest, errMsg: "unexpected relative icon"}
|
||||
case homepage.IconSourceWalkXCode, homepage.IconSourceSelfhSt:
|
||||
return fetchKnownIcon(iconURL)
|
||||
}
|
||||
return &fetchResult{statusCode: http.StatusBadRequest, errMsg: "invalid icon source"}
|
||||
}
|
||||
|
||||
func fetchIconAbsolute(url string) *fetchResult {
|
||||
if result := loadIconCache(url); result != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
resp, err := gphttp.Get(url)
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
if err == nil {
|
||||
err = errors.New(resp.Status)
|
||||
}
|
||||
logging.Error().Err(err).
|
||||
Str("url", url).
|
||||
Msg("failed to get icon")
|
||||
return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "connection error"}
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
icon, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logging.Error().Err(err).
|
||||
Str("url", url).
|
||||
Msg("failed to read icon")
|
||||
return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "internal error"}
|
||||
}
|
||||
|
||||
storeIconCache(url, icon)
|
||||
return &fetchResult{icon: icon}
|
||||
}
|
||||
|
||||
var nameSanitizer = strings.NewReplacer(
|
||||
"_", "-",
|
||||
" ", "-",
|
||||
"(", "",
|
||||
")", "",
|
||||
)
|
||||
|
||||
func sanitizeName(name string) string {
|
||||
return strings.ToLower(nameSanitizer.Replace(name))
|
||||
}
|
||||
|
||||
func fetchKnownIcon(url *homepage.IconURL) *fetchResult {
|
||||
// if icon isn't in the list, no need to fetch
|
||||
if !url.HasIcon() {
|
||||
logging.Debug().
|
||||
Str("value", url.String()).
|
||||
Str("url", url.URL()).
|
||||
Msg("no such icon")
|
||||
return &fetchResult{statusCode: http.StatusNotFound, errMsg: "no such icon"}
|
||||
}
|
||||
|
||||
return fetchIconAbsolute(url.URL())
|
||||
}
|
||||
|
||||
func fetchIcon(filetype, filename string) *fetchResult {
|
||||
result := fetchKnownIcon(homepage.NewSelfhStIconURL(filename, filetype))
|
||||
if result.icon == nil {
|
||||
return result
|
||||
}
|
||||
return fetchKnownIcon(homepage.NewWalkXCodeIconURL(filename, filetype))
|
||||
}
|
||||
|
||||
func findIcon(r routes.HTTPRoute, req *http.Request, uri string) *fetchResult {
|
||||
key := routeKey(r)
|
||||
if result := loadIconCache(key); result != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
result := fetchIcon("png", sanitizeName(r.Name()))
|
||||
cont := r.ContainerInfo()
|
||||
if !result.OK() && cont != nil {
|
||||
result = fetchIcon("png", sanitizeName(cont.Image.Name))
|
||||
}
|
||||
if !result.OK() {
|
||||
// fallback to parse html
|
||||
result = findIconSlow(r, req, uri, 0)
|
||||
}
|
||||
if result.OK() {
|
||||
storeIconCache(key, result.icon)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func findIconSlow(r routes.HTTPRoute, req *http.Request, uri string, depth int) *fetchResult {
|
||||
ctx, cancel := context.WithTimeoutCause(req.Context(), 3*time.Second, errors.New("favicon request timeout"))
|
||||
defer cancel()
|
||||
newReq := req.WithContext(ctx)
|
||||
newReq.Header.Set("Accept-Encoding", "identity") // disable compression
|
||||
u, err := url.ParseRequestURI(strutils.SanitizeURI(uri))
|
||||
if err != nil {
|
||||
logging.Error().Err(err).
|
||||
Str("route", r.Name()).
|
||||
Str("path", uri).
|
||||
Msg("failed to parse uri")
|
||||
return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "cannot parse uri"}
|
||||
}
|
||||
newReq.URL.Path = u.Path
|
||||
newReq.URL.RawPath = u.RawPath
|
||||
newReq.URL.RawQuery = u.RawQuery
|
||||
newReq.RequestURI = u.String()
|
||||
|
||||
c := newContent()
|
||||
r.ServeHTTP(c, newReq)
|
||||
if c.status != http.StatusOK {
|
||||
switch c.status {
|
||||
case 0:
|
||||
return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "connection error"}
|
||||
default:
|
||||
if loc := c.Header().Get("Location"); loc != "" {
|
||||
if depth > MaxRedirectDepth {
|
||||
return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "too many redirects"}
|
||||
}
|
||||
loc = strutils.SanitizeURI(loc)
|
||||
if loc == "/" || loc == newReq.URL.Path {
|
||||
return &fetchResult{statusCode: http.StatusBadGateway, errMsg: "circular redirect"}
|
||||
}
|
||||
return findIconSlow(r, req, loc, depth+1)
|
||||
}
|
||||
}
|
||||
return &fetchResult{statusCode: c.status, errMsg: "upstream error: " + string(c.data)}
|
||||
}
|
||||
// return icon data
|
||||
if !gphttp.GetContentType(c.header).IsHTML() {
|
||||
return &fetchResult{icon: c.data, contentType: c.header.Get("Content-Type")}
|
||||
}
|
||||
// try extract from "link[rel=icon]" from path "/"
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.data))
|
||||
if err != nil {
|
||||
logging.Error().Err(err).
|
||||
Str("route", r.Name()).
|
||||
Msg("failed to parse html")
|
||||
return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "internal error"}
|
||||
}
|
||||
ele := doc.Find("head > link[rel=icon]").First()
|
||||
if ele.Length() == 0 {
|
||||
return &fetchResult{statusCode: http.StatusNotFound, errMsg: "icon element not found"}
|
||||
}
|
||||
href := ele.AttrOr("href", "")
|
||||
if href == "" {
|
||||
return &fetchResult{statusCode: http.StatusNotFound, errMsg: "icon href not found"}
|
||||
}
|
||||
// https://en.wikipedia.org/wiki/Data_URI_scheme
|
||||
if strings.HasPrefix(href, "data:image/") {
|
||||
dataURI, err := dataurl.DecodeString(href)
|
||||
if err != nil {
|
||||
logging.Error().Err(err).
|
||||
Str("route", r.Name()).
|
||||
Msg("failed to decode favicon")
|
||||
return &fetchResult{statusCode: http.StatusInternalServerError, errMsg: "internal error"}
|
||||
}
|
||||
return &fetchResult{icon: dataURI.Data, contentType: dataURI.ContentType()}
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"):
|
||||
return fetchIconAbsolute(href)
|
||||
default:
|
||||
return findIconSlow(r, req, href, 0)
|
||||
}
|
||||
gphttp.WriteBody(w, result.Icon)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
package favicon
|
||||
package homepage
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
222
internal/homepage/favicon.go
Normal file
222
internal/homepage/favicon.go
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
package homepage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/vincent-petithory/dataurl"
|
||||
gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
|
||||
"github.com/yusing/go-proxy/internal/utils/strutils"
|
||||
)
|
||||
|
||||
type FetchResult struct {
|
||||
Icon []byte
|
||||
StatusCode int
|
||||
ErrMsg string
|
||||
|
||||
contentType string
|
||||
}
|
||||
|
||||
const faviconFetchTimeout = 3 * time.Second
|
||||
|
||||
func (res *FetchResult) OK() bool {
|
||||
return len(res.Icon) > 0
|
||||
}
|
||||
|
||||
func (res *FetchResult) ContentType() string {
|
||||
if res.contentType == "" {
|
||||
if bytes.HasPrefix(res.Icon, []byte("<svg")) || bytes.HasPrefix(res.Icon, []byte("<?xml")) {
|
||||
return "image/svg+xml"
|
||||
}
|
||||
return "image/x-icon"
|
||||
}
|
||||
return res.contentType
|
||||
}
|
||||
|
||||
const maxRedirectDepth = 5
|
||||
|
||||
func FetchFavIconFromURL(ctx context.Context, iconURL *IconURL) *FetchResult {
|
||||
switch iconURL.IconSource {
|
||||
case IconSourceAbsolute:
|
||||
return fetchIconAbsolute(ctx, iconURL.URL())
|
||||
case IconSourceRelative:
|
||||
return &FetchResult{StatusCode: http.StatusBadRequest, ErrMsg: "unexpected relative icon"}
|
||||
case IconSourceWalkXCode, IconSourceSelfhSt:
|
||||
return fetchKnownIcon(ctx, iconURL)
|
||||
}
|
||||
return &FetchResult{StatusCode: http.StatusBadRequest, ErrMsg: "invalid icon source"}
|
||||
}
|
||||
|
||||
func fetchIconAbsolute(ctx context.Context, url string) *FetchResult {
|
||||
if result := loadIconCache(url); result != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "request timeout"}
|
||||
}
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: err.Error()}
|
||||
}
|
||||
|
||||
resp, err := gphttp.Do(req)
|
||||
if err == nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "connection error"}
|
||||
}
|
||||
|
||||
icon, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "internal error"}
|
||||
}
|
||||
|
||||
if len(icon) == 0 {
|
||||
return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "empty icon"}
|
||||
}
|
||||
|
||||
res := &FetchResult{Icon: icon}
|
||||
if contentType := resp.Header.Get("Content-Type"); contentType != "" {
|
||||
res.contentType = contentType
|
||||
}
|
||||
// else leave it empty
|
||||
storeIconCache(url, res)
|
||||
return res
|
||||
}
|
||||
|
||||
var nameSanitizer = strings.NewReplacer(
|
||||
"_", "-",
|
||||
" ", "-",
|
||||
"(", "",
|
||||
")", "",
|
||||
)
|
||||
|
||||
func sanitizeName(name string) string {
|
||||
return strings.ToLower(nameSanitizer.Replace(name))
|
||||
}
|
||||
|
||||
func fetchKnownIcon(ctx context.Context, url *IconURL) *FetchResult {
|
||||
// if icon isn't in the list, no need to fetch
|
||||
if !url.HasIcon() {
|
||||
return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "no such icon"}
|
||||
}
|
||||
|
||||
return fetchIconAbsolute(ctx, url.URL())
|
||||
}
|
||||
|
||||
func fetchIcon(ctx context.Context, filetype, filename string) *FetchResult {
|
||||
result := fetchKnownIcon(ctx, NewSelfhStIconURL(filename, filetype))
|
||||
if result.OK() {
|
||||
return result
|
||||
}
|
||||
return fetchKnownIcon(ctx, NewWalkXCodeIconURL(filename, filetype))
|
||||
}
|
||||
|
||||
func FindIcon(ctx context.Context, r route, uri string) *FetchResult {
|
||||
if result := loadIconCache(r.Key()); result != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
result := fetchIcon(ctx, "png", sanitizeName(r.Reference()))
|
||||
if !result.OK() {
|
||||
if r, ok := r.(httpRoute); ok {
|
||||
// fallback to parse html
|
||||
result = findIconSlow(ctx, r, uri, nil)
|
||||
}
|
||||
}
|
||||
if result.OK() {
|
||||
storeIconCache(r.Key(), result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func findIconSlow(ctx context.Context, r httpRoute, uri string, stack []string) *FetchResult {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "request timeout"}
|
||||
default:
|
||||
}
|
||||
|
||||
if len(stack) > maxRedirectDepth {
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "too many redirects"}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeoutCause(ctx, faviconFetchTimeout, errors.New("favicon request timeout"))
|
||||
defer cancel()
|
||||
|
||||
newReq, err := http.NewRequestWithContext(ctx, "GET", r.TargetURL().String(), nil)
|
||||
if err != nil {
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot create request"}
|
||||
}
|
||||
newReq.Header.Set("Accept-Encoding", "identity") // disable compression
|
||||
|
||||
u, err := url.ParseRequestURI(strutils.SanitizeURI(uri))
|
||||
if err != nil {
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "cannot parse uri"}
|
||||
}
|
||||
newReq.URL.Path = u.Path
|
||||
newReq.URL.RawPath = u.RawPath
|
||||
newReq.URL.RawQuery = u.RawQuery
|
||||
newReq.RequestURI = u.String()
|
||||
|
||||
c := newContent()
|
||||
r.ServeHTTP(c, newReq)
|
||||
if c.status != http.StatusOK {
|
||||
switch c.status {
|
||||
case 0:
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "connection error"}
|
||||
default:
|
||||
if loc := c.Header().Get("Location"); loc != "" {
|
||||
loc = strutils.SanitizeURI(loc)
|
||||
if loc == "/" || loc == newReq.URL.Path || slices.Contains(stack, loc) {
|
||||
return &FetchResult{StatusCode: http.StatusBadGateway, ErrMsg: "circular redirect"}
|
||||
}
|
||||
// append current path to stack
|
||||
// handles redirect to the same path with different query
|
||||
return findIconSlow(ctx, r, loc, append(stack, newReq.URL.Path))
|
||||
}
|
||||
}
|
||||
return &FetchResult{StatusCode: c.status, ErrMsg: "upstream error: " + string(c.data)}
|
||||
}
|
||||
// return icon data
|
||||
if !gphttp.GetContentType(c.header).IsHTML() {
|
||||
return &FetchResult{Icon: c.data, contentType: c.header.Get("Content-Type")}
|
||||
}
|
||||
// try extract from "link[rel=icon]" from path "/"
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.data))
|
||||
if err != nil {
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to parse html"}
|
||||
}
|
||||
ele := doc.Find("head > link[rel=icon]").First()
|
||||
if ele.Length() == 0 {
|
||||
return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon element not found"}
|
||||
}
|
||||
href := ele.AttrOr("href", "")
|
||||
if href == "" {
|
||||
return &FetchResult{StatusCode: http.StatusNotFound, ErrMsg: "icon href not found"}
|
||||
}
|
||||
// https://en.wikipedia.org/wiki/Data_URI_scheme
|
||||
if strings.HasPrefix(href, "data:image/") {
|
||||
dataURI, err := dataurl.DecodeString(href)
|
||||
if err != nil {
|
||||
return &FetchResult{StatusCode: http.StatusInternalServerError, ErrMsg: "failed to decode favicon"}
|
||||
}
|
||||
return &FetchResult{Icon: dataURI.Data, contentType: dataURI.ContentType()}
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"):
|
||||
return fetchIconAbsolute(ctx, href)
|
||||
default:
|
||||
return findIconSlow(ctx, r, href, append(stack, newReq.URL.Path))
|
||||
}
|
||||
}
|
||||
160
internal/homepage/icon_cache.go
Normal file
160
internal/homepage/icon_cache.go
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
package homepage
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/yusing/go-proxy/internal/common"
|
||||
"github.com/yusing/go-proxy/internal/logging"
|
||||
"github.com/yusing/go-proxy/internal/task"
|
||||
"github.com/yusing/go-proxy/internal/utils"
|
||||
"github.com/yusing/go-proxy/internal/utils/atomic"
|
||||
)
|
||||
|
||||
type cacheEntry struct {
|
||||
Icon []byte `json:"icon"`
|
||||
ContentType string `json:"content_type"`
|
||||
LastAccess atomic.Value[time.Time] `json:"last_access"`
|
||||
}
|
||||
|
||||
// cache key can be absolute url or route name.
|
||||
var (
|
||||
iconCache = make(map[string]*cacheEntry)
|
||||
iconCacheMu sync.RWMutex
|
||||
)
|
||||
|
||||
const (
|
||||
iconCacheTTL = 3 * 24 * time.Hour
|
||||
cleanUpInterval = time.Minute
|
||||
maxCacheSize = 1024 * 1024 // 1MB
|
||||
maxCacheEntries = 100
|
||||
)
|
||||
|
||||
func InitIconCache() {
|
||||
iconCacheMu.Lock()
|
||||
defer iconCacheMu.Unlock()
|
||||
|
||||
err := utils.LoadJSONIfExist(common.IconCachePath, &iconCache)
|
||||
if err != nil {
|
||||
logging.Error().Err(err).Msg("failed to load icon cache")
|
||||
} else if len(iconCache) > 0 {
|
||||
logging.Info().Int("count", len(iconCache)).Msg("icon cache loaded")
|
||||
}
|
||||
|
||||
go func() {
|
||||
cleanupTicker := time.NewTicker(cleanUpInterval)
|
||||
defer cleanupTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-task.RootContextCanceled():
|
||||
return
|
||||
case <-cleanupTicker.C:
|
||||
pruneExpiredIconCache()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
task.OnProgramExit("save_favicon_cache", func() {
|
||||
iconCacheMu.Lock()
|
||||
defer iconCacheMu.Unlock()
|
||||
|
||||
if len(iconCache) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if err := utils.SaveJSON(common.IconCachePath, &iconCache, 0o644); err != nil {
|
||||
logging.Error().Err(err).Msg("failed to save icon cache")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func pruneExpiredIconCache() {
|
||||
iconCacheMu.Lock()
|
||||
defer iconCacheMu.Unlock()
|
||||
|
||||
nPruned := 0
|
||||
for key, icon := range iconCache {
|
||||
if icon.IsExpired() {
|
||||
delete(iconCache, key)
|
||||
nPruned++
|
||||
}
|
||||
}
|
||||
if len(iconCache) > maxCacheEntries {
|
||||
newIconCache := make(map[string]*cacheEntry, maxCacheEntries)
|
||||
i := 0
|
||||
for key, icon := range iconCache {
|
||||
if i == maxCacheEntries {
|
||||
break
|
||||
}
|
||||
if !icon.IsExpired() {
|
||||
newIconCache[key] = icon
|
||||
i++
|
||||
}
|
||||
}
|
||||
iconCache = newIconCache
|
||||
}
|
||||
if nPruned > 0 {
|
||||
logging.Info().Int("pruned", nPruned).Msg("pruned expired icon cache")
|
||||
}
|
||||
}
|
||||
|
||||
func PruneRouteIconCache(route route) {
|
||||
iconCacheMu.Lock()
|
||||
defer iconCacheMu.Unlock()
|
||||
delete(iconCache, route.Key())
|
||||
}
|
||||
|
||||
func loadIconCache(key string) *FetchResult {
|
||||
iconCacheMu.RLock()
|
||||
defer iconCacheMu.RUnlock()
|
||||
|
||||
icon, ok := iconCache[key]
|
||||
if ok && len(icon.Icon) > 0 {
|
||||
logging.Debug().
|
||||
Str("key", key).
|
||||
Msg("icon found in cache")
|
||||
icon.LastAccess.Store(time.Now())
|
||||
return &FetchResult{Icon: icon.Icon, contentType: icon.ContentType}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func storeIconCache(key string, result *FetchResult) {
|
||||
icon := result.Icon
|
||||
if len(icon) > maxCacheSize {
|
||||
logging.Debug().Int("size", len(icon)).Msg("icon cache size exceeds max cache size")
|
||||
return
|
||||
}
|
||||
iconCacheMu.Lock()
|
||||
defer iconCacheMu.Unlock()
|
||||
entry := &cacheEntry{Icon: icon, ContentType: result.contentType}
|
||||
entry.LastAccess.Store(time.Now())
|
||||
iconCache[key] = entry
|
||||
logging.Debug().Str("key", key).Int("size", len(icon)).Msg("stored icon cache")
|
||||
}
|
||||
|
||||
func (e *cacheEntry) IsExpired() bool {
|
||||
return time.Since(e.LastAccess.Load()) > iconCacheTTL
|
||||
}
|
||||
|
||||
func (e *cacheEntry) UnmarshalJSON(data []byte) error {
|
||||
// check if data is json
|
||||
if json.Valid(data) {
|
||||
err := json.Unmarshal(data, &e)
|
||||
// return only if unmarshal is successful
|
||||
// otherwise fallback to base64
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// fallback to base64
|
||||
icon, err := base64.StdEncoding.DecodeString(string(data))
|
||||
if err == nil {
|
||||
e.Icon = icon
|
||||
e.LastAccess.Store(time.Now())
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/yusing/go-proxy/internal"
|
||||
"github.com/yusing/go-proxy/internal/gperr"
|
||||
)
|
||||
|
||||
|
|
@ -62,10 +61,10 @@ func NewWalkXCodeIconURL(name, format string) *IconURL {
|
|||
// otherwise returns true.
|
||||
func (u *IconURL) HasIcon() bool {
|
||||
if u.IconSource == IconSourceSelfhSt {
|
||||
return internal.HasSelfhstIcon(u.Extra.Name, u.Extra.FileType)
|
||||
return HasSelfhstIcon(u.Extra.Name, u.Extra.FileType)
|
||||
}
|
||||
if u.IconSource == IconSourceWalkXCode {
|
||||
return internal.HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType)
|
||||
return HasWalkxCodeIcon(u.Extra.Name, u.Extra.FileType)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package homepage
|
|||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/yusing/go-proxy/internal/utils/testing"
|
||||
expect "github.com/yusing/go-proxy/internal/utils/testing"
|
||||
)
|
||||
|
||||
func TestIconURL(t *testing.T) {
|
||||
|
|
@ -114,11 +114,11 @@ func TestIconURL(t *testing.T) {
|
|||
u := &IconURL{}
|
||||
err := u.Parse(tc.input)
|
||||
if tc.wantErr {
|
||||
ExpectError(t, ErrInvalidIconURL, err)
|
||||
expect.ErrorIs(t, ErrInvalidIconURL, err)
|
||||
} else {
|
||||
tc.wantValue.FullValue = tc.input
|
||||
ExpectNoError(t, err)
|
||||
ExpectEqual(t, u, tc.wantValue)
|
||||
expect.NoError(t, err)
|
||||
expect.Equal(t, u, tc.wantValue)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
297
internal/homepage/list-icons.go
Normal file
297
internal/homepage/list-icons.go
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
package homepage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lithammer/fuzzysearch/fuzzy"
|
||||
"github.com/yusing/go-proxy/internal/common"
|
||||
"github.com/yusing/go-proxy/internal/logging"
|
||||
"github.com/yusing/go-proxy/internal/utils"
|
||||
)
|
||||
|
||||
type GitHubContents struct { //! keep this, may reuse in future
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
Sha string `json:"sha"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
|
||||
type (
|
||||
IconsMap map[string]map[string]struct{}
|
||||
IconList []string
|
||||
Cache struct {
|
||||
WalkxCode, Selfhst IconsMap
|
||||
DisplayNames ReferenceDisplayNameMap
|
||||
IconList IconList // combined into a single list
|
||||
}
|
||||
ReferenceDisplayNameMap map[string]string
|
||||
)
|
||||
|
||||
func (icons *Cache) needUpdate() bool {
|
||||
return len(icons.WalkxCode) == 0 || len(icons.Selfhst) == 0 || len(icons.IconList) == 0 || len(icons.DisplayNames) == 0
|
||||
}
|
||||
|
||||
const updateInterval = 2 * time.Hour
|
||||
|
||||
var (
|
||||
iconsCache *Cache
|
||||
iconsCahceMu sync.RWMutex
|
||||
lastUpdate time.Time
|
||||
)
|
||||
|
||||
const (
|
||||
walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json"
|
||||
selfhstIcons = "https://cdn.selfh.st/directory/icons.json"
|
||||
)
|
||||
|
||||
func InitIconListCache() {
|
||||
iconsCahceMu.Lock()
|
||||
defer iconsCahceMu.Unlock()
|
||||
|
||||
iconsCache = &Cache{
|
||||
WalkxCode: make(IconsMap),
|
||||
Selfhst: make(IconsMap),
|
||||
DisplayNames: make(ReferenceDisplayNameMap),
|
||||
IconList: []string{},
|
||||
}
|
||||
err := utils.LoadJSONIfExist(common.IconListCachePath, iconsCache)
|
||||
if err != nil {
|
||||
logging.Error().Err(err).Msg("failed to load icon list cache config")
|
||||
} else if len(iconsCache.IconList) > 0 {
|
||||
logging.Info().
|
||||
Int("icons", len(iconsCache.IconList)).
|
||||
Int("display_names", len(iconsCache.DisplayNames)).
|
||||
Msg("icon list cache loaded")
|
||||
}
|
||||
}
|
||||
|
||||
func ListAvailableIcons() (*Cache, error) {
|
||||
iconsCahceMu.RLock()
|
||||
if time.Since(lastUpdate) < updateInterval {
|
||||
if !iconsCache.needUpdate() {
|
||||
iconsCahceMu.RUnlock()
|
||||
return iconsCache, nil
|
||||
}
|
||||
}
|
||||
iconsCahceMu.RUnlock()
|
||||
|
||||
iconsCahceMu.Lock()
|
||||
defer iconsCahceMu.Unlock()
|
||||
|
||||
logging.Info().Msg("updating icon data")
|
||||
icons, err := fetchIconData()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logging.Info().
|
||||
Int("icons", len(icons.IconList)).
|
||||
Int("display_names", len(icons.DisplayNames)).
|
||||
Msg("icons list updated")
|
||||
|
||||
iconsCache = icons
|
||||
lastUpdate = time.Now()
|
||||
|
||||
err = utils.SaveJSON(common.IconListCachePath, iconsCache, 0o644)
|
||||
if err != nil {
|
||||
logging.Warn().Err(err).Msg("failed to save icon list cache")
|
||||
}
|
||||
return icons, nil
|
||||
}
|
||||
|
||||
func SearchIcons(keyword string, limit int) ([]string, error) {
|
||||
icons, err := ListAvailableIcons()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keyword == "" {
|
||||
return utils.Slice(icons.IconList, limit), nil
|
||||
}
|
||||
return utils.Slice(fuzzy.Find(keyword, icons.IconList), limit), nil
|
||||
}
|
||||
|
||||
func HasWalkxCodeIcon(name string, filetype string) bool {
|
||||
icons, err := ListAvailableIcons()
|
||||
if err != nil {
|
||||
logging.Error().Err(err).Msg("failed to list icons")
|
||||
return false
|
||||
}
|
||||
if _, ok := icons.WalkxCode[filetype]; !ok {
|
||||
return false
|
||||
}
|
||||
_, ok := icons.WalkxCode[filetype][name+"."+filetype]
|
||||
return ok
|
||||
}
|
||||
|
||||
func HasSelfhstIcon(name string, filetype string) bool {
|
||||
icons, err := ListAvailableIcons()
|
||||
if err != nil {
|
||||
logging.Error().Err(err).Msg("failed to list icons")
|
||||
return false
|
||||
}
|
||||
if _, ok := icons.Selfhst[filetype]; !ok {
|
||||
return false
|
||||
}
|
||||
_, ok := icons.Selfhst[filetype][name+"."+filetype]
|
||||
return ok
|
||||
}
|
||||
|
||||
func GetDisplayName(reference string) (string, bool) {
|
||||
icons, err := ListAvailableIcons()
|
||||
if err != nil {
|
||||
logging.Error().Err(err).Msg("failed to list icons")
|
||||
return "", false
|
||||
}
|
||||
displayName, ok := icons.DisplayNames[reference]
|
||||
return displayName, ok
|
||||
}
|
||||
|
||||
func fetchIconData() (*Cache, error) {
|
||||
walkxCodeIconMap, walkxCodeIconList, err := fetchWalkxCodeIcons()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n := 0
|
||||
for _, items := range walkxCodeIconMap {
|
||||
n += len(items)
|
||||
}
|
||||
|
||||
selfhstIconMap, selfhstIconList, referenceToNames, err := fetchSelfhstIcons()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Cache{
|
||||
WalkxCode: walkxCodeIconMap,
|
||||
Selfhst: selfhstIconMap,
|
||||
DisplayNames: referenceToNames,
|
||||
IconList: append(walkxCodeIconList, selfhstIconList...),
|
||||
}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
format:
|
||||
|
||||
{
|
||||
"png": [
|
||||
"*.png",
|
||||
],
|
||||
"svg": [
|
||||
"*.svg",
|
||||
],
|
||||
"webp": [
|
||||
"*.webp",
|
||||
]
|
||||
}
|
||||
*/
|
||||
func fetchWalkxCodeIcons() (IconsMap, IconList, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
data := make(map[string][]string)
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
icons := make(IconsMap, len(data))
|
||||
iconList := make(IconList, 0, 2000)
|
||||
for fileType, files := range data {
|
||||
icons[fileType] = make(map[string]struct{}, len(files))
|
||||
for _, icon := range files {
|
||||
icons[fileType][icon] = struct{}{}
|
||||
iconList = append(iconList, "@walkxcode/"+icon)
|
||||
}
|
||||
}
|
||||
return icons, iconList, nil
|
||||
}
|
||||
|
||||
/*
|
||||
format:
|
||||
|
||||
{
|
||||
"Name": "2FAuth",
|
||||
"Reference": "2fauth",
|
||||
"SVG": "Yes",
|
||||
"PNG": "Yes",
|
||||
"WebP": "Yes",
|
||||
"Light": "Yes",
|
||||
"Category": "Self-Hosted",
|
||||
"CreatedAt": "2024-08-16 00:27:23+00:00"
|
||||
}
|
||||
*/
|
||||
func fetchSelfhstIcons() (IconsMap, IconList, ReferenceDisplayNameMap, error) {
|
||||
type SelfhStIcon struct {
|
||||
Name string `json:"Name"`
|
||||
Reference string `json:"Reference"`
|
||||
SVG string `json:"SVG"`
|
||||
PNG string `json:"PNG"`
|
||||
WebP string `json:"WebP"`
|
||||
// Light string
|
||||
// Category string
|
||||
// CreatedAt string
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, selfhstIcons, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
data := make([]SelfhStIcon, 0, 2000)
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
iconList := make(IconList, 0, len(data)*3)
|
||||
icons := make(IconsMap)
|
||||
icons["svg"] = make(map[string]struct{}, len(data))
|
||||
icons["png"] = make(map[string]struct{}, len(data))
|
||||
icons["webp"] = make(map[string]struct{}, len(data))
|
||||
|
||||
referenceToNames := make(ReferenceDisplayNameMap, len(data))
|
||||
|
||||
for _, item := range data {
|
||||
if item.SVG == "Yes" {
|
||||
icons["svg"][item.Reference+".svg"] = struct{}{}
|
||||
iconList = append(iconList, "@selfhst/"+item.Reference+".svg")
|
||||
}
|
||||
if item.PNG == "Yes" {
|
||||
icons["png"][item.Reference+".png"] = struct{}{}
|
||||
iconList = append(iconList, "@selfhst/"+item.Reference+".png")
|
||||
}
|
||||
if item.WebP == "Yes" {
|
||||
icons["webp"][item.Reference+".webp"] = struct{}{}
|
||||
iconList = append(iconList, "@selfhst/"+item.Reference+".webp")
|
||||
}
|
||||
referenceToNames[item.Reference] = item.Name
|
||||
}
|
||||
|
||||
return icons, iconList, referenceToNames, nil
|
||||
}
|
||||
20
internal/homepage/route.go
Normal file
20
internal/homepage/route.go
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
package homepage
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
gpnet "github.com/yusing/go-proxy/internal/net/types"
|
||||
"github.com/yusing/go-proxy/internal/utils/pool"
|
||||
)
|
||||
|
||||
type route interface {
|
||||
pool.Object
|
||||
ProviderName() string
|
||||
Reference() string
|
||||
TargetURL() *gpnet.URL
|
||||
}
|
||||
|
||||
type httpRoute interface {
|
||||
route
|
||||
http.Handler
|
||||
}
|
||||
|
|
@ -24,4 +24,5 @@ var (
|
|||
Get = httpClient.Get
|
||||
Post = httpClient.Post
|
||||
Head = httpClient.Head
|
||||
Do = httpClient.Do
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ import (
|
|||
|
||||
"github.com/yusing/go-proxy/agent/pkg/agent"
|
||||
"github.com/yusing/go-proxy/agent/pkg/agentproxy"
|
||||
"github.com/yusing/go-proxy/internal/api/v1/favicon"
|
||||
"github.com/yusing/go-proxy/internal/common"
|
||||
"github.com/yusing/go-proxy/internal/gperr"
|
||||
"github.com/yusing/go-proxy/internal/homepage"
|
||||
"github.com/yusing/go-proxy/internal/idlewatcher"
|
||||
gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
|
||||
"github.com/yusing/go-proxy/internal/net/gphttp/accesslog"
|
||||
|
|
@ -143,7 +143,7 @@ func (r *ReveseProxyRoute) Start(parent task.Parent) gperr.Error {
|
|||
})
|
||||
}
|
||||
|
||||
r.task.OnCancel("reset_favicon", func() { favicon.PruneRouteIconCache(r) })
|
||||
r.task.OnCancel("reset_favicon", func() { homepage.PruneRouteIconCache(r) })
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue