updated ls-icon and icon fetching mechanism

internal/api/v1/favicon/favicon.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/vincent-petithory/dataurl"
+	"github.com/yusing/go-proxy/internal"
 	U "github.com/yusing/go-proxy/internal/api/v1/utils"
 	"github.com/yusing/go-proxy/internal/homepage"
 	"github.com/yusing/go-proxy/internal/logging"
@@ -78,12 +79,15 @@ func GetFavIcon(w http.ResponseWriter, req *http.Request) {
 		var status int
 		var errMsg string
 
-		homepage := r.RawEntry().Homepage
+		hp := r.RawEntry().Homepage
-		if homepage != nil && homepage.Icon != nil {
+		if hp != nil && hp.Icon != nil {
-			if homepage.Icon.IsRelative {
+			switch hp.Icon.IconSource {
-				icon, status, errMsg = findIcon(r, req, homepage.Icon.Value)
+			case homepage.IconSourceAbsolute:
-			} else {
+				icon, status, errMsg = fetchIconAbsolute(hp.Icon.Value)
-				icon, status, errMsg = getIconAbsolute(homepage.Icon.Value)
+			case homepage.IconSourceRelative:
+				icon, status, errMsg = findIcon(r, req, hp.Icon.Value)
+			case homepage.IconSourceWalkXCode:
+				icon, status, errMsg = fetchWalkxcodeIcon(hp.Icon.Extra.FileType, hp.Icon.Extra.Name)
 			}
 		} else {
 			// try extract from "link[rel=icon]"
@@ -124,7 +128,7 @@ func storeIconCache(key string, icon []byte) {
 	iconCache[key] = icon
 }
 
-func getIconAbsolute(url string) ([]byte, int, string) {
+func fetchIconAbsolute(url string) ([]byte, int, string) {
 	icon, ok := loadIconCache(url)
 	if ok {
 		return icon, http.StatusOK, ""
@@ -165,6 +169,25 @@ func sanitizeName(name string) string {
 	return strings.ToLower(nameSanitizer.Replace(name))
 }
 
+func fetchWalkxcodeIcon(filetype string, name string) ([]byte, int, string) {
+	// if icon isn't in the list, no need to fetch
+	if !internal.HasIcon(name, filetype) {
+		logging.Debug().
+			Str("filetype", filetype).
+			Str("name", name).
+			Msg("icon not found")
+		return nil, http.StatusNotFound, "icon not found"
+	}
+
+	icon, ok := loadIconCache("walkxcode/" + filetype + "/" + name)
+	if ok {
+		return icon, http.StatusOK, ""
+	}
+
+	url := homepage.DashboardIconBaseURL + filetype + "/" + name + "." + filetype
+	return fetchIconAbsolute(url)
+}
+
 func findIcon(r route.HTTPRoute, req *http.Request, uri string) (icon []byte, status int, errMsg string) {
 	key := r.TargetName()
 	icon, ok := loadIconCache(key)
@@ -175,10 +198,10 @@ func findIcon(r route.HTTPRoute, req *http.Request, uri string) (icon []byte, st
 		return icon, http.StatusOK, ""
 	}
 
-	icon, status, errMsg = getIconAbsolute(homepage.DashboardIconBaseURL + "png/" + sanitizeName(r.TargetName()) + ".png")
+	icon, status, errMsg = fetchWalkxcodeIcon("png", sanitizeName(r.TargetName()))
 	cont := r.RawEntry().Container
 	if icon == nil && cont != nil {
-		icon, status, errMsg = getIconAbsolute(homepage.DashboardIconBaseURL + "png/" + sanitizeName(cont.ImageName) + ".png")
+		icon, status, errMsg = fetchWalkxcodeIcon("png", sanitizeName(cont.ImageName))
 	}
 	if icon == nil {
 		// fallback to parse html
@@ -224,10 +247,6 @@ func findIconSlow(r route.HTTPRoute, req *http.Request, uri string) (icon []byte
 				if loc == newReq.URL.Path {
 					return nil, http.StatusBadGateway, "circular redirect"
 				}
-				logging.Debug().Str("route", r.TargetName()).
-					Str("from", uri).
-					Str("to", loc).
-					Msg("favicon redirect")
 				return findIconSlow(r, req, loc)
 			}
 		}
@@ -264,8 +283,10 @@ func findIconSlow(r route.HTTPRoute, req *http.Request, uri string) (icon []byte
 		}
 		return dataURI.Data, http.StatusOK, ""
 	}
-	if href[0] != '/' {
+	switch {
-		return getIconAbsolute(href)
+	case strings.HasPrefix(href, "http://"), strings.HasPrefix(href, "https://"):
+		return fetchIconAbsolute(href)
+	default:
+		return findIconSlow(r, req, path.Clean(href))
 	}
-	return findIconSlow(r, req, href)
 }

internal/homepage/icon_url.go

@@ -6,10 +6,26 @@ import (
 	E "github.com/yusing/go-proxy/internal/error"
 )
 
-type IconURL struct {
+type (
-	Value      string `json:"value"`
+	IconURL struct {
-	IsRelative bool   `json:"is_relative"`
+		Value string `json:"value"`
-}
+		IconSource
+		Extra *IconExtra `json:"extra"`
+	}
+
+	IconExtra struct {
+		FileType string `json:"file_type"`
+		Name     string `json:"name"`
+	}
+
+	IconSource int
+)
+
+const (
+	IconSourceAbsolute IconSource = iota
+	IconSourceRelative
+	IconSourceWalkXCode
+)
 
 const DashboardIconBaseURL = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/"
 
@@ -28,13 +44,19 @@ func (u *IconURL) Parse(v string) error {
 	switch beforeSlash {
 	case "http:", "https:":
 		u.Value = v
+		u.IconSource = IconSourceAbsolute
 		return nil
 	case "@target":
 		u.Value = v[slashIndex:]
-		u.IsRelative = true
+		u.IconSource = IconSourceRelative
 		return nil
-	case "png", "svg": // walkXCode Icons
+	case "png", "svg", "webp": // walkXCode Icons
-		u.Value = DashboardIconBaseURL + v
+		u.Value = v
+		u.IconSource = IconSourceWalkXCode
+		u.Extra = &IconExtra{
+			FileType: beforeSlash,
+			Name:     strings.TrimSuffix(v[slashIndex+1:], "."+beforeSlash),
+		}
 		return nil
 	default:
 		return ErrInvalidIconURL

internal/homepage/icon_url_test.go

@@ -0,0 +1,67 @@
+package homepage
+
+import (
+	"testing"
+
+	. "github.com/yusing/go-proxy/internal/utils/testing"
+)
+
+func TestIconURL(t *testing.T) {
+	tests := []struct {
+		name      string
+		input     string
+		wantValue *IconURL
+		wantErr   bool
+	}{
+		{
+			name:  "absolute",
+			input: "http://example.com/icon.png",
+			wantValue: &IconURL{
+				Value:      "http://example.com/icon.png",
+				IconSource: IconSourceAbsolute,
+			},
+		},
+		{
+			name:  "relative",
+			input: "@target/icon.png",
+			wantValue: &IconURL{
+				Value:      "/icon.png",
+				IconSource: IconSourceRelative,
+			},
+		},
+		{
+			name:  "walkxcode",
+			input: "png/walkxcode.png",
+			wantValue: &IconURL{
+				Value:      "png/walkxcode.png",
+				IconSource: IconSourceWalkXCode,
+				Extra: &IconExtra{
+					FileType: "png",
+					Name:     "walkxcode",
+				},
+			},
+		},
+		{
+			name:    "invalid",
+			input:   "invalid",
+			wantErr: true,
+		},
+		{
+			name:    "empty",
+			input:   "",
+			wantErr: true,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			u := &IconURL{}
+			err := u.Parse(tc.input)
+			if tc.wantErr {
+				ExpectError(t, ErrInvalidIconURL, err)
+			} else {
+				ExpectNoError(t, err)
+				ExpectDeepEqual(t, u, tc.wantValue)
+			}
+		})
+	}
+}

internal/list-icons.go

@@ -2,14 +2,12 @@ package internal
 
 import (
 	"encoding/json"
-	"fmt"
 	"io"
-	"log"
 	"net/http"
-	"os"
+	"sync"
 	"time"
 
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/logging"
 )
 
 type GitHubContents struct { //! keep this, may reuse in future
@@ -20,54 +18,71 @@ type GitHubContents struct { //! keep this, may reuse in future
 	Size int    `json:"size"`
 }
 
-const (
+type Icons map[string]map[string]struct{}
-	iconsCachePath = "/tmp/icons_cache.json"
+
-	updateInterval = 1 * time.Hour
+// no longer cache for `godoxy ls-icons`
+
+const updateInterval = 1 * time.Hour
+
+var (
+	iconsCache   = make(Icons)
+	iconsCahceMu sync.Mutex
+	lastUpdate   time.Time
 )
 
-func ListAvailableIcons() ([]string, error) {
+const walkxcodeIcons = "https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons@master/tree.json"
-	owner := "walkxcode"
-	repo := "dashboard-icons"
-	ref := "main"
 
-	var lastUpdate time.Time
+func ListAvailableIcons() (Icons, error) {
+	iconsCahceMu.Lock()
+	defer iconsCahceMu.Unlock()
 
-	icons := make([]string, 0)
-	info, err := os.Stat(iconsCachePath)
-	if err == nil {
-		lastUpdate = info.ModTime().Local()
-	}
 	if time.Since(lastUpdate) < updateInterval {
-		err := utils.LoadJSON(iconsCachePath, &icons)
+		if len(iconsCache) > 0 {
-		if err == nil {
+			return iconsCache, nil
-			return icons, nil
 		}
 	}
 
-	contents, err := getRepoContents(http.DefaultClient, owner, repo, ref, "")
+	icons, err := getIcons()
 	if err != nil {
 		return nil, err
 	}
-	for _, content := range contents {
+
-		if content.Type != "dir" {
+	iconsCache = icons
-			icons = append(icons, content.Path)
+	lastUpdate = time.Now()
-		}
-	}
-	err = utils.SaveJSON(iconsCachePath, &icons, 0o644)
-	if err != nil {
-		log.Print("error saving cache", err)
-	}
 	return icons, nil
 }
 
-func getRepoContents(client *http.Client, owner string, repo string, ref string, path string) ([]GitHubContents, error) {
+func HasIcon(name string, filetype string) bool {
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s", owner, repo, path, ref), nil)
+	icons, err := ListAvailableIcons()
+	if err != nil {
+		logging.Error().Err(err).Msg("failed to list icons")
+		return false
+	}
+	if _, ok := icons[filetype]; !ok {
+		return false
+	}
+	_, ok := icons[filetype][name+"."+filetype]
+	return ok
+}
+
+/*
+format:
+
+	{
+		"png": [
+			"*.png",
+		],
+		"svg": [
+			"*.svg",
+		]
+	}
+*/
+func getIcons() (Icons, error) {
+	req, err := http.NewRequest(http.MethodGet, walkxcodeIcons, nil)
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Accept", "application/json")
+	resp, err := http.DefaultClient.Do(req)
-
-	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -78,24 +93,17 @@ func getRepoContents(client *http.Client, owner string, repo string, ref string,
 		return nil, err
 	}
 
-	var contents []GitHubContents
+	data := make(map[string][]string)
-	err = json.Unmarshal(body, &contents)
+	err = json.Unmarshal(body, &data)
 	if err != nil {
 		return nil, err
 	}
-
+	icons := make(Icons, len(data))
-	filesAndDirs := make([]GitHubContents, 0)
+	for fileType, files := range data {
-	for _, content := range contents {
+		icons[fileType] = make(map[string]struct{}, len(files))
-		if content.Type == "dir" {
+		for _, icon := range files {
-			subContents, err := getRepoContents(client, owner, repo, ref, content.Path)
+			icons[fileType][icon] = struct{}{}
-			if err != nil {
-				return nil, err
-			}
-			filesAndDirs = append(filesAndDirs, subContents...)
-		} else {
-			filesAndDirs = append(filesAndDirs, content)
 		}
 	}
-
+	return icons, nil
-	return filesAndDirs, nil
 }

schema/providers.schema.json

@@ -85,7 +85,7 @@
               "type": "string",
               "oneOf": [
                 {
-                  "pattern": "^(png|svg)\\/[\\w\\d\\-_]+\\.\\1$",
+                  "pattern": "^(png|svg|webp)\\/[\\w\\d\\-_]+\\.\\1$",
                   "title": "Icon from walkxcode/dashboard-icons"
                 },
                 {