Release 3.9.0+2: tolerant subscription import, full Remnawave server list, subscription info card (expiry / traffic / devices).
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Navis
+216
-45
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
@@ -25,13 +26,45 @@ type Item struct {
|
||||
URI string
|
||||
}
|
||||
|
||||
// Info is subscription usage metadata from the subscription-userinfo header
|
||||
// (upload/download/total in bytes, expire as unix seconds; 0 = unknown).
|
||||
type Info struct {
|
||||
Upload int64 `json:"upload,omitempty"`
|
||||
Download int64 `json:"download,omitempty"`
|
||||
Total int64 `json:"total,omitempty"`
|
||||
Expire int64 `json:"expire,omitempty"`
|
||||
}
|
||||
|
||||
// Result is a parsed subscription: imported items plus per-line skip diagnostics.
|
||||
type Result struct {
|
||||
Items []Item
|
||||
Skipped int
|
||||
Warnings []string
|
||||
Info *Info
|
||||
}
|
||||
|
||||
const maxWarnings = 8
|
||||
|
||||
var (
|
||||
reShareLine = regexp.MustCompile(`(?i)((?:hysteria2|hy2|vless|vmess|trojan|awg|amneziawg|wireguard|naive\+https|naive\+quic|naive|quic|https|http)://[^\s"'<>]+)`)
|
||||
reClashHY2 = regexp.MustCompile(`(?is)type:\s*hysteria2\b.*?server:\s*(\S+).*?port:\s*(\d+).*?(?:password|auth):\s*["']?([^\s"']+)`)
|
||||
// reURILine matches a standalone "scheme://..." line — only such lines are
|
||||
// counted as skipped entries (YAML/HTML noise is ignored silently).
|
||||
reURILine = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*://\S`)
|
||||
)
|
||||
|
||||
// userAgents to try in order. Remnawave and most panels return raw share links
|
||||
// for a generic UA, but return a Clash YAML config for Clash-like UAs — that
|
||||
// YAML loses vless/trojan entries for us, so try the plain UA first.
|
||||
var userAgents = []string{
|
||||
"Navis/3.9 (+https://evilfox.win)",
|
||||
"ClashMeta/1.18.0",
|
||||
}
|
||||
|
||||
// Fetch downloads a subscription body and parses proxy share links.
|
||||
func Fetch(ctx context.Context, rawURL string) ([]Item, error) {
|
||||
// Parsing is tolerant: unsupported or broken entries are skipped (with
|
||||
// warnings), valid ones are imported.
|
||||
func Fetch(ctx context.Context, rawURL string) (*Result, error) {
|
||||
rawURL = strings.TrimSpace(rawURL)
|
||||
if rawURL == "" {
|
||||
return nil, fmt.Errorf("пустой URL подписки")
|
||||
@@ -41,37 +74,116 @@ func Fetch(ctx context.Context, rawURL string) ([]Item, error) {
|
||||
return nil, fmt.Errorf("нужен http(s) URL подписки")
|
||||
}
|
||||
|
||||
var (
|
||||
best *Result
|
||||
info *Info
|
||||
lastErr error
|
||||
)
|
||||
for _, ua := range userAgents {
|
||||
body, header, err := download(ctx, rawURL, ua)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if info == nil {
|
||||
info = ParseUserInfoHeader(header.Get("Subscription-Userinfo"))
|
||||
}
|
||||
res, err := ParseBodyDetailed(body)
|
||||
if res != nil && (best == nil || len(res.Items) > 0 || res.Skipped > best.Skipped) {
|
||||
best = res
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if res != nil && len(res.Items) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if best == nil || len(best.Items) == 0 {
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
return nil, fmt.Errorf("в подписке нет поддерживаемых ссылок")
|
||||
}
|
||||
best.Info = info
|
||||
return best, nil
|
||||
}
|
||||
|
||||
func download(ctx context.Context, rawURL, userAgent string) (string, http.Header, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return "", nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "ClashMeta/1.18.0")
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
req.Header.Set("Accept", "*/*")
|
||||
client := &http.Client{Timeout: 45 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("не удалось скачать подписку: %w", err)
|
||||
return "", nil, fmt.Errorf("не удалось скачать подписку: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("подписка HTTP %d", resp.StatusCode)
|
||||
return "", nil, fmt.Errorf("подписка HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return "", nil, err
|
||||
}
|
||||
items, err := ParseBody(string(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return string(body), resp.Header, nil
|
||||
}
|
||||
|
||||
// ParseUserInfoHeader parses "upload=..; download=..; total=..; expire=<unix>".
|
||||
// Returns nil when the header is absent or has no recognized fields.
|
||||
func ParseUserInfoHeader(v string) *Info {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
return items, nil
|
||||
info := &Info{}
|
||||
found := false
|
||||
for _, part := range strings.Split(v, ";") {
|
||||
kv := strings.SplitN(part, "=", 2)
|
||||
if len(kv) != 2 {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(kv[1]), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(kv[0])) {
|
||||
case "upload":
|
||||
info.Upload, found = n, true
|
||||
case "download":
|
||||
info.Download, found = n, true
|
||||
case "total":
|
||||
info.Total, found = n, true
|
||||
case "expire":
|
||||
info.Expire, found = n, true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// ParseBody accepts plain text lines, base64, or Clash-like blobs with share links.
|
||||
func ParseBody(body string) ([]Item, error) {
|
||||
res, err := ParseBodyDetailed(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res.Items, nil
|
||||
}
|
||||
|
||||
// ParseBodyDetailed parses a subscription body and reports skipped entries.
|
||||
// The returned Result is non-nil even on error (zero valid entries) so callers
|
||||
// can surface the collected warnings.
|
||||
func ParseBodyDetailed(body string) (*Result, error) {
|
||||
body = strings.TrimSpace(body)
|
||||
if body == "" {
|
||||
return nil, fmt.Errorf("пустая подписка")
|
||||
return &Result{}, fmt.Errorf("пустая подписка")
|
||||
}
|
||||
|
||||
candidates := []string{body}
|
||||
@@ -83,56 +195,68 @@ func ParseBody(body string) ([]Item, error) {
|
||||
}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
var best *Result
|
||||
for _, text := range candidates {
|
||||
items, err := parseText(text)
|
||||
if err == nil && len(items) > 0 {
|
||||
return items, nil
|
||||
res := parseText(text)
|
||||
if len(res.Items) > 0 {
|
||||
return res, nil
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if best == nil || res.Skipped > best.Skipped {
|
||||
best = res
|
||||
}
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
if best == nil {
|
||||
best = &Result{}
|
||||
}
|
||||
return nil, fmt.Errorf("в подписке нет поддерживаемых ссылок (naive / hysteria2)")
|
||||
if best.Skipped > 0 {
|
||||
return best, fmt.Errorf("нет поддерживаемых ссылок: %d записей пропущено (%s)",
|
||||
best.Skipped, strings.Join(best.Warnings, "; "))
|
||||
}
|
||||
return best, fmt.Errorf("в подписке нет поддерживаемых ссылок (naive / hysteria2 / awg / vless / vmess / trojan)")
|
||||
}
|
||||
|
||||
func parseText(text string) ([]Item, error) {
|
||||
lines := splitLines(text)
|
||||
// Also extract share URIs embedded in YAML/HTML/JSON.
|
||||
extracted := reShareLine.FindAllString(text, -1)
|
||||
for _, m := range extracted {
|
||||
lines = append(lines, strings.TrimRight(m, ".,;)]}\"'"))
|
||||
}
|
||||
for _, block := range extractClashHY2(text) {
|
||||
lines = append(lines, block)
|
||||
}
|
||||
|
||||
out := make([]Item, 0, len(lines))
|
||||
func parseText(text string) *Result {
|
||||
res := &Result{}
|
||||
seen := map[string]int{}
|
||||
seenURI := map[string]struct{}{}
|
||||
for _, line := range lines {
|
||||
|
||||
// countSkips is true only for raw subscription lines; URIs extracted from
|
||||
// YAML/HTML/JSON noise are best-effort and never counted as "skipped".
|
||||
process := func(line string, countSkips bool) {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
return
|
||||
}
|
||||
line = strings.Trim(line, `"'`)
|
||||
looksURI := reURILine.MatchString(line)
|
||||
skip := func(reason string) {
|
||||
if !countSkips || !looksURI {
|
||||
return
|
||||
}
|
||||
res.Skipped++
|
||||
if len(res.Warnings) < maxWarnings {
|
||||
res.Warnings = append(res.Warnings, shortenLink(line)+" — "+reason)
|
||||
}
|
||||
}
|
||||
|
||||
proto := config.DetectProtocol(line)
|
||||
if proto == "" {
|
||||
continue
|
||||
skip("неподдерживаемая схема")
|
||||
return
|
||||
}
|
||||
// Skip bare https:// without credentials (common noise in HTML/YAML).
|
||||
// Bare https:// without credentials (common noise in HTML/YAML,
|
||||
// or a subscription URL pasted among the links) is not a proxy link.
|
||||
if proto == config.ProtocolNaive && !naiveLinkHasAuth(line) && !strings.HasPrefix(strings.ToLower(line), "naive+") {
|
||||
continue
|
||||
skip("нет логина и пароля в ссылке")
|
||||
return
|
||||
}
|
||||
normalized, detected, remark, err := linknorm.Normalize(proto, line)
|
||||
if err != nil {
|
||||
continue
|
||||
skip(err.Error())
|
||||
return
|
||||
}
|
||||
if _, dup := seenURI[normalized]; dup {
|
||||
continue
|
||||
return
|
||||
}
|
||||
seenURI[normalized] = struct{}{}
|
||||
name := remark
|
||||
@@ -147,12 +271,33 @@ func parseText(text string) ([]Item, error) {
|
||||
name = fmt.Sprintf("%s-%d", base, n)
|
||||
}
|
||||
seen[name] = 1
|
||||
out = append(out, Item{Name: name, Protocol: detected, URI: normalized})
|
||||
res.Items = append(res.Items, Item{Name: name, Protocol: detected, URI: normalized})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("в подписке нет поддерживаемых ссылок (naive / hysteria2)")
|
||||
|
||||
for _, line := range splitLines(text) {
|
||||
process(line, true)
|
||||
}
|
||||
return out, nil
|
||||
// Also extract share URIs embedded in YAML/HTML/JSON.
|
||||
for _, m := range reShareLine.FindAllString(text, -1) {
|
||||
process(strings.TrimRight(m, ".,;)]}\"'"), false)
|
||||
}
|
||||
for _, block := range extractClashHY2(text) {
|
||||
process(block, false)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// shortenLink trims a link for warning messages, hiding query/credentials noise.
|
||||
func shortenLink(line string) string {
|
||||
if i := strings.IndexAny(line, "?#"); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
const max = 48
|
||||
runes := []rune(line)
|
||||
if len(runes) > max {
|
||||
return string(runes[:max]) + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func naiveLinkHasAuth(line string) bool {
|
||||
@@ -175,7 +320,7 @@ func extractClashHY2(text string) []string {
|
||||
}
|
||||
var out []string
|
||||
// Split roughly by list items.
|
||||
chunks := strings.Split(text, "\n- ")
|
||||
chunks := splitClashItems(text)
|
||||
for _, chunk := range chunks {
|
||||
cl := strings.ToLower(chunk)
|
||||
if !strings.Contains(cl, "type:") || !strings.Contains(cl, "hysteria2") {
|
||||
@@ -223,8 +368,34 @@ func extractClashHY2(text string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// reClashItemStart matches a mapping list item ("- name: X", " - type: y"),
|
||||
// but not nested scalar list entries like "- h3" under alpn.
|
||||
var reClashItemStart = regexp.MustCompile(`^\s*-\s+[^\s:]+\s*:`)
|
||||
|
||||
// splitClashItems splits a Clash YAML proxies list into per-proxy chunks,
|
||||
// tolerating any list-item indentation ("- name:" at column 0 or nested).
|
||||
func splitClashItems(text string) []string {
|
||||
lines := splitLines(text)
|
||||
var out []string
|
||||
var cur []string
|
||||
flush := func() {
|
||||
if len(cur) > 0 {
|
||||
out = append(out, strings.Join(cur, "\n"))
|
||||
cur = nil
|
||||
}
|
||||
}
|
||||
for _, line := range lines {
|
||||
if reClashItemStart.MatchString(line) {
|
||||
flush()
|
||||
}
|
||||
cur = append(cur, line)
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
func yamlField(block, key string) string {
|
||||
re := regexp.MustCompile(`(?im)^\s*` + regexp.QuoteMeta(key) + `\s*:\s*["']?([^"'#\n\r]+)["']?`)
|
||||
re := regexp.MustCompile(`(?im)^\s*(?:-\s+)?` + regexp.QuoteMeta(key) + `\s*:\s*["']?([^"'#\n\r]+)["']?`)
|
||||
m := re.FindStringSubmatch(block)
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package subscription
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -32,6 +35,138 @@ func TestParseBodyBase64(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseBodyTolerant: one broken entry must never abort the import —
|
||||
// valid links are kept, unsupported/broken lines are skipped with warnings.
|
||||
func TestParseBodyTolerant(t *testing.T) {
|
||||
body := `vless://7e1c0fbf-1758-4781-97f6-ae8715ed3f60@cz.example.com:443?encryption=none&security=tls&type=tcp#CZ
|
||||
|
||||
hysteria2://secret@de.example.com:443/?sni=de.example.com#DE-HY2
|
||||
https://api.example.com/ZKszcVcC3xbWb8qj
|
||||
foobar://what@is.this:1
|
||||
# comment line
|
||||
`
|
||||
res, err := ParseBodyDetailed(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res.Items) != 2 {
|
||||
t.Fatalf("imported %d items, want 2: %+v", len(res.Items), res.Items)
|
||||
}
|
||||
if res.Items[0].Protocol != "vless" || res.Items[1].Protocol != "hysteria2" {
|
||||
t.Fatalf("protocols: %s, %s", res.Items[0].Protocol, res.Items[1].Protocol)
|
||||
}
|
||||
// https:// without userinfo + unknown scheme → skipped; empty/comment lines are not counted.
|
||||
if res.Skipped != 2 {
|
||||
t.Fatalf("skipped %d, want 2 (%v)", res.Skipped, res.Warnings)
|
||||
}
|
||||
if len(res.Warnings) != 2 {
|
||||
t.Fatalf("warnings: %v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBodyZeroValidListsReasons(t *testing.T) {
|
||||
body := "https://api.example.com/sub-token-line\nfoobar://x@y:1\n"
|
||||
res, err := ParseBodyDetailed(body)
|
||||
if err == nil {
|
||||
t.Fatal("want error for zero valid entries")
|
||||
}
|
||||
if res.Skipped != 2 {
|
||||
t.Fatalf("skipped %d, want 2", res.Skipped)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "2 записей пропущено") {
|
||||
t.Fatalf("error should list skip count: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUserInfoHeader(t *testing.T) {
|
||||
info := ParseUserInfoHeader("upload=0; download=5679561725; total=53687091200000; expire=1799366585")
|
||||
if info == nil {
|
||||
t.Fatal("nil info")
|
||||
}
|
||||
if info.Download != 5679561725 || info.Total != 53687091200000 || info.Expire != 1799366585 || info.Upload != 0 {
|
||||
t.Fatalf("bad parse: %+v", info)
|
||||
}
|
||||
if ParseUserInfoHeader("") != nil {
|
||||
t.Fatal("empty header must give nil")
|
||||
}
|
||||
if ParseUserInfoHeader("garbage") != nil {
|
||||
t.Fatal("garbage header must give nil")
|
||||
}
|
||||
}
|
||||
|
||||
// Remnawave returns Clash YAML for Clash-like UAs and raw base64 links otherwise.
|
||||
// Fetch must obtain the full server list (plain UA) and the userinfo header.
|
||||
func TestFetchPrefersPlainUA(t *testing.T) {
|
||||
links := "vless://7e1c0fbf-1758-4781-97f6-ae8715ed3f60@cz.example.com:443?encryption=none&security=tls&type=tcp#CZ\n" +
|
||||
"trojan://pass@no.example.com:20000?security=tls&type=ws#NO\n" +
|
||||
"hysteria2://secret@de.example.com:443/?sni=de.example.com#DE\n" +
|
||||
"ss://YWVzLTI1Ni1nY206cGFzcw@ss.example.com:8388#SS\n"
|
||||
clashYAML := "proxies:\n - name: only-hy2\n type: hysteria2\n server: de.example.com\n port: 443\n password: secret\n"
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Subscription-Userinfo", "upload=1; download=2; total=100; expire=1799366585")
|
||||
if strings.Contains(strings.ToLower(r.Header.Get("User-Agent")), "clash") {
|
||||
_, _ = w.Write([]byte(clashYAML))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(base64.StdEncoding.EncodeToString([]byte(links))))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := Fetch(context.Background(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res.Items) != 3 {
|
||||
t.Fatalf("imported %d, want 3 (vless+trojan+hy2): %+v", len(res.Items), res.Items)
|
||||
}
|
||||
if res.Skipped != 1 {
|
||||
t.Fatalf("skipped %d, want 1 (ss://)", res.Skipped)
|
||||
}
|
||||
if res.Info == nil || res.Info.Expire != 1799366585 || res.Info.Total != 100 {
|
||||
t.Fatalf("info: %+v", res.Info)
|
||||
}
|
||||
}
|
||||
|
||||
// Indented Clash proxies with nested lists (alpn) must split into separate blocks.
|
||||
func TestParseBodyClashYAMLIndented(t *testing.T) {
|
||||
body := `proxies:
|
||||
- name: DE HY2
|
||||
type: hysteria2
|
||||
server: de.example.com
|
||||
port: 443
|
||||
udp: true
|
||||
password: secret1
|
||||
sni: de.example.com
|
||||
alpn:
|
||||
- h3
|
||||
- name: NO HY2
|
||||
type: hysteria2
|
||||
server: no.example.com
|
||||
port: 9000
|
||||
password: secret2
|
||||
alpn:
|
||||
- h3
|
||||
`
|
||||
items, err := ParseBody(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("got %d items, want 2: %+v", len(items), items)
|
||||
}
|
||||
for i, host := range []string{"de.example.com:443", "no.example.com:9000"} {
|
||||
if !strings.Contains(items[i].URI, host) {
|
||||
t.Fatalf("item %d uri %s missing %s", i, items[i].URI, host)
|
||||
}
|
||||
}
|
||||
for i, pass := range []string{"secret1", "secret2"} {
|
||||
if !strings.Contains(items[i].URI, pass) {
|
||||
t.Fatalf("item %d uri %s missing password %s", i, items[i].URI, pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBodyClashYAML(t *testing.T) {
|
||||
body := `
|
||||
proxies:
|
||||
|
||||
Reference in New Issue
Block a user