diff --git a/gateway/core/corehttp/corehttp.go b/gateway/core/corehttp/corehttp.go index 1b0a79ee6..143327149 100644 --- a/gateway/core/corehttp/corehttp.go +++ b/gateway/core/corehttp/corehttp.go @@ -16,7 +16,7 @@ import ( "github.com/jbenet/goprocess" periodicproc "github.com/jbenet/goprocess/periodic" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("core/server") diff --git a/gateway/core/corehttp/gateway_handler.go b/gateway/core/corehttp/gateway_handler.go index a1549efdd..b9e7f144b 100644 --- a/gateway/core/corehttp/gateway_handler.go +++ b/gateway/core/corehttp/gateway_handler.go @@ -328,8 +328,20 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request size = humanize.Bytes(uint64(s)) } + hash := "" + if r, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name())); err == nil { + // Path may not be resolved. Continue anyways. + hash = r.Cid().String() + } + // See comment above where originalUrlPath is declared. - di := directoryItem{size, dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())} + di := directoryItem{ + Size: size, + Name: dirit.Name(), + Path: gopath.Join(originalUrlPath, dirit.Name()), + Hash: hash, + ShortHash: shortHash(hash), + } dirListing = append(dirListing, di) } if dirit.Err() != nil { @@ -359,14 +371,34 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } } + size := "?" + if s, err := dir.Size(); err == nil { + // Size may not be defined/supported. Continue anyways. + size = humanize.Bytes(uint64(s)) + } + hash := resolvedPath.Cid().String() + // Storage for gateway URL to be used when linking to other rootIDs. This + // will be blank unless subdomain resolution is being used for this request. + var gwURL string + + // Get gateway hostname and build gateway URL. + if h, ok := r.Context().Value("gw-hostname").(string); ok { + gwURL = "//" + h + } else { + gwURL = "" + } + // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ - Listing: dirListing, - Path: urlPath, - BackLink: backLink, - Hash: hash, + GatewayURL: gwURL, + Listing: dirListing, + Size: size, + Path: urlPath, + Breadcrumbs: breadcrumbs(urlPath), + BackLink: backLink, + Hash: hash, } err = listingTemplate.Execute(w, tplData) diff --git a/gateway/core/corehttp/gateway_indexPage.go b/gateway/core/corehttp/gateway_indexPage.go index 5575baea4..c9a948708 100644 --- a/gateway/core/corehttp/gateway_indexPage.go +++ b/gateway/core/corehttp/gateway_indexPage.go @@ -7,22 +7,61 @@ import ( "strings" "github.com/ipfs/go-ipfs/assets" + ipfspath "github.com/ipfs/go-path" ) // structs for directory listing type listingTemplateData struct { - Listing []directoryItem - Path string - BackLink string - Hash string + GatewayURL string + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string } type directoryItem struct { - Size string + Size string + Name string + Path string + Hash string + ShortHash string +} + +type breadcrumb struct { Name string Path string } +func breadcrumbs(urlPath string) []breadcrumb { + var ret []breadcrumb + + p, err := ipfspath.ParsePath(urlPath) + if err != nil { + // No breadcrumbs, fallback to bare Path in template + return ret + } + + segs := p.Segments() + for i, seg := range segs { + if i == 0 { + ret = append(ret, breadcrumb{Name: seg}) + } else { + ret = append(ret, breadcrumb{ + Name: seg, + Path: "/" + strings.Join(segs[0:i+1], "/"), + }) + } + } + + return ret +} + +func shortHash(hash string) string { + return (hash[0:4] + "\u2026" + hash[len(hash)-4:]) +} + var listingTemplate *template.Template func init() { diff --git a/gateway/core/corehttp/gateway_test.go b/gateway/core/corehttp/gateway_test.go index a8a07aa48..f4d6d810d 100644 --- a/gateway/core/corehttp/gateway_test.go +++ b/gateway/core/corehttp/gateway_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "regexp" "strings" "testing" "time" @@ -135,6 +136,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface // listener, and server with handler. yay cycles. dh := &delegatedHandler{} ts := httptest.NewServer(dh) + t.Cleanup(func() { ts.Close() }) dh.Handler, err = makeHandler(n, ts.Listener, @@ -154,10 +156,14 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface return ts, api, n.Context() } +func matchPathOrBreadcrumbs(s string, expected string) bool { + matched, _ := regexp.MatchString("Index of\n[\t ]*"+regexp.QuoteMeta(expected), s) + return matched +} + func TestGatewayGet(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) - defer ts.Close() k, err := api.Unixfs().Add(ctx, files.NewBytesFile([]byte("fnord"))) if err != nil { @@ -238,7 +244,6 @@ func TestGatewayGet(t *testing.T) { func TestPretty404(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) - defer ts.Close() f1 := files.NewMapDirectory(map[string]files.Node{ "ipfs-404.html": files.NewBytesFile([]byte("Custom 404")), @@ -303,7 +308,6 @@ func TestIPNSHostnameRedirect(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() // create /ipns/example.net/foo/index.html @@ -391,7 +395,6 @@ func TestIPNSHostnameBacklinks(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() f1 := files.NewMapDirectory(map[string]files.Node{ "file.txt": files.NewBytesFile([]byte("1")), @@ -442,7 +445,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s := string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net/foo? #<'/") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -475,7 +478,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /") { + if !matchPathOrBreadcrumbs(s, "/") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -508,7 +511,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net/foo? #<'/bar/") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'/bar") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -542,7 +545,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -584,7 +587,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /") { + if !matchPathOrBreadcrumbs(s, "/") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -601,7 +604,6 @@ func TestIPNSHostnameBacklinks(t *testing.T) { func TestCacheControlImmutable(t *testing.T) { ts, _, _ := newTestServerAndNode(t, nil) t.Logf("test server url: %s", ts.URL) - defer ts.Close() req, err := http.NewRequest(http.MethodGet, ts.URL+emptyDir+"/", nil) if err != nil { @@ -627,7 +629,6 @@ func TestCacheControlImmutable(t *testing.T) { func TestGoGetSupport(t *testing.T) { ts, _, _ := newTestServerAndNode(t, nil) t.Logf("test server url: %s", ts.URL) - defer ts.Close() // mimic go-get req, err := http.NewRequest(http.MethodGet, ts.URL+emptyDir+"?go-get=1", nil) @@ -651,7 +652,6 @@ func TestVersion(t *testing.T) { ns := mockNamesys{} ts, _, _ := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() req, err := http.NewRequest(http.MethodGet, ts.URL+"/version", nil) if err != nil { diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index d8656da23..8b2666afb 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strings" cid "github.com/ipfs/go-cid" @@ -24,29 +25,32 @@ import ( var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/version"} -var pathGatewaySpec = config.GatewaySpec{ +var pathGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: false, } -var subdomainGatewaySpec = config.GatewaySpec{ +var subdomainGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: true, } -var defaultKnownGateways = map[string]config.GatewaySpec{ +var defaultKnownGateways = map[string]*config.GatewaySpec{ "localhost": subdomainGatewaySpec, "ipfs.io": pathGatewaySpec, "gateway.ipfs.io": pathGatewaySpec, "dweb.link": subdomainGatewaySpec, } +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + // HostnameOption rewrites an incoming request based on the Host header. func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { childMux := http.NewServeMux() - coreApi, err := coreapi.NewCoreAPI(n) + coreAPI, err := coreapi.NewCoreAPI(n) if err != nil { return nil, err } @@ -55,22 +59,8 @@ func HostnameOption() ServeOption { if err != nil { return nil, err } - knownGateways := make( - map[string]config.GatewaySpec, - len(defaultKnownGateways)+len(cfg.Gateway.PublicGateways), - ) - for hostname, gw := range defaultKnownGateways { - knownGateways[hostname] = gw - } - for hostname, gw := range cfg.Gateway.PublicGateways { - if gw == nil { - // Allows the user to remove gateways but _also_ - // allows us to continuously update the list. - delete(knownGateways, hostname) - } else { - knownGateways[hostname] = *gw - } - } + + knownGateways := prepareKnownGateways(cfg.Gateway.PublicGateways) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // Unfortunately, many (well, ipfs.io) gateways use @@ -81,8 +71,15 @@ func HostnameOption() ServeOption { // and the paths that they serve "gateway" content on. // That way, we can use DNSLink for everything else. + // Support X-Forwarded-Host if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host + host := r.Host + if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" { + host = xHost + } + // HTTP Host & Path check: is this one of our "known gateways"? - if gw, ok := isKnownHostname(r.Host, knownGateways); ok { + if gw, ok := isKnownHostname(host, knownGateways); ok { // This is a known gateway but request is not using // the subdomain feature. @@ -94,7 +91,12 @@ func HostnameOption() ServeOption { if gw.UseSubdomains { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link - if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok { + newURL, err := toSubdomainURL(host, r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { // Just to be sure single Origin can't be abused in // web browsers that ignored the redirect for some // reason, Clear-Site-Data header clears browsing @@ -124,9 +126,9 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink - r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) return } @@ -138,36 +140,68 @@ func HostnameOption() ServeOption { // HTTP Host check: is this one of our subdomain-based "known gateways"? // Example: {cid}.ipfs.localhost, {cid}.ipfs.dweb.link - if gw, hostname, ns, rootID, ok := knownSubdomainDetails(r.Host, knownGateways); ok { - // Looks like we're using known subdomain gateway. + if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok { + // Looks like we're using a known gateway in subdomain mode. + + // Add gateway hostname context for linking to other root ids. + // Example: localhost/ipfs/{cid} + ctx := context.WithValue(r.Context(), "gw-hostname", hostname) // Assemble original path prefix. pathPrefix := "/" + ns + "/" + rootID - // Does this gateway _handle_ this path? + // Does this gateway _handle_ subdomains AND this path? if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) { // If not, resource does not exist, return 404 http.NotFound(w, r) return } - // Do we need to fix multicodec in PeerID represented as CIDv1? - if isPeerIDNamespace(ns) { - keyCid, err := cid.Decode(rootID) - if err == nil && keyCid.Type() != cid.Libp2pKey { - if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { - // Redirect to CID fixed inside of toSubdomainURL() + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSPrefix(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } } // Rewrite the path to not use subdomains r.URL.Path = pathPrefix + r.URL.Path // Serve path request - childMux.ServeHTTP(w, r) + childMux.ServeHTTP(w, r.WithContext(ctx)) return } // We don't have a known gateway. Fallback on DNSLink lookup @@ -176,9 +210,9 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink - r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) return } @@ -190,22 +224,83 @@ func HostnameOption() ServeOption { } } +type gatewayHosts struct { + exact map[string]*config.GatewaySpec + wildcard []wildcardHost +} + +type wildcardHost struct { + re *regexp.Regexp + spec *config.GatewaySpec +} + +func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts { + var hosts gatewayHosts + + hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways)+len(defaultKnownGateways)) + + // First, implicit defaults such as subdomain gateway on localhost + for hostname, gw := range defaultKnownGateways { + hosts.exact[hostname] = gw + } + + // Then apply values from Gateway.PublicGateways, if present in the config + for hostname, gw := range publicGateways { + if gw == nil { + // Remove any implicit defaults, if present. This is useful when one + // wants to disable subdomain gateway on localhost etc. + delete(hosts.exact, hostname) + continue + } + if strings.Contains(hostname, "*") { + // from *.domain.tld, construct a regexp that match any direct subdomain + // of .domain.tld. + // + // Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$ + + escaped := strings.ReplaceAll(hostname, ".", `\.`) + regexed := strings.ReplaceAll(escaped, "*", "[^.]+") + + re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed)) + if err != nil { + log.Warn("invalid wildcard gateway hostname \"%s\"", hostname) + } + + hosts.wildcard = append(hosts.wildcard, wildcardHost{re: re, spec: gw}) + } else { + hosts.exact[hostname] = gw + } + } + + return hosts +} + // isKnownHostname checks Gateway.PublicGateways and returns matching // GatewaySpec with gracefull fallback to version without port -func isKnownHostname(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, ok bool) { +func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, ok bool) { // Try hostname (host+optional port - value from Host header as-is) - if gw, ok := knownGateways[hostname]; ok { + if gw, ok := knownGateways.exact[hostname]; ok { return gw, ok } - // Fallback to hostname without port - gw, ok = knownGateways[stripPort(hostname)] - return gw, ok + // Also test without port + if gw, ok = knownGateways.exact[stripPort(hostname)]; ok { + return gw, ok + } + + // Wildcard support. Test both with and without port. + for _, host := range knownGateways.wildcard { + if host.re.MatchString(hostname) { + return host.spec, true + } + } + + return nil, false } -// Parses Host header and looks for a known subdomain gateway host. +// Parses Host header and looks for a known gateway matching subdomain host. // If found, returns GatewaySpec and subdomain components. // Note: hostname is host + optional port -func knownSubdomainDetails(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { +func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { labels := strings.Split(hostname, ".") // Look for FQDN of a known gateway hostname. // Example: given "dist.ipfs.io.ipns.dweb.link": @@ -230,14 +325,14 @@ func knownSubdomainDetails(hostname string, knownGateways map[string]config.Gate rootID := strings.Join(labels[:i-1], ".") return gw, fqdn, ns, rootID, true } - // not a known subdomain gateway - return gw, "", "", "", false + // no match + return nil, "", "", "", false } // isDNSLinkRequest returns bool that indicates if request // should return data from content path listed in DNSLink record (if exists) -func isDNSLinkRequest(ctx context.Context, ipfs iface.CoreAPI, r *http.Request) bool { - fqdn := stripPort(r.Host) +func isDNSLinkRequest(ctx context.Context, ipfs iface.CoreAPI, host string) bool { + fqdn := stripPort(host) if len(fqdn) == 0 && !isd.IsDomain(fqdn) { return false } @@ -266,18 +361,38 @@ func isPeerIDNamespace(ns string) bool { } } +// Converts an identifier to DNS-safe representation that fits in 63 characters +func toDNSPrefix(rootID string, rootCID cid.Cid) (prefix string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { +func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery parts := strings.SplitN(path, "/", 4) - safeRedirectURL := func(in string) (out string, ok bool) { + safeRedirectURL := func(in string) (out string, err error) { safeURI, err := url.ParseRequestURI(in) if err != nil { - return "", false + return "", err } - return safeURI.String(), true + return safeURI.String(), nil } // Support X-Forwarded-Proto if added by a reverse proxy @@ -297,11 +412,11 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok ns = parts[1] rootID = parts[2] default: - return "", false + return "", nil } if !isSubdomainNamespace(ns) { - return "", false + return "", nil } // add prefix if query is present @@ -320,25 +435,42 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok } // If rootID is a CID, ensure it uses DNS-friendly text representation - if rootCid, err := cid.Decode(rootID); err == nil { - multicodec := rootCid.Type() - - // PeerIDs represented as CIDv1 are expected to have libp2p-key - // multicodec (https://github.com/libp2p/specs/pull/209). - // We ease the transition by fixing multicodec on the fly: - // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 - if isPeerIDNamespace(ns) && multicodec != cid.Libp2pKey { - multicodec = cid.Libp2pKey + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() + var base mbase.Encoding = mbase.Base32 + + // Normalizations specific to /ipns/{libp2p-key} + if isPeerIDNamespace(ns) { + // Using Base36 for /ipns/ for consistency + // Context: https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452372828 + base = mbase.Base36 + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 + if multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } } - // if object turns out to be a valid CID, - // ensure text representation used in subdomain is CIDv1 in Base32 - // https://github.com/ipfs/in-web-browsers/issues/89 - rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32) + // Ensure CID text representation used in subdomain is compatible + // with the way DNS and URIs are implemented in user agents. + // + // 1. Switch to CIDv1 and enable case-insensitive Base encoding + // to avoid issues when user agent force-lowercases the hostname + // before making the request + // (https://github.com/ipfs/in-web-browsers/issues/89) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(base) + if err != nil { + return "", err + } + // 2. Make sure CID fits in a DNS label, adjust encoding if needed + // (https://github.com/ipfs/go-ipfs/issues/7318) + rootID, err = toDNSPrefix(rootID, rootCID) if err != nil { - // should not error, but if it does, its clealy not possible to - // produce a subdomain URL - return "", false + return "", err } } diff --git a/gateway/core/corehttp/hostname_test.go b/gateway/core/corehttp/hostname_test.go index 9a2974648..3a316ede5 100644 --- a/gateway/core/corehttp/hostname_test.go +++ b/gateway/core/corehttp/hostname_test.go @@ -1,9 +1,11 @@ package corehttp import ( + "errors" "net/http/httptest" "testing" + cid "github.com/ipfs/go-cid" config "github.com/ipfs/go-ipfs-config" ) @@ -15,23 +17,26 @@ func TestToSubdomainURL(t *testing.T) { path string // out: url string - ok bool + err error }{ // DNSLink - {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true}, + {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, // Hostname with port - {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, + {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, // CIDv0 → CIDv1base32 - {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, // PeerID as CIDv1 needs to have libp2p-key multicodec - {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, - {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, - // PeerID: ed25519+identity multihash - {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil}, + {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, + {"sub.localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil}, } { - url, ok := toSubdomainURL(test.hostname, test.path, r) - if ok != test.ok || url != test.url { - t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok) + url, err := toSubdomainURL(test.hostname, test.path, r) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.hostname, test.path, url, err, test.url, test.err) } } } @@ -75,61 +80,98 @@ func TestPortStripping(t *testing.T) { } -func TestKnownSubdomainDetails(t *testing.T) { - gwSpec := config.GatewaySpec{ - UseSubdomains: true, - } - knownGateways := map[string]config.GatewaySpec{ - "localhost": gwSpec, - "dweb.link": gwSpec, - "dweb.ipfs.pvt.k12.ma.us": gwSpec, // note the sneaky ".ipfs." ;-) +func TestDNSPrefix(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSPrefix(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } } +} + +func TestKnownSubdomainDetails(t *testing.T) { + gwLocalhost := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwDweb := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwLong := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard1 := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard2 := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + + knownGateways := prepareKnownGateways(map[string]*config.GatewaySpec{ + "localhost": gwLocalhost, + "dweb.link": gwDweb, + "dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-) + "*.wildcard1.tld": gwWildcard1, + "*.*.wildcard2.tld": gwWildcard2, + }) + for _, test := range []struct { // in: hostHeader string // out: + gw *config.GatewaySpec hostname string ns string rootID string ok bool }{ // no subdomain - {"127.0.0.1:8080", "", "", "", false}, - {"[::1]:8080", "", "", "", false}, - {"hey.look.example.com", "", "", "", false}, - {"dweb.link", "", "", "", false}, + {"127.0.0.1:8080", nil, "", "", "", false}, + {"[::1]:8080", nil, "", "", "", false}, + {"hey.look.example.com", nil, "", "", "", false}, + {"dweb.link", nil, "", "", "", false}, // malformed Host header - {".....dweb.link", "", "", "", false}, - {"link", "", "", "", false}, - {"8080:dweb.link", "", "", "", false}, - {" ", "", "", "", false}, - {"", "", "", "", false}, + {".....dweb.link", nil, "", "", "", false}, + {"link", nil, "", "", "", false}, + {"8080:dweb.link", nil, "", "", "", false}, + {" ", nil, "", "", "", false}, + {"", nil, "", "", "", false}, // unknown gateway host - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false}, // cid in subdomain, known gateway - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, // capture everything before .ipfs. - {"foo.bar.boo-buzz.ipfs.dweb.link", "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + {"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, // ipns - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // dnslink in subdomain - {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"en.wikipedia-on-ipfs.org.ipns.localhost", "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"dist.ipfs.io.ipns.localhost:8080", "localhost:8080", "ipns", "dist.ipfs.io", true}, - {"en.wikipedia-on-ipfs.org.ipns.dweb.link", "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.io.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.io", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // other namespaces - {"api.localhost", "", "", "", false}, - {"peerid.p2p.localhost", "localhost", "p2p", "peerid", true}, + {"api.localhost", nil, "", "", "", false}, + {"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true}, + // wildcards + {"wildcard1.tld", nil, "", "", "", false}, + {".wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, } { gw, hostname, ns, rootID, ok := knownSubdomainDetails(test.hostHeader, knownGateways) if ok != test.ok { @@ -144,9 +186,13 @@ func TestKnownSubdomainDetails(t *testing.T) { if hostname != test.hostname { t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) } - if ok && gw.UseSubdomains != gwSpec.UseSubdomains { - t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, gwSpec) + if gw != test.gw { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, test.gw) } } } + +func equalError(a, b error) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Error() == b.Error()) +} diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index f94df7850..b0f4256c8 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,16 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4" // v2.9.0 +const WebUIPath = "/ipfs/bafybeianwe4vy7sprht5sm3hshvxjeqhwcmvbzq73u55sdhqngmohkjgs4" // v2.11.1 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeicitin4p7ggmyjaubqpi3xwnagrwarsy6hiihraafk5rcrxqxju6m", + "/ipfs/bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e", + "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu", + "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq", + "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4", "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub",