From 20eab31a219e937e19376bef5da3146d2c760b12 Mon Sep 17 00:00:00 2001 From: Shotaro Yamada Date: Thu, 28 May 2020 10:53:25 +0900 Subject: [PATCH 01/15] chore: update WebUI to 2.8.0 This commit was moved from ipfs/kubo@58aac04a53d60d683cbe64fc26f6b0abac4e855b --- gateway/core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index 02f2da73a..22143e49a 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq" +const WebUIPath = "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i" // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub", "/ipfs/QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ", "/ipfs/QmenEBWcAk3tN94fSKpKFtUMwty1qNwSYw3DMDFV6cPBXA", From 787581221fe2519c74aee10cb9441bce3914fbb9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 28 May 2020 23:04:35 +0200 Subject: [PATCH 02/15] feat: webui v2.9.0 This commit was moved from ipfs/kubo@0f76ed79f97de7d2c886385ada0030b2e8ee42c8 --- gateway/core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index 22143e49a..f94df7850 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i" +const WebUIPath = "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4" // v2.9.0 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub", "/ipfs/QmXc9raDM1M5G5fpBnVyQ71vR4gbnskwnB9iMEzBuLgvoZ", From b2c1ae60f1dc08e44bb8a5d367c622f3594baca0 Mon Sep 17 00:00:00 2001 From: Rafael Ramalho Date: Mon, 22 Jun 2020 17:39:30 +0100 Subject: [PATCH 03/15] chore: bump webui version This commit was moved from ipfs/kubo@cc4a136360697ec7633fe19db80555de2af5e58b --- gateway/core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index f94df7850..e00cf788f 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4" // v2.9.0 +const WebUIPath = "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq" // v2.10.0 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4", "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", "/ipfs/QmfQkD8pBSBCBxWEwFSu4XaDVSWK6bjnNuaWZjMyQbyDub", From b6d36d97fbd1da2fd1b700ebda97ba11ec5f2e9f Mon Sep 17 00:00:00 2001 From: Rafael Ramalho Date: Tue, 23 Jun 2020 16:28:38 +0100 Subject: [PATCH 04/15] chore:bump webui version to 2.10.1 This commit was moved from ipfs/kubo@4c38ea748a6d7f6d5b9d4900dc77f976d049d427 --- gateway/core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index e00cf788f..2728bc7b3 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq" // v2.10.0 +const WebUIPath = "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu" // v2.10.1 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq", "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4", "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", "/ipfs/bafybeidatpz2hli6fgu3zul5woi27ujesdf5o5a7bu622qj6ugharciwjq", From 84e3c2244d29fc7da570eb0c4e3f85f4d47d3a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Wed, 1 Jul 2020 12:52:14 +0200 Subject: [PATCH 05/15] feat: support X-Forwarded-Host when doing gateway redirect This commit was moved from ipfs/kubo@87dfc46e037c06e7580dcb645f967c736a3830af --- gateway/core/corehttp/hostname.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index d8656da23..edcd8b718 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -81,8 +81,15 @@ func HostnameOption() ServeOption { // and the paths that they serve "gateway" content on. // That way, we can use DNSLink for everything else. + // Support X-Forwarded-Host if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host + host := r.Host + if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" { + host = xHost + } + // HTTP Host & Path check: is this one of our "known gateways"? - if gw, ok := isKnownHostname(r.Host, knownGateways); ok { + if gw, ok := isKnownHostname(host, knownGateways); ok { // This is a known gateway but request is not using // the subdomain feature. @@ -94,7 +101,7 @@ func HostnameOption() ServeOption { if gw.UseSubdomains { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link - if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok { + if newURL, ok := toSubdomainURL(host, r.URL.Path, r); ok { // Just to be sure single Origin can't be abused in // web browsers that ignored the redirect for some // reason, Clear-Site-Data header clears browsing @@ -124,9 +131,9 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { // rewrite path and handle as DNSLink - r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) return } @@ -138,7 +145,7 @@ func HostnameOption() ServeOption { // HTTP Host check: is this one of our subdomain-based "known gateways"? // Example: {cid}.ipfs.localhost, {cid}.ipfs.dweb.link - if gw, hostname, ns, rootID, ok := knownSubdomainDetails(r.Host, knownGateways); ok { + if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok { // Looks like we're using known subdomain gateway. // Assemble original path prefix. @@ -176,9 +183,9 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { // rewrite path and handle as DNSLink - r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) return } @@ -236,8 +243,8 @@ func knownSubdomainDetails(hostname string, knownGateways map[string]config.Gate // isDNSLinkRequest returns bool that indicates if request // should return data from content path listed in DNSLink record (if exists) -func isDNSLinkRequest(ctx context.Context, ipfs iface.CoreAPI, r *http.Request) bool { - fqdn := stripPort(r.Host) +func isDNSLinkRequest(ctx context.Context, ipfs iface.CoreAPI, host string) bool { + fqdn := stripPort(host) if len(fqdn) == 0 && !isd.IsDomain(fqdn) { return false } From c1cfe2f170de6cca072a470280510fdd9035b057 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2020 16:15:34 +0200 Subject: [PATCH 06/15] feat: support ED25519 libp2p-key in subdomains This: - adds subdomain gateway support for ED25519 CIDs in a way that fits in a single DNS label to enable TLS for every IPNS website. - cleans up subdomain redirect logic and adds more explicit error handling. TL;DR on router logic: When CID is longer than 63 characters, router at /ipfs/* and /ipns/* converts to Base36, and if that does not help, returns a human readable 400 Bad Request error. Addressing code review: https://github.com/ipfs/go-ipfs/pull/7441#pullrequestreview-440043209 refactor: use b36 for all libp2p-keys in subdomains Consensus reached in https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452372828 https://github.com/ipfs/go-ipfs/pull/7441#discussion_r451477890 https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452500272 This commit was moved from ipfs/kubo@231fab811d83322e61fe8d9c65d7bcd9fd6d243c --- gateway/core/corehttp/hostname.go | 137 +++++++++++++++++++------ gateway/core/corehttp/hostname_test.go | 54 ++++++++-- 2 files changed, 148 insertions(+), 43 deletions(-) diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index edcd8b718..f29b4d11f 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{ "dweb.link": subdomainGatewaySpec, } +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + // HostnameOption rewrites an incoming request based on the Host header. func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { childMux := http.NewServeMux() - coreApi, err := coreapi.NewCoreAPI(n) + coreAPI, err := coreapi.NewCoreAPI(n) if err != nil { return nil, err } @@ -101,7 +104,12 @@ func HostnameOption() ServeOption { if gw.UseSubdomains { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link - if newURL, ok := toSubdomainURL(host, r.URL.Path, r); ok { + newURL, err := toSubdomainURL(host, r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { // Just to be sure single Origin can't be abused in // web browsers that ignored the redirect for some // reason, Clear-Site-Data header clears browsing @@ -131,7 +139,7 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { + if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -158,16 +166,44 @@ func HostnameOption() ServeOption { return } - // Do we need to fix multicodec in PeerID represented as CIDv1? - if isPeerIDNamespace(ns) { - keyCid, err := cid.Decode(rootID) - if err == nil && keyCid.Type() != cid.Libp2pKey { - if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { - // Redirect to CID fixed inside of toSubdomainURL() + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSPrefix(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } } // Rewrite the path to not use subdomains @@ -183,7 +219,7 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, host) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -273,18 +309,38 @@ func isPeerIDNamespace(ns string) bool { } } +// Converts an identifier to DNS-safe representation that fits in 63 characters +func toDNSPrefix(rootID string, rootCID cid.Cid) (prefix string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { +func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery parts := strings.SplitN(path, "/", 4) - safeRedirectURL := func(in string) (out string, ok bool) { + safeRedirectURL := func(in string) (out string, err error) { safeURI, err := url.ParseRequestURI(in) if err != nil { - return "", false + return "", err } - return safeURI.String(), true + return safeURI.String(), nil } // Support X-Forwarded-Proto if added by a reverse proxy @@ -304,11 +360,11 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok ns = parts[1] rootID = parts[2] default: - return "", false + return "", nil } if !isSubdomainNamespace(ns) { - return "", false + return "", nil } // add prefix if query is present @@ -327,25 +383,42 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok } // If rootID is a CID, ensure it uses DNS-friendly text representation - if rootCid, err := cid.Decode(rootID); err == nil { - multicodec := rootCid.Type() - - // PeerIDs represented as CIDv1 are expected to have libp2p-key - // multicodec (https://github.com/libp2p/specs/pull/209). - // We ease the transition by fixing multicodec on the fly: - // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 - if isPeerIDNamespace(ns) && multicodec != cid.Libp2pKey { - multicodec = cid.Libp2pKey + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() + var base mbase.Encoding = mbase.Base32 + + // Normalizations specific to /ipns/{libp2p-key} + if isPeerIDNamespace(ns) { + // Using Base36 for /ipns/ for consistency + // Context: https://github.com/ipfs/go-ipfs/pull/7441#discussion_r452372828 + base = mbase.Base36 + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/go-ipfs/issues/5287#issuecomment-492163929 + if multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } } - // if object turns out to be a valid CID, - // ensure text representation used in subdomain is CIDv1 in Base32 - // https://github.com/ipfs/in-web-browsers/issues/89 - rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32) + // Ensure CID text representation used in subdomain is compatible + // with the way DNS and URIs are implemented in user agents. + // + // 1. Switch to CIDv1 and enable case-insensitive Base encoding + // to avoid issues when user agent force-lowercases the hostname + // before making the request + // (https://github.com/ipfs/in-web-browsers/issues/89) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(base) + if err != nil { + return "", err + } + // 2. Make sure CID fits in a DNS label, adjust encoding if needed + // (https://github.com/ipfs/go-ipfs/issues/7318) + rootID, err = toDNSPrefix(rootID, rootCID) if err != nil { - // should not error, but if it does, its clealy not possible to - // produce a subdomain URL - return "", false + return "", err } } diff --git a/gateway/core/corehttp/hostname_test.go b/gateway/core/corehttp/hostname_test.go index 9a2974648..cf00e8271 100644 --- a/gateway/core/corehttp/hostname_test.go +++ b/gateway/core/corehttp/hostname_test.go @@ -1,9 +1,11 @@ package corehttp import ( + "errors" "net/http/httptest" "testing" + cid "github.com/ipfs/go-cid" config "github.com/ipfs/go-ipfs-config" ) @@ -15,23 +17,25 @@ func TestToSubdomainURL(t *testing.T) { path string // out: url string - ok bool + err error }{ // DNSLink - {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true}, + {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, // Hostname with port - {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, + {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, // CIDv0 → CIDv1base32 - {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, // PeerID as CIDv1 needs to have libp2p-key multicodec - {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, - {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, - // PeerID: ed25519+identity multihash - {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil}, + {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, } { - url, ok := toSubdomainURL(test.hostname, test.path, r) - if ok != test.ok || url != test.url { - t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok) + url, err := toSubdomainURL(test.hostname, test.path, r) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.hostname, test.path, url, err, test.url, test.err) } } } @@ -75,6 +79,30 @@ func TestPortStripping(t *testing.T) { } +func TestDNSPrefix(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSPrefix(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } + +} + func TestKnownSubdomainDetails(t *testing.T) { gwSpec := config.GatewaySpec{ UseSubdomains: true, @@ -150,3 +178,7 @@ func TestKnownSubdomainDetails(t *testing.T) { } } + +func equalError(a, b error) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Error() == b.Error()) +} From 3937f1f675e9d0e206a6110cc7967819c6217e49 Mon Sep 17 00:00:00 2001 From: Stephen Solka Date: Sat, 18 Jul 2020 14:47:07 -0400 Subject: [PATCH 07/15] use t.Cleanup() to reduce the need to clean up servers This commit was moved from ipfs/kubo@4dbdbe0e029d5af826919376579268da0bc68019 --- gateway/core/corehttp/gateway_test.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gateway/core/corehttp/gateway_test.go b/gateway/core/corehttp/gateway_test.go index a8a07aa48..d8b7eaa5f 100644 --- a/gateway/core/corehttp/gateway_test.go +++ b/gateway/core/corehttp/gateway_test.go @@ -135,6 +135,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface // listener, and server with handler. yay cycles. dh := &delegatedHandler{} ts := httptest.NewServer(dh) + t.Cleanup(func() { ts.Close() }) dh.Handler, err = makeHandler(n, ts.Listener, @@ -157,7 +158,6 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface func TestGatewayGet(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) - defer ts.Close() k, err := api.Unixfs().Add(ctx, files.NewBytesFile([]byte("fnord"))) if err != nil { @@ -238,7 +238,6 @@ func TestGatewayGet(t *testing.T) { func TestPretty404(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) - defer ts.Close() f1 := files.NewMapDirectory(map[string]files.Node{ "ipfs-404.html": files.NewBytesFile([]byte("Custom 404")), @@ -303,7 +302,6 @@ func TestIPNSHostnameRedirect(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() // create /ipns/example.net/foo/index.html @@ -391,7 +389,6 @@ func TestIPNSHostnameBacklinks(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() f1 := files.NewMapDirectory(map[string]files.Node{ "file.txt": files.NewBytesFile([]byte("1")), @@ -601,7 +598,6 @@ func TestIPNSHostnameBacklinks(t *testing.T) { func TestCacheControlImmutable(t *testing.T) { ts, _, _ := newTestServerAndNode(t, nil) t.Logf("test server url: %s", ts.URL) - defer ts.Close() req, err := http.NewRequest(http.MethodGet, ts.URL+emptyDir+"/", nil) if err != nil { @@ -627,7 +623,6 @@ func TestCacheControlImmutable(t *testing.T) { func TestGoGetSupport(t *testing.T) { ts, _, _ := newTestServerAndNode(t, nil) t.Logf("test server url: %s", ts.URL) - defer ts.Close() // mimic go-get req, err := http.NewRequest(http.MethodGet, ts.URL+emptyDir+"?go-get=1", nil) @@ -651,7 +646,6 @@ func TestVersion(t *testing.T) { ns := mockNamesys{} ts, _, _ := newTestServerAndNode(t, ns) t.Logf("test server url: %s", ts.URL) - defer ts.Close() req, err := http.NewRequest(http.MethodGet, ts.URL+"/version", nil) if err != nil { From 4030f32451874701ba73058d15bef991a0070cbe Mon Sep 17 00:00:00 2001 From: Rafael Ramalho Date: Thu, 16 Jul 2020 19:24:37 +0100 Subject: [PATCH 08/15] chore: bump webui version This commit was moved from ipfs/kubo@e3905e1fdc7551980dc3874e71f0b43e1831deff --- gateway/core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index 2728bc7b3..8eff65c2d 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu" // v2.10.1 +const WebUIPath = "/ipfs/bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e" // v2.10.2 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu", "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq", "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4", "/ipfs/bafybeicp23nbcxtt2k2twyfivcbrc6kr3l5lnaiv3ozvwbemtrb7v52r6i", From 542242c6a3fa5b17d01f9c2439f611e2ee776fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Thu, 14 May 2020 20:35:11 +0200 Subject: [PATCH 09/15] feat: wildcard support for public gateways Add support for one or more wildcards in the hostname definition of a public gateway. This is useful for example to support easily multiples environment. Wildcarded hostname are set in the config as for example "*.domain.tld". This commit was moved from ipfs/kubo@13e6bcfb4f7d2db7d80af77f1b32b2394abca9da --- gateway/core/corehttp/hostname.go | 98 +++++++++++++++++++------- gateway/core/corehttp/hostname_test.go | 86 ++++++++++++---------- 2 files changed, 124 insertions(+), 60 deletions(-) diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index f29b4d11f..04f640e2f 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strings" cid "github.com/ipfs/go-cid" @@ -24,17 +25,17 @@ import ( var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/version"} -var pathGatewaySpec = config.GatewaySpec{ +var pathGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: false, } -var subdomainGatewaySpec = config.GatewaySpec{ +var subdomainGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: true, } -var defaultKnownGateways = map[string]config.GatewaySpec{ +var defaultKnownGateways = map[string]*config.GatewaySpec{ "localhost": subdomainGatewaySpec, "ipfs.io": pathGatewaySpec, "gateway.ipfs.io": pathGatewaySpec, @@ -58,22 +59,8 @@ func HostnameOption() ServeOption { if err != nil { return nil, err } - knownGateways := make( - map[string]config.GatewaySpec, - len(defaultKnownGateways)+len(cfg.Gateway.PublicGateways), - ) - for hostname, gw := range defaultKnownGateways { - knownGateways[hostname] = gw - } - for hostname, gw := range cfg.Gateway.PublicGateways { - if gw == nil { - // Allows the user to remove gateways but _also_ - // allows us to continuously update the list. - delete(knownGateways, hostname) - } else { - knownGateways[hostname] = *gw - } - } + + knownGateways := prepareKnownGateways(cfg.Gateway.PublicGateways) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // Unfortunately, many (well, ipfs.io) gateways use @@ -233,22 +220,85 @@ func HostnameOption() ServeOption { } } +type gatewayHosts struct { + exact map[string]*config.GatewaySpec + wildcard []wildcardHost +} + +type wildcardHost struct { + re *regexp.Regexp + spec *config.GatewaySpec +} + +func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts { + var hosts gatewayHosts + + if len(publicGateways) == 0 { + hosts.exact = make( + map[string]*config.GatewaySpec, + len(defaultKnownGateways), + ) + for hostname, gw := range defaultKnownGateways { + hosts.exact[hostname] = gw + } + return hosts + } + + hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways)) + + for hostname, gw := range publicGateways { + if gw == nil { + continue + } + if strings.Contains(hostname, "*") { + // from *.domain.tld, construct a regexp that match any direct subdomain + // of .domain.tld. + // + // Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$ + + escaped := strings.ReplaceAll(hostname, ".", `\.`) + regexed := strings.ReplaceAll(escaped, "*", "[^.]+") + + re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed)) + if err != nil { + log.Warn("invalid wildcard gateway hostname \"%s\"", hostname) + } + + hosts.wildcard = append(hosts.wildcard, wildcardHost{re: re, spec: gw}) + } else { + hosts.exact[hostname] = gw + } + } + + return hosts +} + // isKnownHostname checks Gateway.PublicGateways and returns matching // GatewaySpec with gracefull fallback to version without port -func isKnownHostname(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, ok bool) { +func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, ok bool) { // Try hostname (host+optional port - value from Host header as-is) - if gw, ok := knownGateways[hostname]; ok { + if gw, ok := knownGateways.exact[hostname]; ok { + return gw, ok + } + // Also test without port + if gw, ok = knownGateways.exact[stripPort(hostname)]; ok { return gw, ok } - // Fallback to hostname without port - gw, ok = knownGateways[stripPort(hostname)] + + // Wildcard support. Test both with and without port. + for _, host := range knownGateways.wildcard { + if host.re.MatchString(hostname) { + return host.spec, true + } + } + return gw, ok } // Parses Host header and looks for a known subdomain gateway host. // If found, returns GatewaySpec and subdomain components. // Note: hostname is host + optional port -func knownSubdomainDetails(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { +func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { labels := strings.Split(hostname, ".") // Look for FQDN of a known gateway hostname. // Example: given "dist.ipfs.io.ipns.dweb.link": diff --git a/gateway/core/corehttp/hostname_test.go b/gateway/core/corehttp/hostname_test.go index cf00e8271..472dbcf16 100644 --- a/gateway/core/corehttp/hostname_test.go +++ b/gateway/core/corehttp/hostname_test.go @@ -32,6 +32,7 @@ func TestToSubdomainURL(t *testing.T) { {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, // PeerID: ed25519+identity multihash → CIDv1Base36 {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, + {"sub.localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil}, } { url, err := toSubdomainURL(test.hostname, test.path, r) if url != test.url || !equalError(err, test.err) { @@ -104,60 +105,73 @@ func TestDNSPrefix(t *testing.T) { } func TestKnownSubdomainDetails(t *testing.T) { - gwSpec := config.GatewaySpec{ - UseSubdomains: true, - } - knownGateways := map[string]config.GatewaySpec{ - "localhost": gwSpec, - "dweb.link": gwSpec, - "dweb.ipfs.pvt.k12.ma.us": gwSpec, // note the sneaky ".ipfs." ;-) - } + gwLocalhost := &config.GatewaySpec{} + gwDweb := &config.GatewaySpec{} + gwLong := &config.GatewaySpec{} + gwWildcard1 := &config.GatewaySpec{} + gwWildcard2 := &config.GatewaySpec{} + + knownGateways := prepareKnownGateways(map[string]*config.GatewaySpec{ + "localhost": gwLocalhost, + "dweb.link": gwDweb, + "dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-) + "*.wildcard1.tld": gwWildcard1, + "*.*.wildcard2.tld": gwWildcard2, + }) for _, test := range []struct { // in: hostHeader string // out: + gw *config.GatewaySpec hostname string ns string rootID string ok bool }{ // no subdomain - {"127.0.0.1:8080", "", "", "", false}, - {"[::1]:8080", "", "", "", false}, - {"hey.look.example.com", "", "", "", false}, - {"dweb.link", "", "", "", false}, + {"127.0.0.1:8080", nil, "", "", "", false}, + {"[::1]:8080", nil, "", "", "", false}, + {"hey.look.example.com", nil, "", "", "", false}, + {"dweb.link", nil, "", "", "", false}, // malformed Host header - {".....dweb.link", "", "", "", false}, - {"link", "", "", "", false}, - {"8080:dweb.link", "", "", "", false}, - {" ", "", "", "", false}, - {"", "", "", "", false}, + {".....dweb.link", nil, "", "", "", false}, + {"link", nil, "", "", "", false}, + {"8080:dweb.link", nil, "", "", "", false}, + {" ", nil, "", "", "", false}, + {"", nil, "", "", "", false}, // unknown gateway host - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false}, // cid in subdomain, known gateway - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, // capture everything before .ipfs. - {"foo.bar.boo-buzz.ipfs.dweb.link", "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + {"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, // ipns - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // dnslink in subdomain - {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"en.wikipedia-on-ipfs.org.ipns.localhost", "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"dist.ipfs.io.ipns.localhost:8080", "localhost:8080", "ipns", "dist.ipfs.io", true}, - {"en.wikipedia-on-ipfs.org.ipns.dweb.link", "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.io.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.io", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // other namespaces - {"api.localhost", "", "", "", false}, - {"peerid.p2p.localhost", "localhost", "p2p", "peerid", true}, + {"api.localhost", nil, "", "", "", false}, + {"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true}, + // wildcards + {"wildcard1.tld", nil, "", "", "", false}, + {".wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, } { gw, hostname, ns, rootID, ok := knownSubdomainDetails(test.hostHeader, knownGateways) if ok != test.ok { @@ -172,8 +186,8 @@ func TestKnownSubdomainDetails(t *testing.T) { if hostname != test.hostname { t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) } - if ok && gw.UseSubdomains != gwSpec.UseSubdomains { - t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, gwSpec) + if gw != test.gw { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, test.gw) } } From c0b7f3f0710d4e5a4348e0b8c51e2cc44f45fe0e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 6 Aug 2020 14:00:16 +0200 Subject: [PATCH 10/15] test(gateway): IPNS cleanup and implicit defaults fix This ensures implicit defaults are always present, even when Gateway.PublicGateways is defined in the config. User still can disable them, but needs to do it per hostname. License: MIT Signed-off-by: Marcin Rataj This commit was moved from ipfs/kubo@2ff6f1a80d6a070465e8c157a750ecf17687642b --- gateway/core/corehttp/hostname.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index 04f640e2f..e94be2c82 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -233,21 +233,19 @@ type wildcardHost struct { func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts { var hosts gatewayHosts - if len(publicGateways) == 0 { - hosts.exact = make( - map[string]*config.GatewaySpec, - len(defaultKnownGateways), - ) - for hostname, gw := range defaultKnownGateways { - hosts.exact[hostname] = gw - } - return hosts - } + hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways)+len(defaultKnownGateways)) - hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways)) + // First, implicit defaults such as subdomain gateway on localhost + for hostname, gw := range defaultKnownGateways { + hosts.exact[hostname] = gw + } + // Then apply values from Gateway.PublicGateways, if present in the config for hostname, gw := range publicGateways { if gw == nil { + // Remove any implicit defaults, if present. This is useful when one + // wants to disable subdomain gateway on localhost etc. + delete(hosts.exact, hostname) continue } if strings.Contains(hostname, "*") { From c37f055a823ab57295f2a7d3084ee8ab5436abcc Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 19 Aug 2020 15:59:47 +0200 Subject: [PATCH 11/15] refactor: cleanup/comment https://github.com/ipfs/go-ipfs/pull/7319#discussion_r472734905 License: MIT Signed-off-by: Marcin Rataj This commit was moved from ipfs/kubo@6b6569f3e5c1cb2e229b98c26c88ffa34f0abe81 --- gateway/core/corehttp/hostname.go | 12 ++++++------ gateway/core/corehttp/hostname_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index e94be2c82..880ec8787 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -141,12 +141,12 @@ func HostnameOption() ServeOption { // HTTP Host check: is this one of our subdomain-based "known gateways"? // Example: {cid}.ipfs.localhost, {cid}.ipfs.dweb.link if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok { - // Looks like we're using known subdomain gateway. + // Looks like we're using a known gateway in subdomain mode. // Assemble original path prefix. pathPrefix := "/" + ns + "/" + rootID - // Does this gateway _handle_ this path? + // Does this gateway _handle_ subdomains AND this path? if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) { // If not, resource does not exist, return 404 http.NotFound(w, r) @@ -290,10 +290,10 @@ func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.Ga } } - return gw, ok + return nil, false } -// Parses Host header and looks for a known subdomain gateway host. +// Parses Host header and looks for a known gateway matching subdomain host. // If found, returns GatewaySpec and subdomain components. // Note: hostname is host + optional port func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { @@ -321,8 +321,8 @@ func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *con rootID := strings.Join(labels[:i-1], ".") return gw, fqdn, ns, rootID, true } - // not a known subdomain gateway - return gw, "", "", "", false + // no match + return nil, "", "", "", false } // isDNSLinkRequest returns bool that indicates if request diff --git a/gateway/core/corehttp/hostname_test.go b/gateway/core/corehttp/hostname_test.go index 472dbcf16..3a316ede5 100644 --- a/gateway/core/corehttp/hostname_test.go +++ b/gateway/core/corehttp/hostname_test.go @@ -105,11 +105,11 @@ func TestDNSPrefix(t *testing.T) { } func TestKnownSubdomainDetails(t *testing.T) { - gwLocalhost := &config.GatewaySpec{} - gwDweb := &config.GatewaySpec{} - gwLong := &config.GatewaySpec{} - gwWildcard1 := &config.GatewaySpec{} - gwWildcard2 := &config.GatewaySpec{} + gwLocalhost := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwDweb := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwLong := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard1 := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard2 := &config.GatewaySpec{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} knownGateways := prepareKnownGateways(map[string]*config.GatewaySpec{ "localhost": gwLocalhost, From 529c1727820853ea5461ca3c291d44f6b87b33e7 Mon Sep 17 00:00:00 2001 From: Kevin Neaton Date: Thu, 9 Jul 2020 23:46:05 -0400 Subject: [PATCH 12/15] feat: Directory page UI improvements These changes are needed to prepare for the Directory page UI improvements implemented in https://github.com/ipfs/dir-index-html/issues/37. - update dir-index-html type structs - emit gateway URL for root links - emit CID of each directoryItem - emit size of directory - emit breadcrumbs This commit was moved from ipfs/kubo@044790a83857c4b396ce5c336045d6b10f5e4075 --- gateway/core/corehttp/gateway_handler.go | 42 ++++++++++++++++--- gateway/core/corehttp/gateway_indexPage.go | 49 +++++++++++++++++++--- gateway/core/corehttp/hostname.go | 6 ++- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/gateway/core/corehttp/gateway_handler.go b/gateway/core/corehttp/gateway_handler.go index a1549efdd..b9e7f144b 100644 --- a/gateway/core/corehttp/gateway_handler.go +++ b/gateway/core/corehttp/gateway_handler.go @@ -328,8 +328,20 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request size = humanize.Bytes(uint64(s)) } + hash := "" + if r, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name())); err == nil { + // Path may not be resolved. Continue anyways. + hash = r.Cid().String() + } + // See comment above where originalUrlPath is declared. - di := directoryItem{size, dirit.Name(), gopath.Join(originalUrlPath, dirit.Name())} + di := directoryItem{ + Size: size, + Name: dirit.Name(), + Path: gopath.Join(originalUrlPath, dirit.Name()), + Hash: hash, + ShortHash: shortHash(hash), + } dirListing = append(dirListing, di) } if dirit.Err() != nil { @@ -359,14 +371,34 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } } + size := "?" + if s, err := dir.Size(); err == nil { + // Size may not be defined/supported. Continue anyways. + size = humanize.Bytes(uint64(s)) + } + hash := resolvedPath.Cid().String() + // Storage for gateway URL to be used when linking to other rootIDs. This + // will be blank unless subdomain resolution is being used for this request. + var gwURL string + + // Get gateway hostname and build gateway URL. + if h, ok := r.Context().Value("gw-hostname").(string); ok { + gwURL = "//" + h + } else { + gwURL = "" + } + // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ - Listing: dirListing, - Path: urlPath, - BackLink: backLink, - Hash: hash, + GatewayURL: gwURL, + Listing: dirListing, + Size: size, + Path: urlPath, + Breadcrumbs: breadcrumbs(urlPath), + BackLink: backLink, + Hash: hash, } err = listingTemplate.Execute(w, tplData) diff --git a/gateway/core/corehttp/gateway_indexPage.go b/gateway/core/corehttp/gateway_indexPage.go index 5575baea4..c9a948708 100644 --- a/gateway/core/corehttp/gateway_indexPage.go +++ b/gateway/core/corehttp/gateway_indexPage.go @@ -7,22 +7,61 @@ import ( "strings" "github.com/ipfs/go-ipfs/assets" + ipfspath "github.com/ipfs/go-path" ) // structs for directory listing type listingTemplateData struct { - Listing []directoryItem - Path string - BackLink string - Hash string + GatewayURL string + Listing []directoryItem + Size string + Path string + Breadcrumbs []breadcrumb + BackLink string + Hash string } type directoryItem struct { - Size string + Size string + Name string + Path string + Hash string + ShortHash string +} + +type breadcrumb struct { Name string Path string } +func breadcrumbs(urlPath string) []breadcrumb { + var ret []breadcrumb + + p, err := ipfspath.ParsePath(urlPath) + if err != nil { + // No breadcrumbs, fallback to bare Path in template + return ret + } + + segs := p.Segments() + for i, seg := range segs { + if i == 0 { + ret = append(ret, breadcrumb{Name: seg}) + } else { + ret = append(ret, breadcrumb{ + Name: seg, + Path: "/" + strings.Join(segs[0:i+1], "/"), + }) + } + } + + return ret +} + +func shortHash(hash string) string { + return (hash[0:4] + "\u2026" + hash[len(hash)-4:]) +} + var listingTemplate *template.Template func init() { diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index 880ec8787..8b2666afb 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -143,6 +143,10 @@ func HostnameOption() ServeOption { if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok { // Looks like we're using a known gateway in subdomain mode. + // Add gateway hostname context for linking to other root ids. + // Example: localhost/ipfs/{cid} + ctx := context.WithValue(r.Context(), "gw-hostname", hostname) + // Assemble original path prefix. pathPrefix := "/" + ns + "/" + rootID @@ -197,7 +201,7 @@ func HostnameOption() ServeOption { r.URL.Path = pathPrefix + r.URL.Path // Serve path request - childMux.ServeHTTP(w, r) + childMux.ServeHTTP(w, r.WithContext(ctx)) return } // We don't have a known gateway. Fallback on DNSLink lookup From 000ac7f97bf5832c04448f24f725500a904c327d Mon Sep 17 00:00:00 2001 From: Kevin Neaton Date: Sat, 25 Jul 2020 23:13:33 -0400 Subject: [PATCH 13/15] test: update gateway tests for dir-index-html v1.1.0 This commit was moved from ipfs/kubo@2feff332354b376a211c956902e0ad64de61fa0c --- gateway/core/corehttp/gateway_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/gateway/core/corehttp/gateway_test.go b/gateway/core/corehttp/gateway_test.go index a8a07aa48..68991bc76 100644 --- a/gateway/core/corehttp/gateway_test.go +++ b/gateway/core/corehttp/gateway_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "regexp" "strings" "testing" "time" @@ -154,6 +155,11 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface return ts, api, n.Context() } +func matchPathOrBreadcrumbs(s string, expected string) bool { + matched, _ := regexp.MatchString("Index of\n[\t ]*"+regexp.QuoteMeta(expected), s) + return matched +} + func TestGatewayGet(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) @@ -442,7 +448,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s := string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net/foo? #<'/") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -475,7 +481,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /") { + if !matchPathOrBreadcrumbs(s, "/") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -508,7 +514,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net/foo? #<'/bar/") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'/bar") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -542,7 +548,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /ipns/example.net") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -584,7 +590,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !strings.Contains(s, "Index of /") { + if !matchPathOrBreadcrumbs(s, "/") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { From 030025e7a7bf06dc8da0acefb92466785d54b8c4 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 19 Aug 2020 23:52:53 -0400 Subject: [PATCH 14/15] chore: update go-multiaddr and go-multiaddr-net This commit was moved from ipfs/kubo@b88bdfeb9db0677213f57466b8b617fa3284b107 --- gateway/core/corehttp/corehttp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/core/corehttp/corehttp.go b/gateway/core/corehttp/corehttp.go index 1b0a79ee6..143327149 100644 --- a/gateway/core/corehttp/corehttp.go +++ b/gateway/core/corehttp/corehttp.go @@ -16,7 +16,7 @@ import ( "github.com/jbenet/goprocess" periodicproc "github.com/jbenet/goprocess/periodic" ma "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("core/server") From f6b3ba239ce5d7380d4e4c4c80038ed3a6b8946e Mon Sep 17 00:00:00 2001 From: Rafael Ramalho Date: Thu, 10 Sep 2020 19:47:44 +0100 Subject: [PATCH 15/15] chore: bump webui version (cherry picked from commit 17ef979297081fec20b5b3231b1d5071ac4e8726) This commit was moved from ipfs/kubo@54e743779cf279c0328c6b4b523c888382a71de2 --- gateway/core/corehttp/webui.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gateway/core/corehttp/webui.go b/gateway/core/corehttp/webui.go index 8eff65c2d..b0f4256c8 100644 --- a/gateway/core/corehttp/webui.go +++ b/gateway/core/corehttp/webui.go @@ -1,11 +1,13 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e" // v2.10.2 +const WebUIPath = "/ipfs/bafybeianwe4vy7sprht5sm3hshvxjeqhwcmvbzq73u55sdhqngmohkjgs4" // v2.11.1 // this is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeicitin4p7ggmyjaubqpi3xwnagrwarsy6hiihraafk5rcrxqxju6m", + "/ipfs/bafybeihpetclqvwb4qnmumvcn7nh4pxrtugrlpw4jgjpqicdxsv7opdm6e", "/ipfs/bafybeibnnxd4etu4tq5fuhu3z5p4rfu3buabfkeyr3o3s4h6wtesvvw6mu", "/ipfs/bafybeid6luolenf4fcsuaw5rgdwpqbyerce4x3mi3hxfdtp5pwco7h7qyq", "/ipfs/bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4",