Merge branch 'embedded_cache' into merklizer_marshaler

.github/workflows/ci-lint.yaml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21.0
+          go-version: 1.21.2
       - uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
+          version: v1.54.2

.github/workflows/ci-test.yaml

@@ -13,9 +13,9 @@ jobs:
       matrix:
         containers:
         - 1.18.10-bullseye
-        - 1.19.12-bullseye
-        - 1.20.7-bullseye
-        - 1.21.0-bullseye
+        - 1.19.13-bullseye
+        - 1.20.9-bullseye
+        - 1.21.2-bullseye
     runs-on: ubuntu-latest
     container: golang:${{ matrix.containers }}
     steps:

go.mod

@@ -13,6 +13,7 @@ require (
 	// https://github.com/piprate/json-gold/commit/36fcca9d7e487684a764e552e7d837a14546a157
 	github.com/piprate/json-gold v0.5.1-0.20230111113000-6ddbe6e6f19f
 	github.com/pkg/errors v0.9.1
+	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.0
 	github.com/stretchr/testify v1.8.2
 	golang.org/x/crypto v0.7.0
@@ -42,7 +43,6 @@ require (
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
 	golang.org/x/sys v0.8.0 // indirect

json/parser_test.go

@@ -137,6 +137,14 @@ func TestParser_ParseClaimWithDataSlots(t *testing.T) {
 }
 
 func TestParser_ParseClaimWithMerklizedRoot(t *testing.T) {
+	defer tst.MockHTTPClient(t,
+		map[string]string{
+			"https://www.w3.org/2018/credentials/v1": "../merklize/testdata/httpresp/credentials-v1.jsonld",
+			"https://raw.githubusercontent.com/iden3/claim-schema-vocab/main/schemas/json-ld/iden3credential-v2.json-ld": "../merklize/testdata/httpresp/iden3credential-v2.json-ld",
+			"https://raw.githubusercontent.com/iden3/claim-schema-vocab/main/schemas/json-ld/kyc-v3.json-ld":             "../merklize/testdata/httpresp/kyc-v3.json-ld",
+		},
+		tst.IgnoreUntouchedURLs())()
+
 	credentialBytes, err := os.ReadFile("testdata/credential-merklized.json")
 	require.NoError(t, err)
 

loaders/document_loader.go

@@ -2,27 +2,81 @@ package loaders
 
 import (
 	"errors"
+	"fmt"
 	"io"
+	"net/http"
+	"regexp"
 	"strings"
+	"time"
 
 	shell "github.com/ipfs/go-ipfs-api"
 	"github.com/piprate/json-gold/ld"
+	"github.com/pquerna/cachecontrol"
 )
 
+const (
+	// An HTTP Accept header that prefers JSONLD.
+	acceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1"
+
+	// JSON-LD link header rel
+	linkHeaderRel = "http://www.w3.org/ns/json-ld#context"
+)
+
+var rApplicationJSON = regexp.MustCompile(`^application/(\w*\+)?json$`)
+
+var ErrCacheMiss = errors.New("cache miss")
+
+type CacheEngine interface {
+	Get(key string) (doc *ld.RemoteDocument, expireTime time.Time, err error)
+	Set(key string, doc *ld.RemoteDocument, expireTime time.Time) error
+}
+
 type documentLoader struct {
-	httpLoader *ld.RFC7324CachingDocumentLoader
-	ipfsCli    *shell.Shell
-	ipfsGW     string
+	ipfsCli     *shell.Shell
+	ipfsGW      string
+	cacheEngine CacheEngine
+	noCache     bool
+	httpClient  *http.Client
+}
+
+type DocumentLoaderOption func(*documentLoader)
+
+func WithCacheEngine(cacheEngine CacheEngine) DocumentLoaderOption {
+	return func(loader *documentLoader) {
+		if cacheEngine == nil {
+			loader.noCache = true
+			return
+		}
+
+		loader.cacheEngine = cacheEngine
+	}
+}
+
+func WithHTTPClient(httpClient *http.Client) DocumentLoaderOption {
+	return func(loader *documentLoader) {
+		loader.httpClient = httpClient
+	}
 }
 
 // NewDocumentLoader creates a new document loader with a cache for http.
 // ipfs cache is not implemented yet.
-func NewDocumentLoader(ipfsCli *shell.Shell, ipfsGW string) ld.DocumentLoader {
-	return &documentLoader{
-		httpLoader: ld.NewRFC7324CachingDocumentLoader(nil),
-		ipfsCli:    ipfsCli,
-		ipfsGW:     ipfsGW,
+func NewDocumentLoader(ipfsCli *shell.Shell, ipfsGW string,
+	opts ...DocumentLoaderOption) ld.DocumentLoader {
+	loader := &documentLoader{
+		ipfsCli: ipfsCli,
+		ipfsGW:  ipfsGW,
 	}
+
+	for _, opt := range opts {
+		opt(loader)
+	}
+
+	if loader.cacheEngine == nil && !loader.noCache {
+		// Should not be errors if we call NewMemoryCacheEngine without options
+		loader.cacheEngine, _ = NewMemoryCacheEngine()
+	}
+
+	return loader
 }
 
 func (d *documentLoader) LoadDocument(
@@ -32,7 +86,7 @@ func (d *documentLoader) LoadDocument(
 
 	switch {
 	case strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://"):
-		return d.httpLoader.LoadDocument(u)
+		return d.loadDocumentFromHTTP(u)
 
 	case strings.HasPrefix(u, ipfsPrefix):
 		// supported URLs:
@@ -92,9 +146,127 @@ func (d *documentLoader) loadDocumentFromIPFSGW(
 
 	ipfsURL = strings.TrimRight(d.ipfsGW, "/") + "/ipfs/" +
 		strings.TrimLeft(ipfsURL, "/")
-	doc, err := d.httpLoader.LoadDocument(ipfsURL)
+	doc, err := d.loadDocumentFromHTTP(ipfsURL)
 	if err != nil {
 		return nil, err
 	}
 	return doc.Document, nil
 }
+
+func (d *documentLoader) loadDocumentFromHTTP(
+	u string) (*ld.RemoteDocument, error) {
+
+	var doc *ld.RemoteDocument
+	var cacheFound bool
+	var err error
+
+	// We use shouldCache, and expireTime at the end of this method to create
+	// an object to store in the cache. Set them to sane default values now
+	shouldCache := false
+	var expireTime time.Time
+
+	if d.cacheEngine != nil {
+		doc, expireTime, err = d.cacheEngine.Get(u)
+		switch {
+		case errors.Is(err, ErrCacheMiss):
+			cacheFound = false
+		case err != nil:
+			return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+		default:
+			cacheFound = true
+		}
+	}
+
+	now := time.Now()
+
+	// First we check if we hit in the cache, and the cache entry is valid
+	// We need to check if ExpireTime >= now, so we negate the comparison below
+	if cacheFound && expireTime.After(now) {
+		return doc, nil
+	}
+
+	req, err := http.NewRequest("GET", u, http.NoBody)
+	if err != nil {
+		return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+	}
+	// We prefer application/ld+json, but fallback to application/json
+	// or whatever is available
+	req.Header.Add("Accept", acceptHeader)
+
+	httpClient := d.httpClient
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+
+	res, err := httpClient.Do(req)
+	if err != nil {
+		return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+	}
+	defer func() { _ = res.Body.Close() }()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed,
+			fmt.Sprintf("Bad response status code: %d",
+				res.StatusCode))
+	}
+
+	doc = &ld.RemoteDocument{DocumentURL: res.Request.URL.String()}
+
+	contentType := res.Header.Get("Content-Type")
+	linkHeader := res.Header.Get("Link")
+
+	if len(linkHeader) > 0 {
+		parsedLinkHeader := ld.ParseLinkHeader(linkHeader)
+		contextLink := parsedLinkHeader[linkHeaderRel]
+		if contextLink != nil && contentType != ld.ApplicationJSONLDType {
+			if len(contextLink) > 1 {
+				return nil, ld.NewJsonLdError(ld.MultipleContextLinkHeaders,
+					nil)
+			} else if len(contextLink) == 1 {
+				doc.ContextURL = contextLink[0]["target"]
+			}
+		}
+
+		// If content-type is not application/ld+json, nor any other +json
+		// and a link with rel=alternate and type='application/ld+json' is found,
+		// use that instead
+		alternateLink := parsedLinkHeader["alternate"]
+		if len(alternateLink) > 0 &&
+			alternateLink[0]["type"] == ld.ApplicationJSONLDType &&
+			!rApplicationJSON.MatchString(contentType) {
+
+			finalURL := ld.Resolve(u, alternateLink[0]["target"])
+			doc, err = d.LoadDocument(finalURL)
+			if err != nil {
+				return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+			}
+		}
+	}
+
+	reasons, resExpireTime, err := cachecontrol.CachableResponse(req, res,
+		cachecontrol.Options{})
+	// If there are no errors parsing cache headers and there are no
+	// reasons not to cache, then we cache
+	if err == nil && len(reasons) == 0 {
+		shouldCache = true
+		expireTime = resExpireTime
+	}
+
+	if doc.Document == nil {
+		doc.Document, err = ld.DocumentFromReader(res.Body)
+		if err != nil {
+			return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+		}
+	}
+
+	// If we went down a branch that marked shouldCache true then lets add the
+	// cache entry into the cache
+	if shouldCache && d.cacheEngine != nil {
+		err = d.cacheEngine.Set(u, doc, expireTime)
+		if err != nil {
+			return nil, ld.NewJsonLdError(ld.LoadingDocumentFailed, err)
+		}
+	}
+
+	return doc, nil
+}

loaders/memory_cache.go

@@ -0,0 +1,99 @@
+package loaders
+
+import (
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/piprate/json-gold/ld"
+)
+
+type cachedRemoteDocument struct {
+	remoteDocument *ld.RemoteDocument
+	expireTime     time.Time
+}
+
+type memoryCacheEngine struct {
+	m         sync.RWMutex
+	cache     map[string]*cachedRemoteDocument
+	embedDocs map[string]*ld.RemoteDocument
+}
+
+func (m *memoryCacheEngine) Get(
+	key string) (*ld.RemoteDocument, time.Time, error) {
+
+	if m.embedDocs != nil {
+		doc, ok := m.embedDocs[key]
+		if ok {
+			return doc, time.Now().Add(time.Hour), nil
+		}
+	}
+
+	m.m.RLock()
+	defer m.m.RUnlock()
+
+	cd, ok := m.cache[key]
+	if ok {
+		return cd.remoteDocument, cd.expireTime, nil
+	}
+	return nil, time.Time{}, ErrCacheMiss
+}
+
+func (m *memoryCacheEngine) Set(key string, doc *ld.RemoteDocument,
+	expireTime time.Time) error {
+
+	if m.embedDocs != nil {
+		// if we have the document in the embedded cache, do not overwrite it
+		// with the new value.
+		_, ok := m.embedDocs[key]
+		if ok {
+			return nil
+		}
+	}
+
+	m.m.Lock()
+	defer m.m.Unlock()
+
+	m.cache[key] = &cachedRemoteDocument{
+		remoteDocument: doc,
+		expireTime:     expireTime,
+	}
+
+	return nil
+}
+
+type MemoryCacheEngineOption func(*memoryCacheEngine) error
+
+func WithEmbeddedDocumentBytes(u string, doc []byte) MemoryCacheEngineOption {
+	return func(engine *memoryCacheEngine) error {
+		if engine.embedDocs == nil {
+			engine.embedDocs = make(map[string]*ld.RemoteDocument)
+		}
+
+		var rd = &ld.RemoteDocument{DocumentURL: u}
+		err := json.Unmarshal(doc, &rd.Document)
+		if err != nil {
+			return err
+		}
+
+		engine.embedDocs[u] = rd
+		return nil
+	}
+}
+
+func NewMemoryCacheEngine(
+	opts ...MemoryCacheEngineOption) (CacheEngine, error) {
+
+	e := &memoryCacheEngine{
+		cache: make(map[string]*cachedRemoteDocument),
+	}
+
+	for _, opt := range opts {
+		err := opt(e)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return e, nil
+}