From 4057541945b580be4ad385ea6c9892b4ac0efc50 Mon Sep 17 00:00:00 2001 From: Ashwin Bhat <1727158+ashwinbhat@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:44:29 -0700 Subject: [PATCH] Add support for data library referencing (#2054) ### Background MaterialX documents are required to be complete self contained with all required node definitions. This is done by using the `mx::importLibrary` API. While processing large number of documents the Import and Merge workflows can cause performance and memory challenges. ### Proposal Since the MaterialX library or the Data library hosts definitions, we propose that these definitions could be referenced by a document instead of importing them. This concept of referencing the Data library may have some workflow impact such as, do new definitions to into Data library or the document or can a document have local definitions? The answer depends on the specific artist or studio workflow. To maintain compatibility with existing usage patterns, we have introduced a new set of methods **that allow registration of data library for a document by setting a document reference to it**. 1. setDataLibrary(ConstDocumentPtr dataLibrary); 2. getDataLibrary() 3. hasDataLibrary() ### Using this new Data library registration, two usage patterns are possible **Build library and import** This is the current usage pattern - use `importlibrary `to build your datalibrary - use `importLibrary `to import the datalibrary to the working document. **Build library and register** Proposed usage pattern - use `importlibrary` to build your datalibrary - use `setDataLibrary `to assign the datalibrary to the working document. This can be extended were 3rd party defs can be imported into the datalibrary **Other changes include** - Updates to `getChildOfType` and `getChildrenOfType` to support Data Library. - Updates to redirect _Document::get*_ calls to process local document content and Data Library. --- .../JsMaterialXCore/JsDocument.cpp | 3 ++ source/MaterialXCore/Document.cpp | 26 ++++++++----- source/MaterialXCore/Document.h | 38 ++++++++++++++++--- source/MaterialXCore/Element.cpp | 16 +++++++- .../MaterialXTest/MaterialXCore/Document.cpp | 6 ++- source/MaterialXTest/MaterialXCore/Node.cpp | 2 +- .../MaterialXGenShader/GenShader.cpp | 8 ++-- .../MaterialXGenShader/GenShaderUtil.cpp | 8 ++-- .../MaterialXRender/RenderUtil.cpp | 2 +- source/MaterialXView/Viewer.cpp | 4 +- .../PyMaterialXCore/PyDocument.cpp | 3 ++ 11 files changed, 86 insertions(+), 30 deletions(-) diff --git a/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp b/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp index ec4881732d..18750666b7 100644 --- a/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp +++ b/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp @@ -23,6 +23,9 @@ EMSCRIPTEN_BINDINGS(document) .class_function("createDocument", &mx::Document::createDocument) .function("initialize", &mx::Document::initialize) .function("copy", &mx::Document::copy) + .function("setDataLibrary", &mx::Document::setDataLibrary) + .function("getDataLibrary", &mx::Document::getDataLibrary) + .function("hasDataLibrary", &mx::Document::hasDataLibrary) .function("importLibrary", &mx::Document::importLibrary) .function("getReferencedSourceUris", ems::optional_override([](mx::Document &self) { mx::StringSet set = self.getReferencedSourceUris(); diff --git a/source/MaterialXCore/Document.cpp b/source/MaterialXCore/Document.cpp index 7ebdab0aa7..1cd031d87c 100644 --- a/source/MaterialXCore/Document.cpp +++ b/source/MaterialXCore/Document.cpp @@ -357,34 +357,40 @@ vector Document::getMaterialOutputs() const vector Document::getMatchingNodeDefs(const string& nodeName) const { + // Recurse to data library if present. + vector matchingNodeDefs = hasDataLibrary() ? + getDataLibrary()->getMatchingNodeDefs(nodeName) : + vector(); + // Refresh the cache. _cache->refresh(); // Return all nodedefs matching the given node name. if (_cache->nodeDefMap.count(nodeName)) { - return _cache->nodeDefMap.at(nodeName); - } - else - { - return vector(); + matchingNodeDefs.insert(matchingNodeDefs.end(), _cache->nodeDefMap.at(nodeName).begin(), _cache->nodeDefMap.at(nodeName).end()); } + + return matchingNodeDefs; } vector Document::getMatchingImplementations(const string& nodeDef) const { + // Recurse to data library if present. + vector matchingImplementations = hasDataLibrary() ? + getDataLibrary()->getMatchingImplementations(nodeDef) : + vector(); + // Refresh the cache. _cache->refresh(); // Return all implementations matching the given nodedef string. if (_cache->implementationMap.count(nodeDef)) { - return _cache->implementationMap.at(nodeDef); - } - else - { - return vector(); + matchingImplementations.insert(matchingImplementations.end(), _cache->implementationMap.at(nodeDef).begin(), _cache->implementationMap.at(nodeDef).end()); } + + return matchingImplementations; } bool Document::validate(string* message) const diff --git a/source/MaterialXCore/Document.h b/source/MaterialXCore/Document.h index a14a5edf7b..ab01bbbc11 100644 --- a/source/MaterialXCore/Document.h +++ b/source/MaterialXCore/Document.h @@ -50,18 +50,41 @@ class MX_CORE_API Document : public GraphElement { DocumentPtr doc = createDocument(); doc->copyContentFrom(getSelf()); + doc->setDataLibrary(getDataLibrary()); return doc; } - /// Import the given document as a library within this document. - /// The contents of the library document are copied into this one, and + /// Get a list of source URIs referenced by the document + StringSet getReferencedSourceUris() const; + + /// @name Data Libraries + /// @{ + + /// Store a reference to a data library in this document. + void setDataLibrary(ConstDocumentPtr dataLibrary) + { + _dataLibrary = dataLibrary; + } + + /// Return true if this document has a data library. + bool hasDataLibrary() const + { + return (_dataLibrary != nullptr); + } + + /// Return the data library, if any, referenced by this document. + ConstDocumentPtr getDataLibrary() const + { + return _dataLibrary; + } + + /// Import the given data library into this document. + /// The contents of the data library are copied into this one, and /// are assigned the source URI of the library. - /// @param library The library document to be imported. + /// @param library The data library to be imported. void importLibrary(const ConstDocumentPtr& library); - /// Get a list of source URI's referenced by the document - StringSet getReferencedSourceUris() const; - + /// @} /// @name NodeGraph Elements /// @{ @@ -679,6 +702,9 @@ class MX_CORE_API Document : public GraphElement private: class Cache; + + private: + ConstDocumentPtr _dataLibrary; std::unique_ptr _cache; }; diff --git a/source/MaterialXCore/Element.cpp b/source/MaterialXCore/Element.cpp index 1a48142962..af47a6a2ba 100644 --- a/source/MaterialXCore/Element.cpp +++ b/source/MaterialXCore/Element.cpp @@ -286,13 +286,27 @@ ElementPtr Element::changeChildCategory(ElementPtr child, const string& category template shared_ptr Element::getChildOfType(const string& name) const { - ElementPtr child = getChild(name); + ElementPtr child; + ConstDocumentPtr doc = asA(); + if (doc && doc->hasDataLibrary()) + { + child = doc->getDataLibrary()->getChild(name); + } + if (!child) + { + child = getChild(name); + } return child ? child->asA() : shared_ptr(); } template vector> Element::getChildrenOfType(const string& category) const { vector> children; + ConstDocumentPtr doc = asA(); + if (doc && doc->hasDataLibrary()) + { + children = doc->getDataLibrary()->getChildrenOfType(category); + } for (ElementPtr child : _childOrder) { shared_ptr instance = child->asA(); diff --git a/source/MaterialXTest/MaterialXCore/Document.cpp b/source/MaterialXTest/MaterialXCore/Document.cpp index 1b02966fb8..94f8b98708 100644 --- a/source/MaterialXTest/MaterialXCore/Document.cpp +++ b/source/MaterialXTest/MaterialXCore/Document.cpp @@ -106,7 +106,11 @@ TEST_CASE("Document", "[document]") REQUIRE(customLibrary->validate()); // Import the custom library. - doc->importLibrary(customLibrary); + mx::DocumentPtr customDatalibrary = mx::createDocument(); + customDatalibrary->importLibrary(customLibrary); + + // Set data library + doc->setDataLibrary(customDatalibrary); mx::NodeGraphPtr importedNodeGraph = doc->getNodeGraph("custom:NG_custom"); mx::NodeDefPtr importedNodeDef = doc->getNodeDef("custom:ND_simpleSrf"); mx::ImplementationPtr importedImpl = doc->getImplementation("custom:IM_custom"); diff --git a/source/MaterialXTest/MaterialXCore/Node.cpp b/source/MaterialXTest/MaterialXCore/Node.cpp index b72cd65328..58ee125bad 100644 --- a/source/MaterialXTest/MaterialXCore/Node.cpp +++ b/source/MaterialXTest/MaterialXCore/Node.cpp @@ -672,7 +672,7 @@ TEST_CASE("Node Definition Creation", "[nodedef]") mx::DocumentPtr doc = mx::createDocument(); mx::readFromXmlFile(doc, "resources/Materials/TestSuite/stdlib/definition/definition_from_nodegraph.mtlx", searchPath); - doc->importLibrary(stdlib); + doc->setDataLibrary(stdlib); mx::NodeGraphPtr graph = doc->getNodeGraph("test_colorcorrect"); REQUIRE(graph); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp index 22afffbff4..a959283aad 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp @@ -161,7 +161,7 @@ TEST_CASE("GenShader: Transparency Regression Check", "[genshader]") bool testValue = transparencyTest[i]; mx::DocumentPtr testDoc = mx::createDocument(); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); try { @@ -207,7 +207,7 @@ void testDeterministicGeneration(mx::DocumentPtr libraries, mx::GenContext& cont { mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); // Keep the document alive to make sure // new memory is allocated for each run @@ -272,7 +272,7 @@ void checkPixelDependencies(mx::DocumentPtr libraries, mx::GenContext& context) mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); @@ -385,7 +385,7 @@ TEST_CASE("GenShader: Track Application Variables", "[genshader]") mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlString(testDoc, testDocumentString); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index 6dc24fc15c..4a8e61645a 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -372,7 +372,7 @@ void shaderGenPerformanceTest(mx::GenContext& context) std::shuffle(loadedDocuments.begin(), loadedDocuments.end(), rng); for (const auto& doc : loadedDocuments) { - doc->importLibrary(nodeLibrary); + doc->setDataLibrary(nodeLibrary); std::vector elements = mx::findRenderableElements(doc); REQUIRE(elements.size() > 0); @@ -721,7 +721,7 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons bool importedLibrary = false; try { - doc->importLibrary(_dependLib); + doc->setDataLibrary(_dependLib); importedLibrary = true; } catch (mx::Exception& e) @@ -734,8 +734,8 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons } // Find and register lights - findLights(doc, _lights); - registerLights(doc, _lights, context); + findLights(_dependLib, _lights); + registerLights(_dependLib, _lights, context); // Find elements to render in the document std::vector elements; diff --git a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp index d8265a4050..e3e5e8033e 100644 --- a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp +++ b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp @@ -223,7 +223,7 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) // colliding with implementations in previous test cases. context.clearNodeImplementations(); - doc->importLibrary(dependLib); + doc->setDataLibrary(dependLib); ioTimer.endTimer(); validateTimer.startTimer(); diff --git a/source/MaterialXView/Viewer.cpp b/source/MaterialXView/Viewer.cpp index c0d90d30a7..24aea1eb42 100644 --- a/source/MaterialXView/Viewer.cpp +++ b/source/MaterialXView/Viewer.cpp @@ -1316,8 +1316,8 @@ void Viewer::loadDocument(const mx::FilePath& filename, mx::DocumentPtr librarie mx::readFromXmlFile(doc, filename, _searchPath, &readOptions); _materialSearchPath = mx::getSourceSearchPath(doc); - // Import libraries. - doc->importLibrary(libraries); + // Store data library reference. + doc->setDataLibrary(libraries); // Apply direct lights. applyDirectLights(doc); diff --git a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp index 3f4403c673..e021bc92de 100644 --- a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp +++ b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp @@ -29,6 +29,9 @@ void bindPyDocument(py::module& mod) py::class_(mod, "Document") .def("initialize", &mx::Document::initialize) .def("copy", &mx::Document::copy) + .def("setDataLibrary", &mx::Document::setDataLibrary) + .def("getDataLibrary", &mx::Document::getDataLibrary) + .def("hasDataLibrary", &mx::Document::hasDataLibrary) .def("importLibrary", &mx::Document::importLibrary) .def("getReferencedSourceUris", &mx::Document::getReferencedSourceUris) .def("addNodeGraph", &mx::Document::addNodeGraph,