From a54a62fc2ab0409cea30de08ebfeedf872adadf1 Mon Sep 17 00:00:00 2001 From: Ashwin Bhat <1727158+ashwinbhat@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:44:29 -0700 Subject: [PATCH] Add support for data library referencing (#2054) ### Background MaterialX documents are required to be complete self contained with all required node definitions. This is done by using the `mx::importLibrary` API. While processing large number of documents the Import and Merge workflows can cause performance and memory challenges. ### Proposal Since the MaterialX library or the Data library hosts definitions, we propose that these definitions could be referenced by a document instead of importing them. This concept of referencing the Data library may have some workflow impact such as, do new definitions to into Data library or the document or can a document have local definitions? The answer depends on the specific artist or studio workflow. To maintain compatibility with existing usage patterns, we have introduced a new set of methods **that allow registration of data library for a document by setting a document reference to it**. 1. setDataLibrary(ConstDocumentPtr dataLibrary); 2. getDataLibrary() 3. hasDataLibrary() ### Using this new Data library registration, two usage patterns are possible **Build library and import** This is the current usage pattern - use `importlibrary `to build your datalibrary - use `importLibrary `to import the datalibrary to the working document. **Build library and register** Proposed usage pattern - use `importlibrary` to build your datalibrary - use `setDataLibrary `to assign the datalibrary to the working document. This can be extended were 3rd party defs can be imported into the datalibrary **Other changes include** - Updates to `getChildOfType` and `getChildrenOfType` to support Data Library. - Updates to redirect _Document::get*_ calls to process local document content and Data Library. --- .../JsMaterialXCore/JsDocument.cpp | 3 ++ source/MaterialXCore/Document.cpp | 26 ++++++++----- source/MaterialXCore/Document.h | 38 ++++++++++++++++--- source/MaterialXCore/Element.cpp | 16 +++++++- .../MaterialXTest/MaterialXCore/Document.cpp | 6 ++- source/MaterialXTest/MaterialXCore/Node.cpp | 2 +- .../MaterialXGenShader/GenShader.cpp | 8 ++-- .../MaterialXGenShader/GenShaderUtil.cpp | 8 ++-- .../MaterialXRender/RenderUtil.cpp | 2 +- source/MaterialXView/Viewer.cpp | 4 +- .../PyMaterialXCore/PyDocument.cpp | 3 ++ 11 files changed, 86 insertions(+), 30 deletions(-) diff --git a/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp b/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp index 23901ecc0f..dc83e7e7f7 100644 --- a/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp +++ b/source/JsMaterialX/JsMaterialXCore/JsDocument.cpp @@ -23,6 +23,9 @@ EMSCRIPTEN_BINDINGS(document) .class_function("createDocument", &mx::Document::createDocument) .function("initialize", &mx::Document::initialize) .function("copy", &mx::Document::copy) + .function("setDataLibrary", &mx::Document::setDataLibrary) + .function("getDataLibrary", &mx::Document::getDataLibrary) + .function("hasDataLibrary", &mx::Document::hasDataLibrary) .function("importLibrary", &mx::Document::importLibrary) .function("getReferencedSourceUris", ems::optional_override([](mx::Document &self) { mx::StringSet set = self.getReferencedSourceUris(); diff --git a/source/MaterialXCore/Document.cpp b/source/MaterialXCore/Document.cpp index b6ff1041a1..ba7b4b8fd9 100644 --- a/source/MaterialXCore/Document.cpp +++ b/source/MaterialXCore/Document.cpp @@ -358,34 +358,40 @@ vector Document::getMaterialOutputs() const vector Document::getMatchingNodeDefs(const string& nodeName) const { + // Recurse to data library if present. + vector matchingNodeDefs = hasDataLibrary() ? + getDataLibrary()->getMatchingNodeDefs(nodeName) : + vector(); + // Refresh the cache. _cache->refresh(); // Return all nodedefs matching the given node name. if (_cache->nodeDefMap.count(nodeName)) { - return _cache->nodeDefMap.at(nodeName); - } - else - { - return vector(); + matchingNodeDefs.insert(matchingNodeDefs.end(), _cache->nodeDefMap.at(nodeName).begin(), _cache->nodeDefMap.at(nodeName).end()); } + + return matchingNodeDefs; } vector Document::getMatchingImplementations(const string& nodeDef) const { + // Recurse to data library if present. + vector matchingImplementations = hasDataLibrary() ? + getDataLibrary()->getMatchingImplementations(nodeDef) : + vector(); + // Refresh the cache. _cache->refresh(); // Return all implementations matching the given nodedef string. if (_cache->implementationMap.count(nodeDef)) { - return _cache->implementationMap.at(nodeDef); - } - else - { - return vector(); + matchingImplementations.insert(matchingImplementations.end(), _cache->implementationMap.at(nodeDef).begin(), _cache->implementationMap.at(nodeDef).end()); } + + return matchingImplementations; } bool Document::validate(string* message) const diff --git a/source/MaterialXCore/Document.h b/source/MaterialXCore/Document.h index 8f5abbf8fb..1e53ddbc3d 100644 --- a/source/MaterialXCore/Document.h +++ b/source/MaterialXCore/Document.h @@ -50,18 +50,41 @@ class MX_CORE_API Document : public GraphElement { DocumentPtr doc = createDocument(); doc->copyContentFrom(getSelf()); + doc->setDataLibrary(getDataLibrary()); return doc; } - /// Import the given document as a library within this document. - /// The contents of the library document are copied into this one, and + /// Get a list of source URIs referenced by the document + StringSet getReferencedSourceUris() const; + + /// @name Data Libraries + /// @{ + + /// Store a reference to a data library in this document. + void setDataLibrary(ConstDocumentPtr dataLibrary) + { + _dataLibrary = dataLibrary; + } + + /// Return true if this document has a data library. + bool hasDataLibrary() const + { + return (_dataLibrary != nullptr); + } + + /// Return the data library, if any, referenced by this document. + ConstDocumentPtr getDataLibrary() const + { + return _dataLibrary; + } + + /// Import the given data library into this document. + /// The contents of the data library are copied into this one, and /// are assigned the source URI of the library. - /// @param library The library document to be imported. + /// @param library The data library to be imported. void importLibrary(const ConstDocumentPtr& library); - /// Get a list of source URI's referenced by the document - StringSet getReferencedSourceUris() const; - + /// @} /// @name NodeGraph Elements /// @{ @@ -676,6 +699,9 @@ class MX_CORE_API Document : public GraphElement private: class Cache; + + private: + ConstDocumentPtr _dataLibrary; std::unique_ptr _cache; }; diff --git a/source/MaterialXCore/Element.cpp b/source/MaterialXCore/Element.cpp index 597c67c13e..f0aa0a4c70 100644 --- a/source/MaterialXCore/Element.cpp +++ b/source/MaterialXCore/Element.cpp @@ -278,13 +278,27 @@ ElementPtr Element::changeChildCategory(ElementPtr child, const string& category template shared_ptr Element::getChildOfType(const string& name) const { - ElementPtr child = getChild(name); + ElementPtr child; + ConstDocumentPtr doc = asA(); + if (doc && doc->hasDataLibrary()) + { + child = doc->getDataLibrary()->getChild(name); + } + if (!child) + { + child = getChild(name); + } return child ? child->asA() : shared_ptr(); } template vector> Element::getChildrenOfType(const string& category) const { vector> children; + ConstDocumentPtr doc = asA(); + if (doc && doc->hasDataLibrary()) + { + children = doc->getDataLibrary()->getChildrenOfType(category); + } for (ElementPtr child : _childOrder) { shared_ptr instance = child->asA(); diff --git a/source/MaterialXTest/MaterialXCore/Document.cpp b/source/MaterialXTest/MaterialXCore/Document.cpp index 9670c54283..90c86e5102 100644 --- a/source/MaterialXTest/MaterialXCore/Document.cpp +++ b/source/MaterialXTest/MaterialXCore/Document.cpp @@ -97,7 +97,11 @@ TEST_CASE("Document", "[document]") REQUIRE(customLibrary->validate()); // Import the custom library. - doc->importLibrary(customLibrary); + mx::DocumentPtr customDatalibrary = mx::createDocument(); + customDatalibrary->importLibrary(customLibrary); + + // Set data library + doc->setDataLibrary(customDatalibrary); mx::NodeGraphPtr importedNodeGraph = doc->getNodeGraph("custom:NG_custom"); mx::NodeDefPtr importedNodeDef = doc->getNodeDef("custom:ND_simpleSrf"); mx::ImplementationPtr importedImpl = doc->getImplementation("custom:IM_custom"); diff --git a/source/MaterialXTest/MaterialXCore/Node.cpp b/source/MaterialXTest/MaterialXCore/Node.cpp index 655ec5b7fb..68c6688f29 100644 --- a/source/MaterialXTest/MaterialXCore/Node.cpp +++ b/source/MaterialXTest/MaterialXCore/Node.cpp @@ -621,7 +621,7 @@ TEST_CASE("Node Definition Creation", "[nodedef]") mx::DocumentPtr doc = mx::createDocument(); mx::readFromXmlFile(doc, "resources/Materials/TestSuite/stdlib/definition/definition_from_nodegraph.mtlx", searchPath); - doc->importLibrary(stdlib); + doc->setDataLibrary(stdlib); mx::NodeGraphPtr graph = doc->getNodeGraph("test_colorcorrect"); REQUIRE(graph); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp index 9b25a1d6b7..9306cf8715 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp @@ -162,7 +162,7 @@ TEST_CASE("GenShader: Transparency Regression Check", "[genshader]") bool testValue = transparencyTest[i]; mx::DocumentPtr testDoc = mx::createDocument(); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); try { @@ -208,7 +208,7 @@ void testDeterministicGeneration(mx::DocumentPtr libraries, mx::GenContext& cont { mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); // Keep the document alive to make sure // new memory is allocated for each run @@ -273,7 +273,7 @@ void checkPixelDependencies(mx::DocumentPtr libraries, mx::GenContext& context) mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); @@ -386,7 +386,7 @@ TEST_CASE("GenShader: Track Application Variables", "[genshader]") mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlString(testDoc, testDocumentString); - testDoc->importLibrary(libraries); + testDoc->setDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index 5aa1d7aa2d..724dde2c40 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -374,7 +374,7 @@ void shaderGenPerformanceTest(mx::GenContext& context) std::shuffle(loadedDocuments.begin(), loadedDocuments.end(), rng); for (const auto& doc : loadedDocuments) { - doc->importLibrary(nodeLibrary); + doc->setDataLibrary(nodeLibrary); std::vector elements = mx::findRenderableElements(doc); REQUIRE(elements.size() > 0); @@ -723,7 +723,7 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons bool importedLibrary = false; try { - doc->importLibrary(_dependLib); + doc->setDataLibrary(_dependLib); importedLibrary = true; } catch (mx::Exception& e) @@ -736,8 +736,8 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons } // Find and register lights - findLights(doc, _lights); - registerLights(doc, _lights, context); + findLights(_dependLib, _lights); + registerLights(_dependLib, _lights, context); // Find elements to render in the document std::vector elements; diff --git a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp index cabcaca9a4..4aecdbd926 100644 --- a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp +++ b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp @@ -227,7 +227,7 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) // colliding with implementations in previous test cases. context.clearNodeImplementations(); - doc->importLibrary(dependLib); + doc->setDataLibrary(dependLib); ioTimer.endTimer(); validateTimer.startTimer(); diff --git a/source/MaterialXView/Viewer.cpp b/source/MaterialXView/Viewer.cpp index 7cd70d6c7f..5c4dcb67e8 100644 --- a/source/MaterialXView/Viewer.cpp +++ b/source/MaterialXView/Viewer.cpp @@ -1202,8 +1202,8 @@ void Viewer::loadDocument(const mx::FilePath& filename, mx::DocumentPtr librarie mx::readFromXmlFile(doc, filename, _searchPath, &readOptions); _materialSearchPath = mx::getSourceSearchPath(doc); - // Import libraries. - doc->importLibrary(libraries); + // Store data library reference. + doc->setDataLibrary(libraries); // Apply direct lights. applyDirectLights(doc); diff --git a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp index 101e1bea64..9ed1250ce5 100644 --- a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp +++ b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp @@ -17,6 +17,9 @@ void bindPyDocument(py::module& mod) py::class_(mod, "Document") .def("initialize", &mx::Document::initialize) .def("copy", &mx::Document::copy) + .def("setDataLibrary", &mx::Document::setDataLibrary) + .def("getDataLibrary", &mx::Document::getDataLibrary) + .def("hasDataLibrary", &mx::Document::hasDataLibrary) .def("importLibrary", &mx::Document::importLibrary) .def("getReferencedSourceUris", &mx::Document::getReferencedSourceUris) .def("addNodeGraph", &mx::Document::addNodeGraph,