From 7e61f636d1449687d0a5df7562bc23b0470d7c4e Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Mon, 7 Oct 2024 18:33:03 -0700 Subject: [PATCH] Data Library API Introduce methods to register a data library for a document --- source/MaterialXCore/Definition.cpp | 4 +- source/MaterialXCore/Document.cpp | 17 +++++++ source/MaterialXCore/Document.h | 48 +++++++++++++++++++ source/MaterialXCore/Element.cpp | 5 +- source/MaterialXCore/Interface.cpp | 4 +- source/MaterialXCore/Node.cpp | 29 +++++++++-- source/MaterialXGenShader/ShaderGraph.cpp | 11 +++-- .../MaterialXTest/MaterialXCore/Document.cpp | 6 ++- source/MaterialXTest/MaterialXCore/Node.cpp | 2 +- .../MaterialXGenShader/GenShader.cpp | 8 ++-- .../MaterialXGenShader/GenShaderUtil.cpp | 4 +- .../MaterialXRender/RenderUtil.cpp | 2 +- source/MaterialXView/Viewer.cpp | 2 +- .../PyMaterialXCore/PyDocument.cpp | 3 ++ 14 files changed, 125 insertions(+), 20 deletions(-) diff --git a/source/MaterialXCore/Definition.cpp b/source/MaterialXCore/Definition.cpp index 22c111f03c..6b45baf195 100644 --- a/source/MaterialXCore/Definition.cpp +++ b/source/MaterialXCore/Definition.cpp @@ -185,8 +185,10 @@ StringVec TargetDef::getMatchingTargets() const vector UnitTypeDef::getUnitDefs() const { + const auto datalibrary = getDocument()->hasDataLibrary() ? getDocument()->getRegisteredDataLibrary() : getDocument(); + vector unitDefs; - for (UnitDefPtr unitDef : getDocument()->getChildrenOfType()) + for (UnitDefPtr unitDef : datalibrary->getChildrenOfType()) { if (unitDef->getUnitType() == _name) { diff --git a/source/MaterialXCore/Document.cpp b/source/MaterialXCore/Document.cpp index 7ebdab0aa7..267597e266 100644 --- a/source/MaterialXCore/Document.cpp +++ b/source/MaterialXCore/Document.cpp @@ -357,6 +357,14 @@ vector Document::getMaterialOutputs() const vector Document::getMatchingNodeDefs(const string& nodeName) const { + // Return all nodedefs from datalibrary if available + if (_dataLibrary) + { + auto datalibrarynodes = _dataLibrary->getMatchingNodeDefs(nodeName); + if (!datalibrarynodes.empty()) + return datalibrarynodes; + } + // Refresh the cache. _cache->refresh(); @@ -373,6 +381,15 @@ vector Document::getMatchingNodeDefs(const string& nodeName) const vector Document::getMatchingImplementations(const string& nodeDef) const { + + // Return all implementations from datalibrary if available + if (_dataLibrary) + { + auto datalibrarynodes = _dataLibrary->getMatchingImplementations(nodeDef); + if (!datalibrarynodes.empty()) + return datalibrarynodes; + } + // Refresh the cache. _cache->refresh(); diff --git a/source/MaterialXCore/Document.h b/source/MaterialXCore/Document.h index a14a5edf7b..b0a7c088e2 100644 --- a/source/MaterialXCore/Document.h +++ b/source/MaterialXCore/Document.h @@ -78,12 +78,16 @@ class MX_CORE_API Document : public GraphElement /// Return the NodeGraph, if any, with the given name. NodeGraphPtr getNodeGraph(const string& name) const { + if (_dataLibrary) + return _dataLibrary->getChildOfType(name); return getChildOfType(name); } /// Return a vector of all NodeGraph elements in the document. vector getNodeGraphs() const { + if (_dataLibrary) + return _dataLibrary->getChildrenOfType(); return getChildrenOfType(); } @@ -345,12 +349,16 @@ class MX_CORE_API Document : public GraphElement /// Return the NodeDef, if any, with the given name. NodeDefPtr getNodeDef(const string& name) const { + if (_dataLibrary) + return _dataLibrary->getChildOfType(name); return getChildOfType(name); } /// Return a vector of all NodeDef elements in the document. vector getNodeDefs() const { + if (_dataLibrary) + return _dataLibrary->getChildrenOfType(); return getChildrenOfType(); } @@ -380,12 +388,16 @@ class MX_CORE_API Document : public GraphElement /// Return the AttributeDef, if any, with the given name. AttributeDefPtr getAttributeDef(const string& name) const { + if (_dataLibrary) + return _dataLibrary->getChildOfType(name); return getChildOfType(name); } /// Return a vector of all AttributeDef elements in the document. vector getAttributeDefs() const { + if (_dataLibrary) + return _dataLibrary->getChildrenOfType(); return getChildrenOfType(); } @@ -412,12 +424,16 @@ class MX_CORE_API Document : public GraphElement /// Return the AttributeDef, if any, with the given name. TargetDefPtr getTargetDef(const string& name) const { + if (_dataLibrary) + return _dataLibrary->getChildOfType(name); return getChildOfType(name); } /// Return a vector of all TargetDef elements in the document. vector getTargetDefs() const { + if (_dataLibrary) + return _dataLibrary->getChildrenOfType(); return getChildrenOfType(); } @@ -508,12 +524,16 @@ class MX_CORE_API Document : public GraphElement /// Return the Implementation, if any, with the given name. ImplementationPtr getImplementation(const string& name) const { + if (_dataLibrary) + return _dataLibrary->getChildOfType(name); return getChildOfType(name); } /// Return a vector of all Implementation elements in the document. vector getImplementations() const { + if (_dataLibrary) + return _dataLibrary->getChildrenOfType(); return getChildrenOfType(); } @@ -665,6 +685,32 @@ class MX_CORE_API Document : public GraphElement /// @} + /// @name MaterialX data library + /// @{ + + /// Register the given document as MaterialX data library for document + /// The MaterialX data library can be created using the loadLibraries utility + /// For improved performance it is recommended the data library + /// is on the document instead of importing it + /// @param Data Library document to register. + void registerDataLibrary(ConstDocumentPtr dataLibrary) + { + _dataLibrary = dataLibrary; + } + + /// Gets the registered data library + ConstDocumentPtr getRegisteredDataLibrary() const + { + return _dataLibrary; + } + + /// Returns true if a data library is registered. + bool hasDataLibrary() const + { + return (_dataLibrary != nullptr); + } + /// @} + // // These are deprecated wrappers for older versions of the function interfaces in this module. // Clients using these interfaces should update them to the latest API. @@ -680,6 +726,8 @@ class MX_CORE_API Document : public GraphElement private: class Cache; std::unique_ptr _cache; + // Data library for the document + ConstDocumentPtr _dataLibrary; }; /// Create a new Document. diff --git a/source/MaterialXCore/Element.cpp b/source/MaterialXCore/Element.cpp index 94836245a7..eb67c453e1 100644 --- a/source/MaterialXCore/Element.cpp +++ b/source/MaterialXCore/Element.cpp @@ -564,7 +564,10 @@ bool ValueElement::validate(string* message) const const string& unittype = getUnitType(); if (!unittype.empty()) { - unitTypeDef = getDocument()->getUnitTypeDef(unittype); + + unitTypeDef = getDocument()->hasDataLibrary() ? + getDocument()->getRegisteredDataLibrary()->getUnitTypeDef(unittype) : + getDocument()->getUnitTypeDef(unittype); validateRequire(unitTypeDef != nullptr, res, message, "Unit type definition does not exist in document"); } } diff --git a/source/MaterialXCore/Interface.cpp b/source/MaterialXCore/Interface.cpp index 24d3ccfe0d..dfb84efc04 100644 --- a/source/MaterialXCore/Interface.cpp +++ b/source/MaterialXCore/Interface.cpp @@ -282,7 +282,9 @@ GeomPropDefPtr Input::getDefaultGeomProp() const const string& defaultGeomProp = getAttribute(DEFAULT_GEOM_PROP_ATTRIBUTE); if (!defaultGeomProp.empty()) { - ConstDocumentPtr doc = getDocument(); + ConstDocumentPtr doc = getDocument()->hasDataLibrary() ? + getDocument()->getRegisteredDataLibrary() : + getDocument(); return doc->getChildOfType(defaultGeomProp); } return nullptr; diff --git a/source/MaterialXCore/Node.cpp b/source/MaterialXCore/Node.cpp index 9080036c39..3ed301bc15 100644 --- a/source/MaterialXCore/Node.cpp +++ b/source/MaterialXCore/Node.cpp @@ -70,14 +70,21 @@ string Node::getConnectedNodeName(const string& inputName) const NodeDefPtr Node::getNodeDef(const string& target, bool allowRoughMatch) const { - if (hasNodeDefString()) - { - return resolveNameReference(getNodeDefString()); - } vector nodeDefs = getDocument()->getMatchingNodeDefs(getQualifiedName(getCategory())); vector secondary = getDocument()->getMatchingNodeDefs(getCategory()); - vector roughMatches; nodeDefs.insert(nodeDefs.end(), secondary.begin(), secondary.end()); + + // Search data library if available + if (getDocument()->hasDataLibrary()) + { + vector libraryNodeDefs = getDocument()->getRegisteredDataLibrary()->getMatchingNodeDefs(getQualifiedName(getCategory())); + vector librarySecondardNodeDefs = getDocument()->getRegisteredDataLibrary()->getMatchingNodeDefs(getCategory()); + nodeDefs.insert(nodeDefs.end(), libraryNodeDefs.begin(), libraryNodeDefs.end()); + nodeDefs.insert(nodeDefs.end(), librarySecondardNodeDefs.begin(), librarySecondardNodeDefs.end()); + } + + vector roughMatches; + for (NodeDefPtr nodeDef : nodeDefs) { if (!targetStringsMatch(nodeDef->getTarget(), target) || @@ -714,6 +721,18 @@ NodeDefPtr NodeGraph::getNodeDef() const } } } + // Check datalibrary if available + if (!nodedef && getDocument()->hasDataLibrary()) + { + const auto datalibray = getDocument()->getRegisteredDataLibrary(); + for (auto impl : datalibray->getImplementations()) + { + if (impl->getNodeGraph() == getQualifiedName(getName())) + { + nodedef = impl->getNodeDef(); + } + } + } return nodedef; } diff --git a/source/MaterialXGenShader/ShaderGraph.cpp b/source/MaterialXGenShader/ShaderGraph.cpp index 0f39b46c32..371ef0d544 100644 --- a/source/MaterialXGenShader/ShaderGraph.cpp +++ b/source/MaterialXGenShader/ShaderGraph.cpp @@ -202,10 +202,15 @@ void ShaderGraph::addDefaultGeomNode(ShaderInput* input, const GeomPropDef& geom // input here and ignore the type of the geomprop. They are required to have the same type. string geomNodeDefName = "ND_" + geomprop.getGeomProp() + "_" + input->getType().getName(); NodeDefPtr geomNodeDef = _document->getNodeDef(geomNodeDefName); - if (!geomNodeDef) + if (!geomNodeDef && _document->hasDataLibrary()) { - throw ExceptionShaderGenError("Could not find a nodedef named '" + geomNodeDefName + - "' for defaultgeomprop on input '" + input->getFullName() + "'"); + geomNodeDef = _document->getRegisteredDataLibrary()->getNodeDef(geomNodeDefName); + if (!geomNodeDef) + { + + throw ExceptionShaderGenError("Could not find a nodedef named '" + geomNodeDefName + + "' for defaultgeomprop on input '" + input->getFullName() + "'"); + } } ShaderNodePtr geomNode = ShaderNode::create(this, geomNodeName, *geomNodeDef, context); diff --git a/source/MaterialXTest/MaterialXCore/Document.cpp b/source/MaterialXTest/MaterialXCore/Document.cpp index fb35e58ccb..ad2b6dadc4 100644 --- a/source/MaterialXTest/MaterialXCore/Document.cpp +++ b/source/MaterialXTest/MaterialXCore/Document.cpp @@ -104,7 +104,11 @@ TEST_CASE("Document", "[document]") REQUIRE(customLibrary->validate()); // Import the custom library. - doc->importLibrary(customLibrary); + mx::DocumentPtr customdatalibrary = mx::createDocument(); + customdatalibrary->importLibrary(customLibrary); + + // Register data library + doc->registerDataLibrary(customdatalibrary); mx::NodeGraphPtr importedNodeGraph = doc->getNodeGraph("custom:NG_custom"); mx::NodeDefPtr importedNodeDef = doc->getNodeDef("custom:ND_simpleSrf"); mx::ImplementationPtr importedImpl = doc->getImplementation("custom:IM_custom"); diff --git a/source/MaterialXTest/MaterialXCore/Node.cpp b/source/MaterialXTest/MaterialXCore/Node.cpp index b72cd65328..841bc2acaa 100644 --- a/source/MaterialXTest/MaterialXCore/Node.cpp +++ b/source/MaterialXTest/MaterialXCore/Node.cpp @@ -672,7 +672,7 @@ TEST_CASE("Node Definition Creation", "[nodedef]") mx::DocumentPtr doc = mx::createDocument(); mx::readFromXmlFile(doc, "resources/Materials/TestSuite/stdlib/definition/definition_from_nodegraph.mtlx", searchPath); - doc->importLibrary(stdlib); + doc->registerDataLibrary(stdlib); mx::NodeGraphPtr graph = doc->getNodeGraph("test_colorcorrect"); REQUIRE(graph); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp index 22afffbff4..9ec90478a1 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShader.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShader.cpp @@ -161,7 +161,7 @@ TEST_CASE("GenShader: Transparency Regression Check", "[genshader]") bool testValue = transparencyTest[i]; mx::DocumentPtr testDoc = mx::createDocument(); - testDoc->importLibrary(libraries); + testDoc->registerDataLibrary(libraries); try { @@ -207,7 +207,7 @@ void testDeterministicGeneration(mx::DocumentPtr libraries, mx::GenContext& cont { mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->registerDataLibrary(libraries); // Keep the document alive to make sure // new memory is allocated for each run @@ -272,7 +272,7 @@ void checkPixelDependencies(mx::DocumentPtr libraries, mx::GenContext& context) mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlFile(testDoc, testFile); - testDoc->importLibrary(libraries); + testDoc->registerDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); @@ -385,7 +385,7 @@ TEST_CASE("GenShader: Track Application Variables", "[genshader]") mx::DocumentPtr testDoc = mx::createDocument(); mx::readFromXmlString(testDoc, testDocumentString); - testDoc->importLibrary(libraries); + testDoc->registerDataLibrary(libraries); mx::ElementPtr element = testDoc->getChild(testElement); CHECK(element); diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index 6dc24fc15c..acc97a4815 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -372,7 +372,7 @@ void shaderGenPerformanceTest(mx::GenContext& context) std::shuffle(loadedDocuments.begin(), loadedDocuments.end(), rng); for (const auto& doc : loadedDocuments) { - doc->importLibrary(nodeLibrary); + doc->registerDataLibrary(nodeLibrary); std::vector elements = mx::findRenderableElements(doc); REQUIRE(elements.size() > 0); @@ -721,6 +721,8 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons bool importedLibrary = false; try { + //TODO: Enable setDataLibrary and ensures all implementations are accounted for. + // doc->setDataLibrary(_dependLib); doc->importLibrary(_dependLib); importedLibrary = true; } diff --git a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp index d8265a4050..e84bd6eaac 100644 --- a/source/MaterialXTest/MaterialXRender/RenderUtil.cpp +++ b/source/MaterialXTest/MaterialXRender/RenderUtil.cpp @@ -223,7 +223,7 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath) // colliding with implementations in previous test cases. context.clearNodeImplementations(); - doc->importLibrary(dependLib); + doc->registerDataLibrary(dependLib); ioTimer.endTimer(); validateTimer.startTimer(); diff --git a/source/MaterialXView/Viewer.cpp b/source/MaterialXView/Viewer.cpp index c0d90d30a7..5496633c97 100644 --- a/source/MaterialXView/Viewer.cpp +++ b/source/MaterialXView/Viewer.cpp @@ -1317,7 +1317,7 @@ void Viewer::loadDocument(const mx::FilePath& filename, mx::DocumentPtr librarie _materialSearchPath = mx::getSourceSearchPath(doc); // Import libraries. - doc->importLibrary(libraries); + doc->registerDataLibrary(libraries); // Apply direct lights. applyDirectLights(doc); diff --git a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp index 3f4403c673..64fe721097 100644 --- a/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp +++ b/source/PyMaterialX/PyMaterialXCore/PyDocument.cpp @@ -30,6 +30,9 @@ void bindPyDocument(py::module& mod) .def("initialize", &mx::Document::initialize) .def("copy", &mx::Document::copy) .def("importLibrary", &mx::Document::importLibrary) + .def("setDataLibrary", &mx::Document::registerDataLibrary) + .def("getDataLibrary", &mx::Document::getRegisteredDataLibrary) + .def("hasDataLibrary", &mx::Document::hasDataLibrary) .def("getReferencedSourceUris", &mx::Document::getReferencedSourceUris) .def("addNodeGraph", &mx::Document::addNodeGraph, py::arg("name") = mx::EMPTY_STRING)