diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9067ab0..6fbafb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: submodules: true - name: Configure - run: cmake -B ${{github.workspace}}/build "-DCMAKE_SYSTEM_VERSION=10.0.19041.0" -DNVRHI_WITH_RTXMU=${{matrix.rtxmu}} -DNVRHI_BUILD_SHARED=ON -DNVRHI_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install + run: cmake -B ${{github.workspace}}/build "-DCMAKE_SYSTEM_VERSION=10.0.22621.0" -DNVRHI_WITH_RTXMU=${{matrix.rtxmu}} -DNVRHI_BUILD_SHARED=ON -DNVRHI_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install - name: Build run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} diff --git a/README.md b/README.md index 30556e4..9fc669e 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Various early versions of NVRHI have been used in various projects created at NV * Windows or Linux (x64 or ARM64) * CMake 3.10 * A C++ 17 compiler (Visual Studio 2019, GCC 8 or Clang 6) -* Windows SDK version 10.0.19041.0 or later for DX12 support +* Windows SDK version 10.0.22621.0 or later for DX12 support ## Building NVRHI diff --git a/include/nvrhi/d3d12.h b/include/nvrhi/d3d12.h index 688f897..be34d14 100644 --- a/include/nvrhi/d3d12.h +++ b/include/nvrhi/d3d12.h @@ -56,6 +56,10 @@ namespace nvrhi::d3d12 virtual void updateGraphicsVolatileBuffers() = 0; virtual void updateComputeVolatileBuffers() = 0; + + virtual void clearSamplerFeedbackTexture(ISamplerFeedbackTexture* texture) = 0; + virtual void decodeSamplerFeedbackTexture(IBuffer* buffer, ISamplerFeedbackTexture* texture, nvrhi::Format format) = 0; + virtual void setSamplerFeedbackTextureState(ISamplerFeedbackTexture* texture, ResourceStates stateBits) = 0; }; typedef RefCountPtr CommandListHandle; @@ -100,6 +104,8 @@ namespace nvrhi::d3d12 virtual GraphicsPipelineHandle createHandleForNativeGraphicsPipeline(IRootSignature* rootSignature, ID3D12PipelineState* pipelineState, const GraphicsPipelineDesc& desc, const FramebufferInfo& framebufferInfo) = 0; virtual MeshletPipelineHandle createHandleForNativeMeshletPipeline(IRootSignature* rootSignature, ID3D12PipelineState* pipelineState, const MeshletPipelineDesc& desc, const FramebufferInfo& framebufferInfo) = 0; [[nodiscard]] virtual IDescriptorHeap* getDescriptorHeap(DescriptorHeapType heapType) = 0; + virtual SamplerFeedbackTextureHandle createSamplerFeedbackTexture(ITexture* pairedTexture, const SamplerFeedbackTextureDesc& desc) = 0; + virtual SamplerFeedbackTextureHandle createSamplerFeedbackForNativeTexture(ObjectType objectType, Object texture, ITexture* pairedTexture) = 0; }; typedef RefCountPtr DeviceHandle; diff --git a/include/nvrhi/nvrhi.h b/include/nvrhi/nvrhi.h index 5b40531..8fd9214 100644 --- a/include/nvrhi/nvrhi.h +++ b/include/nvrhi/nvrhi.h @@ -421,6 +421,7 @@ namespace nvrhi // and memory is bound to the texture later using bindTextureMemory. // On DX12, the texture resource is created at the time of memory binding. bool isVirtual = false; + bool isTiled = false; Color clearValue; bool useClearValue = false; @@ -541,6 +542,79 @@ namespace nvrhi }; typedef RefCountPtr StagingTextureHandle; + struct TiledTextureCoordinate + { + uint16_t mipLevel = 0; + uint16_t arrayLevel = 0; + uint32_t x = 0; + uint32_t y = 0; + uint32_t z = 0; + }; + + struct TiledTextureRegion + { + uint32_t tilesNum = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t depth = 0; + }; + + struct TextureTilesMapping + { + TiledTextureCoordinate* tiledTextureCoordinates = nullptr; + TiledTextureRegion* tiledTextureRegions = nullptr; + uint64_t* byteOffsets = nullptr; + uint32_t numTextureRegions = 0; + IHeap* heap = nullptr; + }; + + struct PackedMipDesc + { + uint32_t numStandardMips = 0; + uint32_t numPackedMips = 0; + uint32_t numTilesForPackedMips = 0; + uint32_t startTileIndexInOverallResource = 0; + }; + + struct TileShape + { + uint32_t widthInTexels = 0; + uint32_t heightInTexels = 0; + uint32_t depthInTexels = 0; + }; + + struct SubresourceTiling + { + uint32_t widthInTiles = 0; + uint32_t heightInTiles = 0; + uint32_t depthInTiles = 0; + uint32_t startTileIndexInOverallResource = 0; + }; + + enum SamplerFeedbackFormat : uint8_t + { + MinMipOpaque = 0x0, + MipRegionUsedOpaque = 0x1, + }; + + struct SamplerFeedbackTextureDesc + { + SamplerFeedbackFormat samplerFeedbackFormat = SamplerFeedbackFormat::MinMipOpaque; + uint32_t samplerFeedbackMipRegionX = 0; + uint32_t samplerFeedbackMipRegionY = 0; + uint32_t samplerFeedbackMipRegionZ = 0; + ResourceStates initialState = ResourceStates::Unknown; + bool keepInitialState = false; + }; + + class ISamplerFeedbackTexture : public IResource + { + public: + [[nodiscard]] virtual const SamplerFeedbackTextureDesc& getDesc() const = 0; + virtual TextureHandle getPairedTexture() = 0; + }; + typedef RefCountPtr SamplerFeedbackTextureHandle; + ////////////////////////////////////////////////////////////////////////// // Input Layout ////////////////////////////////////////////////////////////////////////// @@ -600,7 +674,7 @@ namespace nvrhi bool isVolatile = false; // Indicates that the buffer is created with no backing memory, - // and memory is bound to the texture later using bindBufferMemory. + // and memory is bound to the buffer later using bindBufferMemory. // On DX12, the buffer resource is created at the time of memory binding. bool isVirtual = false; @@ -1560,6 +1634,7 @@ namespace nvrhi Sampler, RayTracingAccelStruct, PushConstants, + SamplerFeedbackTexture_UAV, Count }; @@ -1601,6 +1676,7 @@ namespace nvrhi NVRHI_BINDING_LAYOUT_ITEM_INITIALIZER(VolatileConstantBuffer) NVRHI_BINDING_LAYOUT_ITEM_INITIALIZER(Sampler) NVRHI_BINDING_LAYOUT_ITEM_INITIALIZER(RayTracingAccelStruct) + NVRHI_BINDING_LAYOUT_ITEM_INITIALIZER(SamplerFeedbackTexture_UAV) static BindingLayoutItem PushConstants(const uint32_t slot, const size_t size) { @@ -1918,6 +1994,19 @@ namespace nvrhi return result; } + static BindingSetItem SamplerFeedbackTexture_UAV(uint32_t slot, ISamplerFeedbackTexture* texture) + { + BindingSetItem result; + result.slot = slot; + result.type = ResourceType::SamplerFeedbackTexture_UAV; + result.resourceHandle = texture; + result.format = Format::UNKNOWN; + result.dimension = TextureDimension::Unknown; + result.subresources = AllSubresources; + result.unused = 0; + return result; + } + BindingSetItem& setFormat(Format value) { format = value; return *this; } BindingSetItem& setDimension(TextureDimension value) { dimension = value; return *this; } BindingSetItem& setSubresources(TextureSubresourceSet value) { subresources = value; return *this; } @@ -2652,6 +2741,9 @@ namespace nvrhi virtual void *mapStagingTexture(IStagingTexture* tex, const TextureSlice& slice, CpuAccessMode cpuAccess, size_t *outRowPitch) = 0; virtual void unmapStagingTexture(IStagingTexture* tex) = 0; + virtual void getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) = 0; + virtual void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue = CommandQueue::Graphics) = 0; + virtual BufferHandle createBuffer(const BufferDesc& d) = 0; virtual void *mapBuffer(IBuffer* buffer, CpuAccessMode cpuAccess) = 0; virtual void unmapBuffer(IBuffer* buffer) = 0; diff --git a/src/common/state-tracking.h b/src/common/state-tracking.h index 694bbf1..5c8d118 100644 --- a/src/common/state-tracking.h +++ b/src/common/state-tracking.h @@ -43,6 +43,7 @@ namespace nvrhi const TextureDesc& descRef; ResourceStates permanentState = ResourceStates::Unknown; bool stateInitialized = false; + bool isSamplerFeedback = false; explicit TextureStateExtension(const TextureDesc& desc) : descRef(desc) diff --git a/src/d3d11/d3d11-backend.h b/src/d3d11/d3d11-backend.h index e32e03a..33dd665 100644 --- a/src/d3d11/d3d11-backend.h +++ b/src/d3d11/d3d11-backend.h @@ -442,6 +442,9 @@ namespace nvrhi::d3d11 void *mapStagingTexture(IStagingTexture* tex, const TextureSlice& slice, CpuAccessMode cpuAccess, size_t *outRowPitch) override; void unmapStagingTexture(IStagingTexture* tex) override; + void getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) override; + void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue = CommandQueue::Graphics) override; + BufferHandle createBuffer(const BufferDesc& d) override; void *mapBuffer(IBuffer* b, CpuAccessMode mapFlags) override; void unmapBuffer(IBuffer* b) override; diff --git a/src/d3d11/d3d11-device.cpp b/src/d3d11/d3d11-device.cpp index 4cfc195..1a6f7ff 100644 --- a/src/d3d11/d3d11-device.cpp +++ b/src/d3d11/d3d11-device.cpp @@ -172,6 +172,28 @@ namespace nvrhi::d3d11 return m_ImmediateCommandList; } + void Device::getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) + { + (void)texture; + (void)numTiles; + (void)desc; + (void)tileShape; + (void)subresourceTilingsNum; + (void)subresourceTilings; + + utils::NotSupported(); + } + + void Device::updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue) + { + (void)texture; + (void)tileMappings; + (void)numTileMappings; + (void)executionQueue; + + utils::NotSupported(); + } + bool Device::queryFeatureSupport(Feature feature, void* pInfo, size_t infoSize) { (void)pInfo; diff --git a/src/d3d12/d3d12-backend.h b/src/d3d12/d3d12-backend.h index 9acc9ef..368eb34 100644 --- a/src/d3d12/d3d12-backend.h +++ b/src/d3d12/d3d12-backend.h @@ -106,6 +106,7 @@ namespace nvrhi::d3d12 RefCountPtr device; RefCountPtr device2; RefCountPtr device5; + RefCountPtr device8; #ifdef NVRHI_WITH_RTXMU std::unique_ptr rtxMemUtil; #endif @@ -243,6 +244,7 @@ namespace nvrhi::d3d12 HANDLE sharedHandle = nullptr; HeapHandle heap; + Texture(const Context& context, DeviceResources& resources, TextureDesc desc, const D3D12_RESOURCE_DESC& resourceDesc) : TextureStateExtension(this->desc) , desc(std::move(desc)) @@ -254,7 +256,7 @@ namespace nvrhi::d3d12 } ~Texture() override; - + const TextureDesc& getDesc() const override { return desc; } Object getNativeObject(ObjectType objectType) override; @@ -356,6 +358,38 @@ namespace nvrhi::d3d12 Object getNativeObject(ObjectType objectType) override; }; + class SamplerFeedbackTexture : public RefCounter, public TextureStateExtension + { + public: + const SamplerFeedbackTextureDesc desc; + const TextureDesc textureDesc; // used with state tracking + RefCountPtr resource; + TextureHandle pairedTexture; + + SamplerFeedbackTexture(const Context& context, DeviceResources& resources, SamplerFeedbackTextureDesc desc, TextureDesc textureDesc, ITexture* pairedTexture) + : desc(std::move(desc)) + , textureDesc(std::move(textureDesc)) + , m_Context(context) + , m_Resources(resources) + , pairedTexture(pairedTexture) + , TextureStateExtension(SamplerFeedbackTexture::textureDesc) + { + TextureStateExtension::stateInitialized = true; + TextureStateExtension::isSamplerFeedback = true; + } + + const SamplerFeedbackTextureDesc& getDesc() const override { return desc; } + TextureHandle getPairedTexture() override { return pairedTexture; } + + void createUAV(size_t descriptor) const; + + Object getNativeObject(ObjectType objectType) override; + + private: + const Context& m_Context; + DeviceResources& m_Resources; + }; + class Sampler : public RefCounter { public: @@ -868,6 +902,7 @@ namespace nvrhi::d3d12 ~CommandList() override; std::shared_ptr executed(Queue* pQueue); void requireTextureState(ITexture* texture, TextureSubresourceSet subresources, ResourceStates state); + void requireSamplerFeedbackTextureState(ISamplerFeedbackTexture* texture, ResourceStates state); void requireBufferState(IBuffer* buffer, ResourceStates state); ID3D12CommandList* getD3D12CommandList() const { return m_ActiveCommandList->commandList; } @@ -884,6 +919,9 @@ namespace nvrhi::d3d12 void clearTextureFloat(ITexture* t, TextureSubresourceSet subresources, const Color& clearColor) override; void clearDepthStencilTexture(ITexture* t, TextureSubresourceSet subresources, bool clearDepth, float depth, bool clearStencil, uint8_t stencil) override; void clearTextureUInt(ITexture* t, TextureSubresourceSet subresources, uint32_t clearColor) override; + void clearSamplerFeedbackTexture(ISamplerFeedbackTexture* texture) override; + void decodeSamplerFeedbackTexture(IBuffer* buffer, ISamplerFeedbackTexture* texture, Format format) override; + void setSamplerFeedbackTextureState(ISamplerFeedbackTexture* texture, ResourceStates stateBits) override; void copyTexture(ITexture* dest, const TextureSlice& destSlice, ITexture* src, const TextureSlice& srcSlice) override; void copyTexture(IStagingTexture* dest, const TextureSlice& destSlice, ITexture* src, const TextureSlice& srcSlice) override; @@ -1058,6 +1096,12 @@ namespace nvrhi::d3d12 void *mapStagingTexture(IStagingTexture* tex, const TextureSlice& slice, CpuAccessMode cpuAccess, size_t *outRowPitch) override; void unmapStagingTexture(IStagingTexture* tex) override; + void getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) override; + void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue = CommandQueue::Graphics) override; + + SamplerFeedbackTextureHandle createSamplerFeedbackTexture(ITexture* pairedTexture, const SamplerFeedbackTextureDesc& desc) override; + SamplerFeedbackTextureHandle createSamplerFeedbackForNativeTexture(ObjectType objectType, Object texture, ITexture* pairedTexture) override; + BufferHandle createBuffer(const BufferDesc& d) override; void *mapBuffer(IBuffer* b, CpuAccessMode mapFlags) override; void unmapBuffer(IBuffer* b) override; @@ -1160,6 +1204,7 @@ namespace nvrhi::d3d12 bool m_VariableRateShadingSupported = false; bool m_OpacityMicromapSupported = false; bool m_ShaderExecutionReorderingSupported = false; + bool m_SamplerFeedbackSupported = false; bool m_AftermathEnabled = false; AftermathCrashDumpHelper m_AftermathCrashDumpHelper; diff --git a/src/d3d12/d3d12-device.cpp b/src/d3d12/d3d12-device.cpp index 98c0291..0a08f7d 100644 --- a/src/d3d12/d3d12-device.cpp +++ b/src/d3d12/d3d12-device.cpp @@ -128,6 +128,11 @@ namespace nvrhi::d3d12 m_MeshletsSupported = m_Options7.MeshShaderTier >= D3D12_MESH_SHADER_TIER_1; } + if (SUCCEEDED(m_Context.device->QueryInterface(&m_Context.device8)) && hasOptions7) + { + m_SamplerFeedbackSupported = m_Options7.SamplerFeedbackTier >= D3D12_SAMPLER_FEEDBACK_TIER_0_9; + } + if (hasOptions6) { m_VariableRateShadingSupported = m_Options6.VariableShadingRateTier >= D3D12_VARIABLE_SHADING_RATE_TIER_2; @@ -328,6 +333,8 @@ namespace nvrhi::d3d12 return Object(m_Context.device); case ObjectTypes::Nvrhi_D3D12_Device: return Object(this); + case ObjectTypes::D3D12_CommandQueue: + return Object(getQueue(CommandQueue::Graphics)->queue.Get()); default: return nullptr; } @@ -379,6 +386,101 @@ namespace nvrhi::d3d12 pWaitQueue->queue->Wait(pExecutionQueue->fence, instanceID); } + void Device::getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* _subresourceTilings) + { + ID3D12Resource* resource = checked_cast(texture)->resource; + D3D12_RESOURCE_DESC resourceDesc = resource->GetDesc(); + + D3D12_PACKED_MIP_INFO packedMipDesc = {}; + D3D12_TILE_SHAPE standardTileShapeForNonPackedMips = {}; + D3D12_SUBRESOURCE_TILING subresourceTilings[16]; + + m_Context.device->GetResourceTiling(resource, numTiles, desc ? &packedMipDesc : nullptr, tileShape ? &standardTileShapeForNonPackedMips : nullptr, subresourceTilingsNum, 0, subresourceTilings); + + if (desc) + { + desc->numStandardMips = packedMipDesc.NumStandardMips; + desc->numPackedMips = packedMipDesc.NumPackedMips; + desc->startTileIndexInOverallResource = packedMipDesc.StartTileIndexInOverallResource; + desc->numTilesForPackedMips = packedMipDesc.NumTilesForPackedMips; + } + + if (tileShape) + { + tileShape->widthInTexels = standardTileShapeForNonPackedMips.WidthInTexels; + tileShape->heightInTexels = standardTileShapeForNonPackedMips.HeightInTexels; + tileShape->depthInTexels = standardTileShapeForNonPackedMips.DepthInTexels; + } + + for (uint32_t i = 0; i < *subresourceTilingsNum; ++i) + { + _subresourceTilings[i].widthInTiles = subresourceTilings[i].WidthInTiles; + _subresourceTilings[i].heightInTiles = subresourceTilings[i].HeightInTiles; + _subresourceTilings[i].depthInTiles = subresourceTilings[i].DepthInTiles; + _subresourceTilings[i].startTileIndexInOverallResource = subresourceTilings[i].StartTileIndexInOverallResource; + } + } + + void Device::updateTextureTileMappings(ITexture* _texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue) + { + Queue* queue = getQueue(executionQueue); + Texture* texture = checked_cast(_texture); + + D3D12_TILE_SHAPE tileShape; + D3D12_SUBRESOURCE_TILING subresourceTiling; + m_Context.device->GetResourceTiling(texture->resource, nullptr, nullptr, &tileShape, nullptr, 0, &subresourceTiling); + + for (size_t i = 0; i < numTileMappings; i++) + { + ID3D12Heap* heap = tileMappings[i].heap ? checked_cast(tileMappings[i].heap)->heap : nullptr; + + uint32_t numRegions = tileMappings[i].numTextureRegions; + std::vector resourceCoordinates(numRegions); + std::vector regionSizes(numRegions); + std::vector rangeFlags(numRegions, heap ? D3D12_TILE_RANGE_FLAG_NONE : D3D12_TILE_RANGE_FLAG_NULL); + std::vector heapStartOffsets(numRegions); + std::vector rangeTileCounts(numRegions); + + for (uint32_t j = 0; j < numRegions; ++j) + { + const TiledTextureCoordinate& tiledTextureCoordinate = tileMappings[i].tiledTextureCoordinates[j]; + const TiledTextureRegion& tiledTextureRegion = tileMappings[i].tiledTextureRegions[j]; + + resourceCoordinates[j].Subresource = tiledTextureCoordinate.mipLevel * texture->desc.arraySize + tiledTextureCoordinate.arrayLevel; + resourceCoordinates[j].X = tiledTextureCoordinate.x; + resourceCoordinates[j].Y = tiledTextureCoordinate.y; + resourceCoordinates[j].Z = tiledTextureCoordinate.z; + + if (tiledTextureRegion.tilesNum) + { + regionSizes[j].NumTiles = tiledTextureRegion.tilesNum; + regionSizes[j].UseBox = false; + } + else + { + uint32_t tilesX = (tiledTextureRegion.width + (tileShape.WidthInTexels - 1)) / tileShape.WidthInTexels; + uint32_t tilesY = (tiledTextureRegion.height + (tileShape.HeightInTexels - 1)) / tileShape.HeightInTexels; + uint32_t tilesZ = (tiledTextureRegion.depth + (tileShape.DepthInTexels - 1)) / tileShape.DepthInTexels; + + regionSizes[j].Width = tilesX; + regionSizes[j].Height = (uint16_t)tilesY; + regionSizes[j].Depth = (uint16_t)tilesZ; + + regionSizes[j].NumTiles = tilesX * tilesY * tilesZ; + regionSizes[j].UseBox = true; + } + + // Offset in tiles + if (heap) + heapStartOffsets[j] = (uint32_t)(tileMappings[i].byteOffsets[j] / D3D12_TILED_RESOURCE_TILE_SIZE_IN_BYTES); + + rangeTileCounts[j] = regionSizes[j].NumTiles; + } + + queue->queue->UpdateTileMappings(texture->resource, tileMappings[i].numTextureRegions, resourceCoordinates.data(), regionSizes.data(), heap, numRegions, rangeFlags.data(), heap ? heapStartOffsets.data() : nullptr, rangeTileCounts.data(), D3D12_TILE_MAPPING_FLAG_NONE); + } + } + void Device::runGarbageCollection() { for (const auto& pQueue : m_Queues) diff --git a/src/d3d12/d3d12-resource-bindings.cpp b/src/d3d12/d3d12-resource-bindings.cpp index 055863a..579d8fd 100644 --- a/src/d3d12/d3d12-resource-bindings.cpp +++ b/src/d3d12/d3d12-resource-bindings.cpp @@ -285,6 +285,19 @@ namespace nvrhi::d3d12 found = true; break; } + else if (range.RangeType == D3D12_DESCRIPTOR_RANGE_TYPE_UAV && bindingType == ResourceType::SamplerFeedbackTexture_UAV) + { + SamplerFeedbackTexture* texture = checked_cast(binding.resourceHandle); + + texture->createUAV(descriptorHandle.ptr); + pResource = texture; + + // TODO: Automatic state transition into Unordered Access here + + hasUavBindings = true; + found = true; + break; + } } if (pResource) @@ -440,6 +453,7 @@ namespace nvrhi::d3d12 case ResourceType::TypedBuffer_UAV: case ResourceType::StructuredBuffer_UAV: case ResourceType::RawBuffer_UAV: + case ResourceType::SamplerFeedbackTexture_UAV: range.RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; break; @@ -582,6 +596,7 @@ namespace nvrhi::d3d12 case ResourceType::TypedBuffer_UAV: case ResourceType::StructuredBuffer_UAV: case ResourceType::RawBuffer_UAV: + case ResourceType::SamplerFeedbackTexture_UAV: rangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; break; @@ -770,6 +785,11 @@ namespace nvrhi::d3d12 texture->createUAV(descriptorHandle.ptr, binding.format, binding.dimension, binding.subresources); break; } + case ResourceType::SamplerFeedbackTexture_UAV: { + SamplerFeedbackTexture* texture = checked_cast(binding.resourceHandle); + texture->createUAV(descriptorHandle.ptr); + break; + } case ResourceType::TypedBuffer_SRV: case ResourceType::StructuredBuffer_SRV: case ResourceType::RawBuffer_SRV: { diff --git a/src/d3d12/d3d12-state-tracking.cpp b/src/d3d12/d3d12-state-tracking.cpp index 7cf7ebf..3d30eed 100644 --- a/src/d3d12/d3d12-state-tracking.cpp +++ b/src/d3d12/d3d12-state-tracking.cpp @@ -81,6 +81,13 @@ namespace nvrhi::d3d12 m_StateTracker.requireTextureState(texture, subresources, state); } + + void CommandList::requireSamplerFeedbackTextureState(ISamplerFeedbackTexture* _texture, ResourceStates state) + { + SamplerFeedbackTexture* texture = checked_cast(_texture); + + m_StateTracker.requireTextureState(texture, AllSubresources, state); + } void CommandList::requireBufferState(IBuffer* _buffer, ResourceStates state) { @@ -106,7 +113,18 @@ namespace nvrhi::d3d12 // Convert the texture barriers into D3D equivalents for (const auto& barrier : textureBarriers) { - const Texture* texture = static_cast(barrier.texture); + const Texture* texture = nullptr; + ID3D12Resource* resource = nullptr; + + if (barrier.texture->isSamplerFeedback) + { + resource = static_cast(barrier.texture)->resource; + } + else + { + texture = static_cast(barrier.texture); + resource = texture->resource; + } D3D12_RESOURCE_BARRIER d3dbarrier{}; const D3D12_RESOURCE_STATES stateBefore = convertResourceStates(barrier.stateBefore); @@ -116,7 +134,7 @@ namespace nvrhi::d3d12 d3dbarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; d3dbarrier.Transition.StateBefore = stateBefore; d3dbarrier.Transition.StateAfter = stateAfter; - d3dbarrier.Transition.pResource = texture->resource; + d3dbarrier.Transition.pResource = resource; if (barrier.entireTexture) { d3dbarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; @@ -134,7 +152,7 @@ namespace nvrhi::d3d12 else if (stateAfter & D3D12_RESOURCE_STATE_UNORDERED_ACCESS) { d3dbarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV; - d3dbarrier.UAV.pResource = texture->resource; + d3dbarrier.UAV.pResource = resource; m_D3DBarriers.push_back(d3dbarrier); } } diff --git a/src/d3d12/d3d12-texture.cpp b/src/d3d12/d3d12-texture.cpp index 1672ba5..93a5543 100644 --- a/src/d3d12/d3d12-texture.cpp +++ b/src/d3d12/d3d12-texture.cpp @@ -217,6 +217,17 @@ namespace nvrhi::d3d12 } } + Object SamplerFeedbackTexture::getNativeObject(ObjectType objectType) + { + switch (objectType) + { + case ObjectTypes::D3D12_Resource: + return Object(resource); + default: + return nullptr; + } + } + static D3D12_RESOURCE_DESC convertTextureDesc(const TextureDesc& d) { const auto& formatMapping = getDxgiFormatMapping(d.format); @@ -311,26 +322,42 @@ namespace nvrhi::d3d12 heapFlags |= D3D12_HEAP_FLAG_SHARED_CROSS_ADAPTER; isShared = true; } + if (d.isTiled) + { + rd.Layout = D3D12_TEXTURE_LAYOUT_64KB_UNDEFINED_SWIZZLE; + } Texture* texture = new Texture(m_Context, m_Resources, d, rd); + D3D12_CLEAR_VALUE clearValue = convertTextureClearValue(d); + HRESULT hr = S_OK; + if (d.isVirtual) { // The resource is created in bindTextureMemory return TextureHandle::Create(texture); } - heapProps.Type = D3D12_HEAP_TYPE_DEFAULT; - - D3D12_CLEAR_VALUE clearValue = convertTextureClearValue(d); - - HRESULT hr = m_Context.device->CreateCommittedResource( - &heapProps, - heapFlags, - &texture->resourceDesc, - convertResourceStates(d.initialState), - d.useClearValue ? &clearValue : nullptr, - IID_PPV_ARGS(&texture->resource)); + if (d.isTiled) + { + hr = m_Context.device->CreateReservedResource( + &texture->resourceDesc, + convertResourceStates(d.initialState), + d.useClearValue ? &clearValue : nullptr, + IID_PPV_ARGS(&texture->resource)); + } + else + { + heapProps.Type = D3D12_HEAP_TYPE_DEFAULT; + + hr = m_Context.device->CreateCommittedResource( + &heapProps, + heapFlags, + &texture->resourceDesc, + convertResourceStates(d.initialState), + d.useClearValue ? &clearValue : nullptr, + IID_PPV_ARGS(&texture->resource)); + } if (FAILED(hr)) { @@ -344,7 +371,7 @@ namespace nvrhi::d3d12 return nullptr; } - if(isShared) + if (isShared) { hr = m_Context.device->CreateSharedHandle( texture->resource, @@ -851,6 +878,96 @@ namespace nvrhi::d3d12 tex->mappedAccess = CpuAccessMode::None; } + SamplerFeedbackTextureHandle Device::createSamplerFeedbackTexture(ITexture* pairedTexture, const SamplerFeedbackTextureDesc& desc) + { + Texture* texPair = checked_cast(pairedTexture); + TextureDesc descPair = texPair->desc; + D3D12_RESOURCE_DESC rdPair = texPair->resourceDesc; + + D3D12_RESOURCE_DESC1 rdFeedback = {}; + memcpy(&rdFeedback, &rdPair, sizeof(D3D12_RESOURCE_DESC)); + D3D12_HEAP_PROPERTIES heapPropsDefault = {}; + heapPropsDefault.Type = D3D12_HEAP_TYPE_DEFAULT; + rdFeedback.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN; + rdFeedback.Flags = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; + switch (desc.samplerFeedbackFormat) + { + case SamplerFeedbackFormat::MinMipOpaque: + rdFeedback.Format = DXGI_FORMAT_SAMPLER_FEEDBACK_MIN_MIP_OPAQUE; + break; + case SamplerFeedbackFormat::MipRegionUsedOpaque: + rdFeedback.Format = DXGI_FORMAT_SAMPLER_FEEDBACK_MIP_REGION_USED_OPAQUE; + break; + } + rdFeedback.SamplerFeedbackMipRegion = D3D12_MIP_REGION{ desc.samplerFeedbackMipRegionX, desc.samplerFeedbackMipRegionY, desc.samplerFeedbackMipRegionZ }; + + TextureDesc textureDesc = pairedTexture->getDesc(); + textureDesc.initialState = desc.initialState; + textureDesc.keepInitialState = desc.keepInitialState; + + SamplerFeedbackTexture* texture = new SamplerFeedbackTexture(m_Context, m_Resources, desc, textureDesc, pairedTexture); + + HRESULT hr = m_Context.device8->CreateCommittedResource2( + &heapPropsDefault, + D3D12_HEAP_FLAG_NONE, + &rdFeedback, + convertResourceStates(desc.initialState), + nullptr, // clear value + nullptr, + IID_PPV_ARGS(&texture->resource)); + + if (FAILED(hr)) + { + std::stringstream ss; + ss << "Failed to create texture " << utils::DebugNameToString(descPair.debugName) << ", error code = 0x"; + ss.setf(std::ios::hex, std::ios::basefield); + ss << hr; + m_Context.error(ss.str()); + + delete texture; + return nullptr; + } + + std::wstringstream ssName; + ssName << "Sampler Feedback Texture: " << utils::DebugNameToString(descPair.debugName); + texture->resource->SetName(ssName.str().c_str()); + + return SamplerFeedbackTextureHandle::Create(texture); + } + + SamplerFeedbackTextureHandle Device::createSamplerFeedbackForNativeTexture(ObjectType objectType, Object _texture, ITexture* pairedTexture) + { + if (_texture.pointer == nullptr) + return nullptr; + + if (objectType != ObjectTypes::D3D12_Resource) + return nullptr; + + ID3D12Resource2* pResource = static_cast(_texture.pointer); + + D3D12_RESOURCE_DESC1 rdFeedback = pResource->GetDesc1(); + SamplerFeedbackTextureDesc desc = {}; + desc.samplerFeedbackFormat = rdFeedback.Format == DXGI_FORMAT_SAMPLER_FEEDBACK_MIN_MIP_OPAQUE ? SamplerFeedbackFormat::MinMipOpaque : SamplerFeedbackFormat::MipRegionUsedOpaque; + desc.samplerFeedbackMipRegionX = rdFeedback.SamplerFeedbackMipRegion.Width; + desc.samplerFeedbackMipRegionY = rdFeedback.SamplerFeedbackMipRegion.Height; + desc.samplerFeedbackMipRegionZ = rdFeedback.SamplerFeedbackMipRegion.Depth; + + TextureDesc textureDesc = pairedTexture->getDesc(); + textureDesc.initialState = ResourceStates::Unknown; + textureDesc.keepInitialState = false; + + SamplerFeedbackTexture* texture = new SamplerFeedbackTexture(m_Context, m_Resources, desc, textureDesc, pairedTexture); + texture->resource = pResource; + + return SamplerFeedbackTextureHandle::Create(texture); + } + + void SamplerFeedbackTexture::createUAV(size_t descriptor) const + { + ID3D12Resource* pairedResource = checked_cast(pairedTexture.Get())->resource; + + m_Context.device8->CreateSamplerFeedbackUnorderedAccessView(pairedResource, resource, { descriptor }); + } void CommandList::clearTextureFloat(ITexture* _t, TextureSubresourceSet subresources, const Color & clearColor) { @@ -1010,6 +1127,35 @@ namespace nvrhi::d3d12 } } + void CommandList::clearSamplerFeedbackTexture(ISamplerFeedbackTexture* _texture) + { + SamplerFeedbackTexture* texture = checked_cast(_texture); + + m_ActiveCommandList->commandList->DiscardResource(texture->resource, nullptr); + } + + void CommandList::decodeSamplerFeedbackTexture(IBuffer* _buffer, ISamplerFeedbackTexture* _texture, nvrhi::Format format) + { + Buffer* buffer = checked_cast(_buffer); + SamplerFeedbackTexture* texture = checked_cast(_texture); + + if (m_EnableAutomaticBarriers) + { + requireBufferState(buffer, ResourceStates::ResolveDest); + requireSamplerFeedbackTextureState(texture, ResourceStates::ResolveSource); + } + commitBarriers(); + + const DxgiFormatMapping& formatMapping = getDxgiFormatMapping(format); + + m_ActiveCommandList->commandList4->ResolveSubresourceRegion(buffer->resource, 0, 0, 0, texture->resource, 0, nullptr, formatMapping.srvFormat, D3D12_RESOLVE_MODE_DECODE_SAMPLER_FEEDBACK); + } + + void CommandList::setSamplerFeedbackTextureState(ISamplerFeedbackTexture* texture, ResourceStates stateBits) + { + requireSamplerFeedbackTextureState(texture, stateBits); + } + void CommandList::copyTexture(ITexture* _dst, const TextureSlice& dstSlice, ITexture* _src, const TextureSlice& srcSlice) { diff --git a/src/validation/validation-backend.h b/src/validation/validation-backend.h index 52a317f..b38a9e6 100644 --- a/src/validation/validation-backend.h +++ b/src/validation/validation-backend.h @@ -260,6 +260,9 @@ namespace nvrhi::validation void *mapStagingTexture(IStagingTexture* tex, const TextureSlice& slice, CpuAccessMode cpuAccess, size_t *outRowPitch) override; void unmapStagingTexture(IStagingTexture* tex) override; + void getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) override; + void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue = CommandQueue::Graphics) override; + BufferHandle createBuffer(const BufferDesc& d) override; void *mapBuffer(IBuffer* b, CpuAccessMode mapFlags) override; void unmapBuffer(IBuffer* b) override; diff --git a/src/validation/validation-device.cpp b/src/validation/validation-device.cpp index b7c38e3..bf177d6 100644 --- a/src/validation/validation-device.cpp +++ b/src/validation/validation-device.cpp @@ -241,6 +241,16 @@ namespace nvrhi::validation return m_Device->createTexture(patchedDesc); } + void DeviceWrapper::getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) + { + m_Device->getTextureTiling(texture, numTiles, desc, tileShape, subresourceTilingsNum, subresourceTilings); + } + + void DeviceWrapper::updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue) + { + m_Device->updateTextureTileMappings(texture, tileMappings, numTileMappings, executionQueue); + } + MemoryRequirements DeviceWrapper::getTextureMemoryRequirements(ITexture* texture) { if (texture == nullptr) diff --git a/src/vulkan/vulkan-backend.h b/src/vulkan/vulkan-backend.h index f9cf394..a1599e9 100644 --- a/src/vulkan/vulkan-backend.h +++ b/src/vulkan/vulkan-backend.h @@ -251,6 +251,8 @@ namespace nvrhi::vulkan // submits a command buffer to this queue, returns submissionID uint64_t submit(ICommandList* const* ppCmd, size_t numCmd); + void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings); + // retire any command buffers that have finished execution from the pending execution list void retireCommandBuffers(); @@ -400,6 +402,7 @@ namespace nvrhi::vulkan vk::ImageCreateInfo imageInfo; vk::ExternalMemoryImageCreateInfo externalMemoryImageInfo; vk::Image image; + static constexpr uint32_t tileByteSize = 65536; HeapHandle heap; @@ -1076,6 +1079,9 @@ namespace nvrhi::vulkan void *mapStagingTexture(IStagingTexture* tex, const TextureSlice& slice, CpuAccessMode cpuAccess, size_t *outRowPitch) override; void unmapStagingTexture(IStagingTexture* tex) override; + void getTextureTiling(ITexture* texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) override; + void updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue = CommandQueue::Graphics) override; + BufferHandle createBuffer(const BufferDesc& d) override; void *mapBuffer(IBuffer* b, CpuAccessMode mapFlags) override; void unmapBuffer(IBuffer* b) override; diff --git a/src/vulkan/vulkan-device.cpp b/src/vulkan/vulkan-device.cpp index 380343e..fcb661a 100644 --- a/src/vulkan/vulkan-device.cpp +++ b/src/vulkan/vulkan-device.cpp @@ -447,6 +447,88 @@ namespace nvrhi::vulkan return submissionID; } + void Device::getTextureTiling(ITexture* _texture, uint32_t* numTiles, PackedMipDesc* desc, TileShape* tileShape, uint32_t* subresourceTilingsNum, SubresourceTiling* subresourceTilings) + { + Texture* texture = checked_cast(_texture); + uint32_t numStandardMips = 0; + uint32_t tileWidth = 1; + uint32_t tileHeight = 1; + uint32_t tileDepth = 1; + + { + auto memoryRequirements = m_Context.device.getImageSparseMemoryRequirements(texture->image); + if (!memoryRequirements.empty()) + { + numStandardMips = memoryRequirements[0].imageMipTailFirstLod; + } + + if (desc) + { + desc->numStandardMips = numStandardMips; + desc->numPackedMips = texture->imageInfo.mipLevels - memoryRequirements[0].imageMipTailFirstLod; + desc->startTileIndexInOverallResource = (uint32_t)(memoryRequirements[0].imageMipTailOffset / texture->tileByteSize); + desc->numTilesForPackedMips = (uint32_t)(memoryRequirements[0].imageMipTailSize / texture->tileByteSize); + } + } + + { + auto formatProperties = m_Context.physicalDevice.getSparseImageFormatProperties(texture->imageInfo.format, texture->imageInfo.imageType, texture->imageInfo.samples, texture->imageInfo.usage, texture->imageInfo.tiling); + if (!formatProperties.empty()) + { + tileWidth = formatProperties[0].imageGranularity.width; + tileHeight = formatProperties[0].imageGranularity.height; + tileDepth = formatProperties[0].imageGranularity.depth; + } + + if (tileShape) + { + tileShape->widthInTexels = tileWidth; + tileShape->heightInTexels = tileHeight; + tileShape->depthInTexels = tileDepth; + } + } + + if (subresourceTilingsNum) + { + *subresourceTilingsNum = std::min(*subresourceTilingsNum, texture->desc.mipLevels); + uint32_t startTileIndexInOverallResource = 0; + + uint32_t width = texture->desc.width; + uint32_t height = texture->desc.height; + uint32_t depth = texture->desc.depth; + + for (uint32_t i = 0; i < *subresourceTilingsNum; ++i) + { + if (i < numStandardMips) + { + subresourceTilings[i].widthInTiles = (width + tileWidth - 1) / tileWidth; + subresourceTilings[i].heightInTiles = (height + tileHeight - 1) / tileHeight; + subresourceTilings[i].depthInTiles = (depth + tileDepth - 1) / tileDepth; + subresourceTilings[i].startTileIndexInOverallResource = startTileIndexInOverallResource; + } + else + { + subresourceTilings[i].widthInTiles = 0; + subresourceTilings[i].heightInTiles = 0; + subresourceTilings[i].depthInTiles = 0; + subresourceTilings[i].startTileIndexInOverallResource = UINT32_MAX; + } + + width = std::max(width / 2, tileWidth); + height = std::max(height / 2, tileHeight); + depth = std::max(depth / 2, tileDepth); + + startTileIndexInOverallResource += subresourceTilings[i].widthInTiles * subresourceTilings[i].heightInTiles * subresourceTilings[i].depthInTiles; + } + } + + if (numTiles) + { + auto memoryRequirements = m_Context.device.getImageMemoryRequirements(texture->image); + *numTiles = (uint32_t)(memoryRequirements.size / texture->tileByteSize); + } + } + HeapHandle Device::createHeap(const HeapDesc& d) { vk::MemoryRequirements memoryRequirements; @@ -541,5 +623,4 @@ namespace nvrhi::vulkan { messageCallback->message(MessageSeverity::Warning, message.c_str()); } - } // namespace nvrhi::vulkan diff --git a/src/vulkan/vulkan-queue.cpp b/src/vulkan/vulkan-queue.cpp index b719b41..ebd5552 100644 --- a/src/vulkan/vulkan-queue.cpp +++ b/src/vulkan/vulkan-queue.cpp @@ -183,6 +183,79 @@ namespace nvrhi::vulkan return m_LastSubmittedID; } + void Queue::updateTextureTileMappings(ITexture* _texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings) + { + Texture* texture = checked_cast(_texture); + + std::vector sparseImageMemoryBinds; + std::vector sparseMemoryBinds; + + for (size_t i = 0; i < numTileMappings; i++) + { + uint32_t numRegions = tileMappings[i].numTextureRegions; + Heap* heap = tileMappings[i].heap ? checked_cast(tileMappings[i].heap) : nullptr; + vk::DeviceMemory deviceMemory = heap ? heap->memory : VK_NULL_HANDLE; + + for (uint32_t j = 0; j < numRegions; ++j) + { + const TiledTextureCoordinate& tiledTextureCoordinate = tileMappings[i].tiledTextureCoordinates[j]; + const TiledTextureRegion& tiledTextureRegion = tileMappings[i].tiledTextureRegions[j]; + + if (tiledTextureRegion.tilesNum) + { + sparseMemoryBinds.push_back(vk::SparseMemoryBind() + .setResourceOffset(0) + .setSize(tiledTextureRegion.tilesNum * texture->tileByteSize) + .setMemory(deviceMemory) + .setMemoryOffset(deviceMemory ? tileMappings[i].byteOffsets[j] : 0)); + } + else + { + vk::ImageSubresource subresource = {}; + subresource.arrayLayer = tiledTextureCoordinate.arrayLevel; + subresource.mipLevel = tiledTextureCoordinate.mipLevel; + + vk::Offset3D offset3D; + offset3D.x = tiledTextureCoordinate.x; + offset3D.y = tiledTextureCoordinate.y; + offset3D.z = tiledTextureCoordinate.z; + + vk::Extent3D extent3D; + extent3D.width = tiledTextureRegion.width; + extent3D.height = tiledTextureRegion.height; + extent3D.depth = tiledTextureRegion.depth; + + sparseImageMemoryBinds.push_back(vk::SparseImageMemoryBind() + .setSubresource(subresource) + .setOffset(offset3D) + .setExtent(extent3D) + .setMemory(deviceMemory) + .setMemoryOffset(deviceMemory ? tileMappings[i].byteOffsets[j] : 0)); + } + } + } + + vk::BindSparseInfo bindSparseInfo = {}; + + vk::SparseImageMemoryBindInfo sparseImageMemoryBindInfo; + if (!sparseImageMemoryBinds.empty()) + { + sparseImageMemoryBindInfo.setImage(texture->image); + sparseImageMemoryBindInfo.setBinds(sparseImageMemoryBinds); + bindSparseInfo.setImageBinds(sparseImageMemoryBindInfo); + } + + vk::SparseImageOpaqueMemoryBindInfo sparseImageOpaqueMemoryBindInfo; + if (!sparseMemoryBinds.empty()) + { + sparseImageOpaqueMemoryBindInfo.setImage(texture->image); + sparseImageOpaqueMemoryBindInfo.setBinds(sparseMemoryBinds); + bindSparseInfo.setImageOpaqueBinds(sparseImageOpaqueMemoryBindInfo); + } + + m_Queue.bindSparse(bindSparseInfo, vk::Fence()); + } + uint64_t Queue::updateLastFinishedID() { m_LastFinishedID = m_Context.device.getSemaphoreCounterValue(trackingSemaphore); @@ -266,6 +339,13 @@ namespace nvrhi::vulkan queueWaitForSemaphore(waitQueueID, getQueueSemaphore(executionQueueID), instance); } + void Device::updateTextureTileMappings(ITexture* texture, const TextureTilesMapping* tileMappings, uint32_t numTileMappings, CommandQueue executionQueue) + { + Queue& queue = *m_Queues[uint32_t(executionQueue)]; + + queue.updateTextureTileMappings(texture, tileMappings, numTileMappings); + } + uint64_t Device::queueGetCompletedInstance(CommandQueue queue) { return m_Context.device.getSemaphoreCounterValue(getQueueSemaphore(queue)); diff --git a/src/vulkan/vulkan-texture.cpp b/src/vulkan/vulkan-texture.cpp index bd52773..8fa664e 100644 --- a/src/vulkan/vulkan-texture.cpp +++ b/src/vulkan/vulkan-texture.cpp @@ -212,6 +212,9 @@ namespace nvrhi::vulkan if (d.isTypeless) flags |= vk::ImageCreateFlagBits::eMutableFormat | vk::ImageCreateFlagBits::eExtendedUsage; + if (d.isTiled) + flags |= vk::ImageCreateFlagBits::eSparseBinding | vk::ImageCreateFlagBits::eSparseResidency; + return flags; }