From a11edc2a8f44c0f5c7420edb56bb4b3cf8b0e261 Mon Sep 17 00:00:00 2001 From: xtanion Date: Sat, 10 Dec 2022 11:48:08 +0530 Subject: [PATCH 1/2] standard and advanced material implementation --- .../tutorials/01_introductory/viz_gltf_pbr.py | 41 +++++ fury/gltf.py | 150 +++++++++++++++--- fury/lib.py | 1 + 3 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 docs/tutorials/01_introductory/viz_gltf_pbr.py diff --git a/docs/tutorials/01_introductory/viz_gltf_pbr.py b/docs/tutorials/01_introductory/viz_gltf_pbr.py new file mode 100644 index 000000000..2540b27ac --- /dev/null +++ b/docs/tutorials/01_introductory/viz_gltf_pbr.py @@ -0,0 +1,41 @@ +from fury import window, material, utils +from fury.gltf import glTF +from fury.io import load_cubemap_texture, load_image +from fury.data import fetch_gltf, read_viz_gltf, read_viz_cubemap +from fury.lib import Texture, ImageFlip + +# rgb_array = load_image( +# '/home/shivam/Downloads/skybox.jpg') + +# grid = utils.rgb_to_vtk(rgb_array) +# cubemap = Texture() +# flip = ImageFlip() +# flip.SetInputDataObject(grid) +# flip.SetFilteredAxis(1) +# cubemap.InterpolateOn() +# cubemap.MipmapOn() +# cubemap.SetInputConnection(0, flip.GetOutputPort(0)) +# cubemap.UseSRGBColorSpaceOn() + +scene = window.Scene(skybox=None) +# scene.SetBackground(0.5, 0.3, 0.3) + +fetch_gltf('DamagedHelmet') +filename = read_viz_gltf('DamagedHelmet') + +gltf_obj = glTF(filename, apply_normals=True) +actors = gltf_obj.actors() + +scene.add(*actors) +scene.UseImageBasedLightingOn() + +cameras = gltf_obj.cameras +if cameras: + scene.SetActiveCamera(cameras[0]) + +interactive = True + +if interactive: + window.show(scene, size=(1280, 720)) + +window.record(scene, out_path='viz_gltf.png', size=(1280, 720)) diff --git a/fury/gltf.py b/fury/gltf.py index bf3e3d4ef..e9c8c27bd 100644 --- a/fury/gltf.py +++ b/fury/gltf.py @@ -6,8 +6,8 @@ import pygltflib as gltflib from pygltflib.utils import glb2gltf, gltf2glb from PIL import Image -from fury.lib import Texture, Camera, numpy_support, Transform, Matrix4x4 -from fury import transform, utils, io, actor +from fury.lib import Texture, Camera, numpy_support, Transform, Matrix4x4, PolyDataTangents +from fury import transform, utils, io, actor, material from fury.animation import Animation from fury.animation.interpolator import (linear_interpolator, slerp, step_interpolator, @@ -113,10 +113,34 @@ def actors(self): actor.SetUserTransform(_transform) if self.materials[i] is not None: - base_col_tex = self.materials[i]['baseColorTexture'] - actor.SetTexture(base_col_tex) - base_color = self.materials[i]['baseColor'] - actor.GetProperty().SetColor(tuple(base_color[:3])) + pbr = self.materials[i]['pbr'] + if pbr is not None: + base_color = pbr['baseColor'] + actor.GetProperty().SetColor(tuple(base_color[:3])) + + metal = pbr['metallicValue'] + rough = pbr['roughnessValue'] + actor.GetProperty().SetInterpolationToPBR() + actor.GetProperty().SetMetallic(metal) + actor.GetProperty().SetRoughness(rough) + + base_col_tex = pbr['baseColorTexture'] + metal_rough_tex = pbr['metallicRoughnessTexture'] + + actor.GetProperty().SetBaseColorTexture(base_col_tex) + actor.GetProperty().SetORMTexture(metal_rough_tex) + + emissive = self.materials[i]['emissive'] + if emissive is not None: + actor.GetProperty().SetEmissiveTexture(emissive) + actor.GetProperty().SetEmissiveFactor( + self.materials[i]['emissive_factor'] + ) + normal = self.materials[i]['normal'] + if normal is not None: + print('applying normal map') + actor.GetProperty().SetNormalTexture(normal) + actor.GetProperty().SetNormalScale(1.0) self._actors.append(actor) @@ -240,14 +264,22 @@ def load_mesh(self, mesh_id, transform_mat, parent): if attributes.NORMAL is not None and self.apply_normals: normals = self.get_acc_data(attributes.NORMAL) - normals = transform.apply_transformation(normals, - transform_mat) + # normals = transform.apply_transformation(normals, + # transform_mat) utils.set_polydata_normals(polydata, normals) if attributes.TEXCOORD_0 is not None: tcoords = self.get_acc_data(attributes.TEXCOORD_0) utils.set_polydata_tcoords(polydata, tcoords) + if attributes.TANGENT is not None: + tangents = self.get_acc_data(attributes.TANGENT) + utils.set_polydata_tangents(polydata, tangents[:, :3]) + elif attributes.NORMAL is not None and self.apply_normals: + doa = [0, 1, .5] + tangents = utils.tangents_from_direction_of_anisotropy(normals, doa) + utils.set_polydata_tangents(polydata, tangents) + if attributes.COLOR_0 is not None: color = self.get_acc_data(attributes.COLOR_0) color = color[:, :-1]*255 @@ -385,24 +417,59 @@ def get_materials(self, mat_id): """ material = self.gltf.materials[mat_id] - bct = None - + pbr_dict = None + pbr = material.pbrMetallicRoughness + if pbr is not None: + bct, orm = None, None + if pbr.baseColorTexture is not None: + bct = pbr.baseColorTexture.index + bct = self.get_texture(bct, True) + if pbr.metallicRoughnessTexture is not None: + mrt = pbr.metallicRoughnessTexture.index + # find if there's any occulsion tex present + occ = material.occlusionTexture + if occ is not None and occ.index == mrt: + orm = self.get_texture(mrt) + else: + mrt = self.get_texture(mrt, rgb=True) + occ_tex = self.get_texture(occ.index, rgb=True) if occ else None + # generate orm texture + orm = self.generate_orm(mrt, occ_tex) + colors = pbr.baseColorFactor + metalvalue = pbr.metallicFactor + roughvalue = pbr.roughnessFactor + pbr_dict = {'baseColorTexture': bct, + 'metallicRoughnessTexture': orm, + 'baseColor': colors, + 'metallicValue': metalvalue, + 'roughnessValue': roughvalue} + normal = material.normalTexture + normal_tex = self.get_texture(normal.index) if normal else None + occlusion = material.occlusionTexture + occ_tex = self.get_texture(occlusion.index) if occlusion else None + # must update pbr_dict with ORM texture + emissive = material.emissiveTexture + emi_tex = self.get_texture(emissive.index, True) if emissive else None + + + return { + 'pbr' : pbr_dict, + 'normal' : normal_tex, + 'occlusion' : occ_tex, + 'emissive' : emi_tex, + 'emissive_factor' : material.emissiveFactor + } - if pbr.baseColorTexture is not None: - bct = pbr.baseColorTexture.index - bct = self.get_texture(bct) - colors = pbr.baseColorFactor - return {'baseColorTexture': bct, - 'baseColor': colors} - - def get_texture(self, tex_id): + def get_texture(self, tex_id, srgb_colorspace=False, rgb=False): """Read and convert image into vtk texture. Parameters ---------- tex_id : int Texture index + srgb_colorspace : bool + Use vtkSRGB colorspace. (default=False) Returns ------- @@ -444,14 +511,57 @@ def get_texture(self, tex_id): else: image_path = os.path.join(self.pwd, file) - rgb = io.load_image(image_path) - grid = utils.rgb_to_vtk(rgb) + rgb_array = io.load_image(image_path) + if rgb: + return rgb_array + grid = utils.rgb_to_vtk(rgb_array) atexture = Texture() atexture.InterpolateOn() atexture.EdgeClampOn() atexture.SetInputDataObject(grid) + if srgb_colorspace: + atexture.UseSRGBColorSpaceOn() + atexture.Update() return atexture + + def generate_orm(self, metallic_roughness=None, occlusion=None): + """Generates ORM texture from O, R & M textures. + We do this by swapping Red channel of metallic_roughness with the + occlusion texture and adding metallic to Blue channel. + + Parameters + ---------- + metallic_roughness : ndarray + occlusion : ndarray + """ + shape = metallic_roughness.shape + rgb_array = np.copy(metallic_roughness) + # metallic is red if name starts as metallicRoughness, otherwise its + # in the green channel + # https://github.com/KhronosGroup/glTF/issues/857#issuecomment-290530762 + metal_arr = metallic_roughness[:, :, 2] + rough_arr = metallic_roughness[:, :, 1] + if occlusion is None: + occ_arr = np.full((shape[0], shape[1]), 256) + # print(occ_arr) + else: + if len(list(occlusion.shape)) > 2: + # occ_arr = np.dot(occlusion, np.array([0.2989, 0.5870, 0.1140])) + occ_arr = occlusion.sum(2) / 3 + # both equation grayscales but second one is less computation. + rgb_array[:, :, 0][:] = metal_arr # blue channel + rgb_array[:, :, 1][:] = rough_arr + rgb_array[:, :, 2][:] = occ_arr # red channel + + grid = utils.rgb_to_vtk(rgb_array) + atexture = Texture() + atexture.InterpolateOn() + atexture.EdgeClampOn() + atexture.SetInputDataObject(grid) + + return atexture + def load_camera(self, camera_id, transform_mat): """Load the camera data of a node. diff --git a/fury/lib.py b/fury/lib.py index 5b4f343a5..e13858cd1 100644 --- a/fury/lib.py +++ b/fury/lib.py @@ -121,6 +121,7 @@ ContourFilter = fcvtk.vtkContourFilter TubeFilter = fcvtk.vtkTubeFilter Glyph3D = fcvtk.vtkGlyph3D +PolyDataTangents = fcvtk.vtkPolyDataTangents ############################################################## # vtkFiltersGeneral Module From 4361ab3ca050bb5c01808f0db8dcae8f42d25309 Mon Sep 17 00:00:00 2001 From: xtanion Date: Thu, 6 Apr 2023 23:32:13 +0530 Subject: [PATCH 2/2] resolving conflicts --- fury/gltf.py | 10 ++++++---- fury/lib.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fury/gltf.py b/fury/gltf.py index e9c8c27bd..a0cd539ce 100644 --- a/fury/gltf.py +++ b/fury/gltf.py @@ -4,14 +4,16 @@ import numpy as np import copy import pygltflib as gltflib -from pygltflib.utils import glb2gltf, gltf2glb from PIL import Image -from fury.lib import Texture, Camera, numpy_support, Transform, Matrix4x4, PolyDataTangents -from fury import transform, utils, io, actor, material +from pygltflib.utils import glb2gltf, gltf2glb + +from fury import actor, io, transform, utils, material from fury.animation import Animation -from fury.animation.interpolator import (linear_interpolator, slerp, +from fury.animation.interpolator import (linear_interpolator, + slerp, step_interpolator, tan_cubic_spline_interpolator) +from fury.lib import Camera, Matrix4x4, Texture, Transform, numpy_support, PolyDataTangents comp_type = { 5120: {'size': 1, 'dtype': np.byte}, diff --git a/fury/lib.py b/fury/lib.py index e13858cd1..28c71788c 100644 --- a/fury/lib.py +++ b/fury/lib.py @@ -121,6 +121,7 @@ ContourFilter = fcvtk.vtkContourFilter TubeFilter = fcvtk.vtkTubeFilter Glyph3D = fcvtk.vtkGlyph3D +TriangleFilter = fcvtk.vtkTriangleFilter PolyDataTangents = fcvtk.vtkPolyDataTangents ##############################################################