update

fudan-generative-vision · Jun 13, 2024 · 9fe337c · 9fe337c
1 parent 531941e
commit 9fe337c
Show file tree

Hide file tree

Showing 20 changed files with 44 additions and 37 deletions.
diff --git a/src/assets/img/best_visual_results.jpg b/src/assets/img/best_visual_results.jpg
diff --git a/src/assets/img/best_visual_results.png b/src/assets/img/best_visual_results.png
diff --git a/src/assets/img/framework.jpg b/src/assets/img/framework.jpg
diff --git a/src/assets/img/framework.png b/src/assets/img/framework.png
diff --git a/src/assets/video/ablation/exp_1.mp4 b/src/assets/video/ablation/exp_1.mp4
diff --git a/src/assets/video/ablation/exp_2.mp4 b/src/assets/video/ablation/exp_2.mp4
diff --git a/src/assets/video/ablation/lip_1.mp4 b/src/assets/video/ablation/lip_1.mp4
diff --git a/src/assets/video/ablation/lip_2.mp4 b/src/assets/video/ablation/lip_2.mp4
diff --git a/src/assets/video/ablation/pose_1.mp4 b/src/assets/video/ablation/pose_1.mp4
diff --git a/src/assets/video/ablation/pose_2.mp4 b/src/assets/video/ablation/pose_2.mp4
diff --git a/src/assets/video/portrait_style/3.mp4 b/src/assets/video/portrait_style/3.mp4
diff --git a/src/assets/video/portrait_style/4.mp4 b/src/assets/video/portrait_style/4.mp4
diff --git a/src/assets/video/singing/10.mp4 b/src/assets/video/singing/10.mp4
diff --git a/src/assets/video/singing/4.mp4 b/src/assets/video/singing/4.mp4
diff --git a/src/assets/video/singing/5.mp4 b/src/assets/video/singing/5.mp4
diff --git a/src/assets/video/singing/6.mp4 b/src/assets/video/singing/6.mp4
diff --git a/src/assets/video/singing/7.mp4 b/src/assets/video/singing/7.mp4
diff --git a/src/assets/video/singing/8.mp4 b/src/assets/video/singing/8.mp4
diff --git a/src/assets/video/singing/9.mp4 b/src/assets/video/singing/9.mp4
diff --git a/src/index.json b/src/index.json
@@ -88,96 +88,103 @@
                 "github": "https://github.com/fudan-generative-vision/hallo",
                 "huggingface": "https://huggingface.co/fudan-generative-ai/hallo"
             },
-            "mainVideo": "assets/video/cross_id/1.mp4"
+            "mainVideo": ""
         }
     },
     {
         "template": "abstract",
         "props": {
-            "figure": "assets/img/best_visual_results.png",
+            "figure": "assets/img/best_visual_results.jpg",
             "content": "The field of portrait image animation, driven by speech audio input, has experienced significant advancements in the generation of realistic and dynamic portraits. This research delves into the complexities of synchronizing facial movements and creating visually appealing, temporally consistent animations within the framework of diffusion-based methodologies. Moving away from traditional paradigms that rely on parametric models for intermediate facial representations, our innovative approach embraces the end-to-end diffusion paradigm and introduces a hierarchical audio-driven visual synthesis module to enhance the precision of alignment between audio inputs and visual outputs, encompassing lip, expression, and pose motion. Our proposed network architecture seamlessly integrates diffusion-based generative models, a UNet-based denoiser, temporal alignment techniques, and a reference network. The proposed hierarchical audio-driven visual synthesis offers adaptive control over expression and pose diversity, enabling more effective personalization tailored to different identities. Through a comprehensive evaluation that incorporates both qualitative and quantitative analyses, our approach demonstrates obvious enhancements in image and video quality, lip synchronization precision, and motion diversity."
         }
     },
     {
         "template": "framework",
         "props": {
-            "image": "assets/img/framework.png",
+            "image": "assets/img/framework.jpg",
             "description": "Specifically, we integrates a reference image containing a portrait with corresponding audio input to drive portrait animation. Optional visual synthesis weights can be used to balance lip, expression, and pose weights. ReferenceNet encodes global visual texture information for consistent and controllable character animation. Face and audio encoders generate high-fidelity portrait identity features and encode audio as motion information respectively. The module of hierarchical audio-driven visual synthesis establishes relationships between audio and visual components (lips, expression, pose), with a UNet denoiser used in the diffusion process."
         }
     },
     {
-        "template": "video-comparision",
+        "template": "video-carousel",
         "props": {
-            "title": "Ablation Study-Motion Scale Control",
-            "subtitle": "Lip Control",
+            "id": "vc1",
+            "title": "Virtual Character",
             "items": [
-                [
-                    "assets/video/ablation/lip_1.mp4",
-                    "assets/video/ablation/lip_2.mp4"
-                ]
+                "assets/video/portrait_style/2.mp4",
+                "assets/video/portrait_style/1.mp4"
             ]
         }
     },
     {
-        "template": "video-comparision",
+        "template": "video-carousel",
         "props": {
+            "id": "vc2",
             "title": "",
-            "subtitle": "Expression Control",
             "items": [
-                [
-                    "assets/video/ablation/exp_1.mp4",
-                    "assets/video/ablation/exp_2.mp4"
-                ]
+                "assets/video/portrait_style/3.mp4",
+                "assets/video/portrait_style/4.mp4"
             ]
         }
     },
     {
-        "template": "video-comparision",
+        "template": "single-video",
         "props": {
-            "title": "",
-            "subtitle": "Pose Control",
+            "title": "Real character",
             "items": [
-                [
-                    "assets/video/ablation/pose_1.mp4",
-                    "assets/video/ablation/pose_2.mp4"
-                ]
+                "assets/video/cross_id/1.mp4",
+                "assets/video/cross_id/2.mp4"
             ]
         }
     },
     {
         "template": "single-video",
         "props": {
-            "title": "Singing Portrait",
+            "id": "mc1",
+            "title": "Motion Control (pose, expression, lip)",
+            "subtitle": "Pose Control",
             "items": [
-                "assets/video/singing/1.mp4",
-                "assets/video/singing/2.mp4"
+                "assets/video/ablation/pose_1.mp4",
+                "assets/video/ablation/pose_2.mp4"
             ]
         }
     },
     {
-        "template": "video-comparision",
+        "template": "single-video",
         "props": {
-            "title": "Portrait Style",
+            "id": "mc2",
+            "title": "",
+            "subtitle": "Expression Control",
             "items": [
-                [
-                    "assets/video/portrait_style/1.mp4",
-                    "assets/video/portrait_style/2.mp4"
-                ]
+                "assets/video/ablation/exp_1.mp4",
+                "assets/video/ablation/exp_2.mp4"
             ]
         }
     },
     {
-        "template": "video-comparision",
+        "template": "single-video",
         "props": {
+            "id": "mc3",
             "title": "",
+            "subtitle": "Lip Control",
             "items": [
-                [
-                    "assets/video/portrait_style/3.mp4",
-                    "assets/video/portrait_style/4.mp4"
-                ]
+                "assets/video/ablation/lip_1.mp4",
+                "assets/video/ablation/lip_2.mp4"
             ]
         }
     },
+    {
+        "template": "video-carousel",
+        "props": {
+            "title": "Singing",
+            "items": [
+                "assets/video/singing/6.mp4",
+                "assets/video/singing/5.mp4",
+                "assets/video/singing/8.mp4"
+            ],
+            "count": 3
+        }
+    },
     {
         "template": "video-carousel",
         "props": {