diff --git a/package-lock.json b/package-lock.json index 3f94f071e..331215544 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,9 @@ "@material-ui/core": "^4.12.3", "@material-ui/icons": "^4.11.2", "@material-ui/lab": "^4.0.0-alpha.60", + "@tensorflow-models/face-landmarks-detection": "^1.0.2", + "@tensorflow/tfjs-backend-wasm": "^4.0.0", + "@tensorflow/tfjs-core": "^4.0.0", "@twilio-labs/plugin-rtc": "^0.8.4", "@twilio/conversations": "^2.1.0", "@twilio/video-processors": "^1.0.1", @@ -4522,6 +4525,18 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/@mediapipe/face_detection": { + "version": "0.4.1646425229", + "resolved": "https://registry.npmjs.org/@mediapipe/face_detection/-/face_detection-0.4.1646425229.tgz", + "integrity": "sha512-aeCN+fRAojv9ch3NXorP6r5tcGVLR3/gC1HmtqB0WEZBRXrdP6/3W/sGR0dHr1iT6ueiK95G9PVjbzFosf/hrg==", + "peer": true + }, + "node_modules/@mediapipe/face_mesh": { + "version": "0.4.1633559619", + "resolved": "https://registry.npmjs.org/@mediapipe/face_mesh/-/face_mesh-0.4.1633559619.tgz", + "integrity": "sha512-Vc8cdjxS5+O2gnjWH9KncYpUCVXT0h714KlWAsyqJvJbIgUJBqpppbIx8yWcAzBDxm/5cYSuBI5p5ySIPxzcEg==", + "peer": true + }, "node_modules/@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -11160,76 +11175,128 @@ "node": ">=10" } }, - "node_modules/@tensorflow-models/body-pix": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", - "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", + "node_modules/@tensorflow-models/face-detection": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@tensorflow-models/face-detection/-/face-detection-1.0.1.tgz", + "integrity": "sha512-oIEEqHy4qbkGn5xT1ldzwxK5s948+t7ts0+WoIfTPKwScpNa8YY4GJf+fpyhtOxUGGAZymXx25jrvF9YlAt9dg==", + "peer": true, + "dependencies": { + "rimraf": "^3.0.2" + }, "peerDependencies": { - "@tensorflow/tfjs-backend-webgl": "^3.6.0", - "@tensorflow/tfjs-converter": "^3.6.0", - "@tensorflow/tfjs-core": "^3.6.0" + "@mediapipe/face_detection": "~0.4.0", + "@tensorflow/tfjs-backend-webgl": "^3.14.0", + "@tensorflow/tfjs-converter": "^3.14.0", + "@tensorflow/tfjs-core": "^3.14.0" + } + }, + "node_modules/@tensorflow-models/face-landmarks-detection": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@tensorflow-models/face-landmarks-detection/-/face-landmarks-detection-1.0.2.tgz", + "integrity": "sha512-e10xKlBssv2nYg8hV93rNg7ne8NIJVT9Y1d/bpUCcBpPLJpykLw2DQ3nfPnoBpqhKDykFtKDGQVmeXvqc7H+KA==", + "dependencies": { + "rimraf": "^3.0.2" + }, + "peerDependencies": { + "@mediapipe/face_mesh": "~0.4.0", + "@tensorflow-models/face-detection": "~1.0.0", + "@tensorflow/tfjs-backend-webgl": "^3.12.0", + "@tensorflow/tfjs-converter": "^3.12.0", + "@tensorflow/tfjs-core": "^3.12.0" + } + }, + "node_modules/@tensorflow/tfjs-backend-wasm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-4.0.0.tgz", + "integrity": "sha512-Z5gqdmqywauB2ZrWFduvFc82aMMTBFN7EtWEhGAGRXVcVH2L4nIXx0bs/5aUBA0Qs9B7hiy83b+NquWrDdBeGQ==", + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "4.0.0", + "@types/emscripten": "~0.0.34" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "4.0.0" } }, - "node_modules/@tensorflow/tfjs-backend-cpu": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.18.0.tgz", - "integrity": "sha512-LcSqlylzGtpgngcMFIL3q9Q3eVaPRJ7ITZt7ivhzkCj4R5ZsnPa9qM3DCVihkQ77heAwSw4hPTo2jp5C4mJ4Cg==", + "node_modules/@tensorflow/tfjs-backend-wasm/node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.0.0.tgz", + "integrity": "sha512-Y9ok6VBMir1MVbkcK+h34hF4ZzHZKOrdjAOeE5eFbSuegJHY/AXWcZEU589VTnEE4AU+SUdjjfp6yvAtU17GAA==", "dependencies": { - "@types/seedrandom": "2.4.27", - "seedrandom": "2.4.3" + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" }, "engines": { "yarn": ">= 1.3.2" }, "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" + "@tensorflow/tfjs-core": "4.0.0" } }, "node_modules/@tensorflow/tfjs-backend-webgl": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.18.0.tgz", - "integrity": "sha512-3NknSzS1oX2BEBOrpjPMZl823S12RgshQthmIbG6QADHb4bCJA8aM4UjWpw+3bNQnRKbRDQdFbuvj10Un79s2A==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.21.0.tgz", + "integrity": "sha512-N4zitIAT9IX8B8oe489qM3f3VcESxGZIZvHmVP8varOQakTvTX859aaPo1s8hK1qCy4BjSGbweooZe4U8D4kTQ==", "dependencies": { - "@tensorflow/tfjs-backend-cpu": "3.18.0", + "@tensorflow/tfjs-backend-cpu": "3.21.0", "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", + "@types/seedrandom": "^2.4.28", "@types/webgl-ext": "0.0.30", "@types/webgl2": "0.0.6", - "seedrandom": "2.4.3" + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs-backend-webgl/node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz", + "integrity": "sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==", + "dependencies": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" }, "engines": { "yarn": ">= 1.3.2" }, "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" + "@tensorflow/tfjs-core": "3.21.0" } }, "node_modules/@tensorflow/tfjs-converter": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.18.0.tgz", - "integrity": "sha512-hpChA+zVNQOVwRnCfqDb1WI9jbEAKA6DuEm4m75Zb3dIlE6VVooDmAaHBhlc++z2q2G1sBzF9A4Bv48SUpN6vA==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz", + "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==", "peerDependencies": { - "@tensorflow/tfjs-core": "3.18.0" + "@tensorflow/tfjs-core": "3.21.0" } }, "node_modules/@tensorflow/tfjs-core": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.18.0.tgz", - "integrity": "sha512-gMxisZozqsr5sCKlphF/eVBLg91MjlBiN60tjX8hJAu0WlSn6Gi5k65GNIL+Pq6hrxpvImcfdCmTH/2XJVZ0Mg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.0.0.tgz", + "integrity": "sha512-RrTgHPT8Xo6vsBvaRkee1YOoH3lv0lEtS3ISGRcHsUy8mkT8ZIMNou4zaJuIBQ7bVPXOeOFFeTbjDvWVtYypxQ==", "dependencies": { "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", + "@types/offscreencanvas": "~2019.7.0", + "@types/seedrandom": "^2.4.28", "@types/webgl-ext": "0.0.30", - "@webgpu/types": "^0.1.16", + "@webgpu/types": "0.1.21", "long": "4.0.0", "node-fetch": "~2.6.1", - "seedrandom": "2.4.3" + "seedrandom": "^3.0.5" }, "engines": { "yarn": ">= 1.3.2" } }, + "node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": { + "version": "2019.7.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.0.tgz", + "integrity": "sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg==" + }, "node_modules/@testing-library/dom": { "version": "6.16.0", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-6.16.0.tgz", @@ -11667,6 +11734,54 @@ "node": ">=14" } }, + "node_modules/@twilio/video-processors/node_modules/@tensorflow-models/body-pix": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", + "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", + "peerDependencies": { + "@tensorflow/tfjs-backend-webgl": "^3.6.0", + "@tensorflow/tfjs-converter": "^3.6.0", + "@tensorflow/tfjs-core": "^3.6.0" + } + }, + "node_modules/@twilio/video-processors/node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz", + "integrity": "sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==", + "dependencies": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@twilio/video-processors/node_modules/@tensorflow/tfjs-core": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz", + "integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==", + "dependencies": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@webgpu/types": "0.1.16", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, + "node_modules/@twilio/video-processors/node_modules/@webgpu/types": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz", + "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==" + }, "node_modules/@twilio/video-room-monitor": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@twilio/video-room-monitor/-/video-room-monitor-1.0.1.tgz", @@ -11835,6 +11950,11 @@ "dotenv": "*" } }, + "node_modules/@types/emscripten": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-0.0.34.tgz", + "integrity": "sha512-QSb9ojDincskc+uKMI0KXp8e1NALFINCrMlp8VGKGcTSxeEyRTTKyjWw75NYrCZHUsVEEEpr1tYHpbtaC++/sQ==" + }, "node_modules/@types/enzyme": { "version": "3.10.12", "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.12.tgz", @@ -12358,9 +12478,9 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, "node_modules/@types/seedrandom": { - "version": "2.4.27", - "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", - "integrity": "sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==" + "version": "2.4.30", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz", + "integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==" }, "node_modules/@types/serve-index": { "version": "1.9.1", @@ -36736,9 +36856,9 @@ "integrity": "sha512-o/mRQGk9Rcer/jEEw/yw4mwo3EU/NvYvp577/Btqrym9Qy5/MdWGBqipbALgd2lrdWTJ5/gqDusxfnQBxOxT2Q==" }, "node_modules/seedrandom": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", - "integrity": "sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "node_modules/select-hose": { "version": "2.0.0", @@ -45408,6 +45528,18 @@ "integrity": "sha512-H1rQc1ZOHANWBvPcW+JpGwr+juXSxM8Q8YCkm3GhZd8REu1fHR3z99CErO1p9pkcfcxZnMdIZdIsXkOHY0NilA==", "dev": true }, + "@mediapipe/face_detection": { + "version": "0.4.1646425229", + "resolved": "https://registry.npmjs.org/@mediapipe/face_detection/-/face_detection-0.4.1646425229.tgz", + "integrity": "sha512-aeCN+fRAojv9ch3NXorP6r5tcGVLR3/gC1HmtqB0WEZBRXrdP6/3W/sGR0dHr1iT6ueiK95G9PVjbzFosf/hrg==", + "peer": true + }, + "@mediapipe/face_mesh": { + "version": "0.4.1633559619", + "resolved": "https://registry.npmjs.org/@mediapipe/face_mesh/-/face_mesh-0.4.1633559619.tgz", + "integrity": "sha512-Vc8cdjxS5+O2gnjWH9KncYpUCVXT0h714KlWAsyqJvJbIgUJBqpppbIx8yWcAzBDxm/5cYSuBI5p5ySIPxzcEg==", + "peer": true + }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -50400,53 +50532,93 @@ "defer-to-connect": "^2.0.0" } }, - "@tensorflow-models/body-pix": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", - "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", - "requires": {} + "@tensorflow-models/face-detection": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@tensorflow-models/face-detection/-/face-detection-1.0.1.tgz", + "integrity": "sha512-oIEEqHy4qbkGn5xT1ldzwxK5s948+t7ts0+WoIfTPKwScpNa8YY4GJf+fpyhtOxUGGAZymXx25jrvF9YlAt9dg==", + "peer": true, + "requires": { + "rimraf": "^3.0.2" + } }, - "@tensorflow/tfjs-backend-cpu": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.18.0.tgz", - "integrity": "sha512-LcSqlylzGtpgngcMFIL3q9Q3eVaPRJ7ITZt7ivhzkCj4R5ZsnPa9qM3DCVihkQ77heAwSw4hPTo2jp5C4mJ4Cg==", + "@tensorflow-models/face-landmarks-detection": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@tensorflow-models/face-landmarks-detection/-/face-landmarks-detection-1.0.2.tgz", + "integrity": "sha512-e10xKlBssv2nYg8hV93rNg7ne8NIJVT9Y1d/bpUCcBpPLJpykLw2DQ3nfPnoBpqhKDykFtKDGQVmeXvqc7H+KA==", "requires": { - "@types/seedrandom": "2.4.27", - "seedrandom": "2.4.3" + "rimraf": "^3.0.2" + } + }, + "@tensorflow/tfjs-backend-wasm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-4.0.0.tgz", + "integrity": "sha512-Z5gqdmqywauB2ZrWFduvFc82aMMTBFN7EtWEhGAGRXVcVH2L4nIXx0bs/5aUBA0Qs9B7hiy83b+NquWrDdBeGQ==", + "requires": { + "@tensorflow/tfjs-backend-cpu": "4.0.0", + "@types/emscripten": "~0.0.34" + }, + "dependencies": { + "@tensorflow/tfjs-backend-cpu": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.0.0.tgz", + "integrity": "sha512-Y9ok6VBMir1MVbkcK+h34hF4ZzHZKOrdjAOeE5eFbSuegJHY/AXWcZEU589VTnEE4AU+SUdjjfp6yvAtU17GAA==", + "requires": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + } + } } }, "@tensorflow/tfjs-backend-webgl": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.18.0.tgz", - "integrity": "sha512-3NknSzS1oX2BEBOrpjPMZl823S12RgshQthmIbG6QADHb4bCJA8aM4UjWpw+3bNQnRKbRDQdFbuvj10Un79s2A==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.21.0.tgz", + "integrity": "sha512-N4zitIAT9IX8B8oe489qM3f3VcESxGZIZvHmVP8varOQakTvTX859aaPo1s8hK1qCy4BjSGbweooZe4U8D4kTQ==", "requires": { - "@tensorflow/tfjs-backend-cpu": "3.18.0", + "@tensorflow/tfjs-backend-cpu": "3.21.0", "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", + "@types/seedrandom": "^2.4.28", "@types/webgl-ext": "0.0.30", "@types/webgl2": "0.0.6", - "seedrandom": "2.4.3" + "seedrandom": "^3.0.5" + }, + "dependencies": { + "@tensorflow/tfjs-backend-cpu": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz", + "integrity": "sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==", + "requires": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + } + } } }, "@tensorflow/tfjs-converter": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.18.0.tgz", - "integrity": "sha512-hpChA+zVNQOVwRnCfqDb1WI9jbEAKA6DuEm4m75Zb3dIlE6VVooDmAaHBhlc++z2q2G1sBzF9A4Bv48SUpN6vA==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz", + "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==", "requires": {} }, "@tensorflow/tfjs-core": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.18.0.tgz", - "integrity": "sha512-gMxisZozqsr5sCKlphF/eVBLg91MjlBiN60tjX8hJAu0WlSn6Gi5k65GNIL+Pq6hrxpvImcfdCmTH/2XJVZ0Mg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.0.0.tgz", + "integrity": "sha512-RrTgHPT8Xo6vsBvaRkee1YOoH3lv0lEtS3ISGRcHsUy8mkT8ZIMNou4zaJuIBQ7bVPXOeOFFeTbjDvWVtYypxQ==", "requires": { "@types/long": "^4.0.1", - "@types/offscreencanvas": "~2019.3.0", - "@types/seedrandom": "2.4.27", + "@types/offscreencanvas": "~2019.7.0", + "@types/seedrandom": "^2.4.28", "@types/webgl-ext": "0.0.30", - "@webgpu/types": "^0.1.16", + "@webgpu/types": "0.1.21", "long": "4.0.0", "node-fetch": "~2.6.1", - "seedrandom": "2.4.3" + "seedrandom": "^3.0.5" + }, + "dependencies": { + "@types/offscreencanvas": { + "version": "2019.7.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.0.tgz", + "integrity": "sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg==" + } } }, "@testing-library/dom": { @@ -50798,6 +50970,43 @@ "@tensorflow/tfjs-backend-webgl": "^3.2.0", "@tensorflow/tfjs-converter": "^3.2.0", "@tensorflow/tfjs-core": "^3.2.0" + }, + "dependencies": { + "@tensorflow-models/body-pix": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-2.2.0.tgz", + "integrity": "sha512-e8PEf8xWlWhXJtGvlTQpdRyUuDXKAFJT/f8SWiBfvim5BAUyFp2dTeVV3gjDDp8MUEo7T3lofTZUexy/aG4YEA==", + "requires": {} + }, + "@tensorflow/tfjs-backend-cpu": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz", + "integrity": "sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==", + "requires": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + } + }, + "@tensorflow/tfjs-core": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz", + "integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==", + "requires": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@webgpu/types": "0.1.16", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + } + }, + "@webgpu/types": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz", + "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==" + } } }, "@twilio/video-room-monitor": { @@ -50962,6 +51171,11 @@ "dotenv": "*" } }, + "@types/emscripten": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-0.0.34.tgz", + "integrity": "sha512-QSb9ojDincskc+uKMI0KXp8e1NALFINCrMlp8VGKGcTSxeEyRTTKyjWw75NYrCZHUsVEEEpr1tYHpbtaC++/sQ==" + }, "@types/enzyme": { "version": "3.10.12", "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.12.tgz", @@ -51460,9 +51674,9 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, "@types/seedrandom": { - "version": "2.4.27", - "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", - "integrity": "sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==" + "version": "2.4.30", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz", + "integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==" }, "@types/serve-index": { "version": "1.9.1", @@ -69989,9 +70203,9 @@ "integrity": "sha512-o/mRQGk9Rcer/jEEw/yw4mwo3EU/NvYvp577/Btqrym9Qy5/MdWGBqipbALgd2lrdWTJ5/gqDusxfnQBxOxT2Q==" }, "seedrandom": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", - "integrity": "sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "select-hose": { "version": "2.0.0", diff --git a/package.json b/package.json index 6f33c8781..d651ac3d5 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,9 @@ "@material-ui/core": "^4.12.3", "@material-ui/icons": "^4.11.2", "@material-ui/lab": "^4.0.0-alpha.60", + "@tensorflow-models/face-landmarks-detection": "^1.0.2", + "@tensorflow/tfjs-backend-wasm": "^4.0.0", + "@tensorflow/tfjs-core": "^4.0.0", "@twilio-labs/plugin-rtc": "^0.8.4", "@twilio/conversations": "^2.1.0", "@twilio/video-processors": "^1.0.1", diff --git a/src/components/MaskSelectionDialog/MaskSelectionDialog.tsx b/src/components/MaskSelectionDialog/MaskSelectionDialog.tsx new file mode 100644 index 000000000..737f75a0d --- /dev/null +++ b/src/components/MaskSelectionDialog/MaskSelectionDialog.tsx @@ -0,0 +1,51 @@ +import React from 'react'; +import MaskSelectionHeader from './MaskSelectionHeader/MaskSelectionHeader'; +import MaskThumbnail from './MaskThumbnail/MaskThumbnail'; +import Drawer from '@material-ui/core/Drawer'; +import { makeStyles, Theme } from '@material-ui/core/styles'; +import { maskConfig } from '../VideoProvider/useMaskSettings/useMaskSettings'; +import useVideoContext from '../../hooks/useVideoContext/useVideoContext'; + +const useStyles = makeStyles((theme: Theme) => ({ + drawer: { + display: 'flex', + width: theme.rightDrawerWidth, + height: `calc(100% - ${theme.footerHeight}px)`, + }, + thumbnailContainer: { + display: 'flex', + flexWrap: 'wrap', + padding: '5px', + overflowY: 'auto', + }, +})); + +function MaskSelectionDialog() { + const classes = useStyles(); + const { isMaskSelectionOpen, setIsMaskSelectionOpen } = useVideoContext(); + + const imageNames = maskConfig.imageNames; + const images = maskConfig.images; + + return ( + + setIsMaskSelectionOpen(false)} /> +
+ + {images.map((image, index) => ( + + ))} +
+ + ); +} + +export default MaskSelectionDialog; diff --git a/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.test.tsx b/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.test.tsx new file mode 100644 index 000000000..397c78bab --- /dev/null +++ b/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.test.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import CloseIcon from '../../../icons/CloseIcon'; +import MaskSelectionHeader from './MaskSelectionHeader'; + +const mockCloseDialog = jest.fn(); + +describe('The Background Selection Header Component', () => { + it('should close the selection dialog when "X" is clicked', () => { + const wrapper = shallow(); + wrapper + .find(CloseIcon) + .parent() + .simulate('click'); + expect(mockCloseDialog).toHaveBeenCalled(); + }); +}); diff --git a/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.tsx b/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.tsx new file mode 100644 index 000000000..a7ed38a2d --- /dev/null +++ b/src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.tsx @@ -0,0 +1,43 @@ +import React from 'react'; +import { makeStyles, createStyles } from '@material-ui/core/styles'; +import CloseIcon from '../../../icons/CloseIcon'; + +const useStyles = makeStyles(() => + createStyles({ + container: { + minHeight: '56px', + background: '#F4F4F6', + borderBottom: '1px solid #E4E7E9', + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + padding: '0 1em', + }, + text: { + fontWeight: 'bold', + }, + closeMaskSelection: { + cursor: 'pointer', + display: 'flex', + background: 'transparent', + border: '0', + padding: '0.4em', + }, + }) +); + +interface MaskSelectionHeaderProps { + onClose: () => void; +} + +export default function MaskSelectionHeader({ onClose }: MaskSelectionHeaderProps) { + const classes = useStyles(); + return ( +
+
Mask Effects
+ +
+ ); +} diff --git a/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.test.tsx b/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.test.tsx new file mode 100644 index 000000000..bc50ba594 --- /dev/null +++ b/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.test.tsx @@ -0,0 +1,65 @@ +import React from 'react'; +import MaskThumbnail from './MaskThumbnail'; +import BlurIcon from '@material-ui/icons/BlurOnOutlined'; +import NoneIcon from '@material-ui/icons/NotInterestedOutlined'; +import { shallow } from 'enzyme'; +import useVideoContext from '../../../hooks/useVideoContext/useVideoContext'; + +jest.mock('../../../hooks/useVideoContext/useVideoContext'); +const mockUseVideoContext = useVideoContext as jest.Mock; +const mockSetMaskSettings = jest.fn(); +mockUseVideoContext.mockImplementation(() => ({ + maskSettings: { + type: 'blur', + index: 0, + }, + setMaskSettings: mockSetMaskSettings, +})); + +describe('The MaskThumbnail component', () => { + it('should update the mask settings when clicked', () => { + const wrapper = shallow(); + wrapper.simulate('click'); + expect(mockSetMaskSettings).toHaveBeenCalledWith({ index: 5, type: 'none' }); + }); + + it('should not be selected when thumbnail prop and maskSettings type are not equivalent (icon)', () => { + const wrapper = shallow(); + expect(wrapper.find('.selected').exists()).toBe(false); + }); + + it('should be selected when thumbnail prop and maskSettings type are equivalent (image)', () => { + mockUseVideoContext.mockImplementationOnce(() => ({ + maskSettings: { + type: 'image', + index: 1, + }, + setMaskSettings: mockSetMaskSettings, + })); + const wrapper = shallow(); + expect(wrapper.find('.selected').exists()).toBe(true); + }); + + it('should not be selected when thumbnail and maskSettings type are not equivlanet (image)', () => { + mockUseVideoContext.mockImplementationOnce(() => ({ + maskSettings: { + type: 'image', + index: 1, + }, + setMaskSettings: mockSetMaskSettings, + })); + const wrapper = shallow(); + expect(wrapper.find('.selected').exists()).toBe(false); + }); + + it("should contain the NoneIcon when thumbnail is set to 'none'", () => { + const wrapper = shallow(); + expect(wrapper.containsMatchingElement()).toBe(true); + }); + + it("should not have any icons when thumbnail is set to 'image'", () => { + const wrapper = shallow(); + expect(wrapper.containsMatchingElement()).toBe(false); + expect(wrapper.containsMatchingElement()).toBe(false); + }); +}); diff --git a/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.tsx b/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.tsx new file mode 100644 index 000000000..9f0e23514 --- /dev/null +++ b/src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.tsx @@ -0,0 +1,132 @@ +import React from 'react'; +import clsx from 'clsx'; +import BlurIcon from '@material-ui/icons/BlurOnOutlined'; +import { makeStyles, Theme, createStyles } from '@material-ui/core/styles'; +import NoneIcon from '@material-ui/icons/NotInterestedOutlined'; +import useVideoContext from '../../../hooks/useVideoContext/useVideoContext'; + +export type Thumbnail = 'none' | 'image'; + +interface MaskThumbnailProps { + thumbnail: Thumbnail; + imagePath?: string; + name?: string; + index?: number; +} + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + thumbContainer: { + margin: '5px', + width: 'calc(50% - 10px)', + display: 'flex', + position: 'relative', + '&::after': { + content: '""', + paddingBottom: '55.5%', + }, + }, + thumbIconContainer: { + width: '100%', + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + borderRadius: '10px', + border: `solid ${theme.palette.grey[400]}`, + '&.selected': { + border: `solid ${theme.palette.primary.main}`, + '& svg': { + color: `${theme.palette.primary.main}`, + }, + }, + }, + thumbIcon: { + height: 50, + width: 50, + color: `${theme.palette.grey[400]}`, + '&.selected': { + color: `${theme.palette.primary.main}`, + }, + }, + thumbImage: { + width: '100%', + height: '100%', + position: 'absolute', + top: 0, + bottom: 0, + left: 0, + right: 0, + objectFit: 'cover', + borderRadius: '10px', + border: `solid ${theme.palette.grey[400]}`, + '&:hover': { + cursor: 'pointer', + '& svg': { + color: `${theme.palette.primary.main}`, + }, + '& $thumbOverlay': { + visibility: 'visible', + }, + }, + '&.selected': { + border: `solid ${theme.palette.primary.main}`, + '& svg': { + color: `${theme.palette.primary.main}`, + }, + }, + }, + thumbOverlay: { + position: 'absolute', + color: 'transparent', + padding: '20px', + fontSize: '14px', + fontWeight: 'bold', + width: '100%', + height: '100%', + borderRadius: '10px', + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + '&:hover': { + background: 'rgba(95, 93, 128, 0.6)', + color: 'white', + }, + }, + }) +); + +export default function MaskThumbnail({ thumbnail, imagePath, name, index }: MaskThumbnailProps) { + const classes = useStyles(); + const { maskSettings, setMaskSettings } = useVideoContext(); + const isImage = thumbnail === 'image'; + const thumbnailSelected = isImage + ? maskSettings.index === index && maskSettings.type === 'image' + : maskSettings.type === thumbnail; + const icons = { + none: NoneIcon, + blur: BlurIcon, + image: null, + }; + const ThumbnailIcon = icons[thumbnail]; + + return ( +
+ setMaskSettings({ + type: thumbnail, + index: index, + }) + } + > + {ThumbnailIcon ? ( +
+ +
+ ) : ( + {name} + )} +
{name}
+
+ ); +} diff --git a/src/components/MenuBar/Menu/Menu.tsx b/src/components/MenuBar/Menu/Menu.tsx index b023a6187..2501ee42f 100644 --- a/src/components/MenuBar/Menu/Menu.tsx +++ b/src/components/MenuBar/Menu/Menu.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef } from 'react'; import AboutDialog from '../../AboutDialog/AboutDialog'; import BackgroundIcon from '../../../icons/BackgroundIcon'; +import MaskIcon from '../../../icons/MaskIcon'; import CollaborationViewIcon from '@material-ui/icons/AccountBox'; import DeviceSelectionDialog from '../../DeviceSelectionDialog/DeviceSelectionDialog'; import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; @@ -39,7 +40,7 @@ export default function Menu(props: { buttonClassName?: string }) { const { isFetching, updateRecordingRules, roomType, setIsGalleryViewActive, isGalleryViewActive } = useAppState(); const { setIsChatWindowOpen } = useChatContext(); const isRecording = useIsRecording(); - const { room, setIsBackgroundSelectionOpen } = useVideoContext(); + const { room, setIsBackgroundSelectionOpen, setIsMaskSelectionOpen } = useVideoContext(); const anchorRef = useRef(null); const { flipCameraDisabled, toggleFacingMode, flipCameraSupported } = useFlipCameraToggle(); @@ -85,6 +86,7 @@ export default function Menu(props: { buttonClassName?: string }) { { setIsBackgroundSelectionOpen(true); + setIsMaskSelectionOpen(false); setIsChatWindowOpen(false); setMenuOpen(false); }} @@ -96,6 +98,22 @@ export default function Menu(props: { buttonClassName?: string }) { )} + {isSupported && ( + { + setIsBackgroundSelectionOpen(false); + setIsMaskSelectionOpen(true); + setIsChatWindowOpen(false); + setMenuOpen(false); + }} + > + + + + Face Effects + + )} + {flipCameraSupported && ( diff --git a/src/components/Room/Room.tsx b/src/components/Room/Room.tsx index 999957f2c..14ad37243 100644 --- a/src/components/Room/Room.tsx +++ b/src/components/Room/Room.tsx @@ -1,5 +1,6 @@ import React, { useEffect, useRef } from 'react'; import BackgroundSelectionDialog from '../BackgroundSelectionDialog/BackgroundSelectionDialog'; +import MaskSelectionDialog from '../MaskSelectionDialog/MaskSelectionDialog'; import ChatWindow from '../ChatWindow/ChatWindow'; import clsx from 'clsx'; import { GalleryView } from '../GalleryView/GalleryView'; @@ -73,7 +74,7 @@ export function useSetSpeakerViewOnScreenShare( export default function Room() { const classes = useStyles(); const { isChatWindowOpen } = useChatContext(); - const { isBackgroundSelectionOpen, room } = useVideoContext(); + const { isBackgroundSelectionOpen, isMaskSelectionOpen, room } = useVideoContext(); const { isGalleryViewActive, setIsGalleryViewActive } = useAppState(); const theme = useTheme(); const isMobile = useMediaQuery(theme.breakpoints.down('sm')); @@ -86,7 +87,7 @@ export default function Room() { return (
{/* @@ -111,6 +112,7 @@ export default function Room() { +
); } diff --git a/src/components/VideoProvider/index.tsx b/src/components/VideoProvider/index.tsx index e2fdcadab..62c7f0f35 100644 --- a/src/components/VideoProvider/index.tsx +++ b/src/components/VideoProvider/index.tsx @@ -5,6 +5,7 @@ import { SelectedParticipantProvider } from './useSelectedParticipant/useSelecte import AttachVisibilityHandler from './AttachVisibilityHandler/AttachVisibilityHandler'; import useBackgroundSettings, { BackgroundSettings } from './useBackgroundSettings/useBackgroundSettings'; +import useMaskSettings, { MaskSettings } from './useMaskSettings/useMaskSettings'; import useHandleRoomDisconnection from './useHandleRoomDisconnection/useHandleRoomDisconnection'; import useHandleTrackPublicationFailed from './useHandleTrackPublicationFailed/useHandleTrackPublicationFailed'; import useLocalTracks from './useLocalTracks/useLocalTracks'; @@ -32,9 +33,13 @@ export interface IVideoContext { toggleScreenShare: () => void; getAudioAndVideoTracks: () => Promise; isBackgroundSelectionOpen: boolean; + isMaskSelectionOpen: boolean; setIsBackgroundSelectionOpen: (value: boolean) => void; + setIsMaskSelectionOpen: (value: boolean) => void; backgroundSettings: BackgroundSettings; + maskSettings: MaskSettings; setBackgroundSettings: (settings: BackgroundSettings) => void; + setMaskSettings: (settings: MaskSettings) => void; } export const VideoContext = createContext(null!); @@ -79,10 +84,12 @@ export function VideoProvider({ options, children, onError = () => {} }: VideoPr useRestartAudioTrackOnDeviceChange(localTracks); const [isBackgroundSelectionOpen, setIsBackgroundSelectionOpen] = useState(false); + const [isMaskSelectionOpen, setIsMaskSelectionOpen] = useState(false); const videoTrack = localTracks.find(track => !track.name.includes('screen') && track.kind === 'video') as | LocalVideoTrack | undefined; const [backgroundSettings, setBackgroundSettings] = useBackgroundSettings(videoTrack, room); + const [maskSettings, setMaskSettings] = useMaskSettings(videoTrack, room); return ( {} }: VideoPr toggleScreenShare, getAudioAndVideoTracks, isBackgroundSelectionOpen, + isMaskSelectionOpen, setIsBackgroundSelectionOpen, + setIsMaskSelectionOpen, backgroundSettings, + maskSettings, setBackgroundSettings, + setMaskSettings, }} > {children} diff --git a/src/components/VideoProvider/useMaskSettings/useMaskSettings.test.tsx b/src/components/VideoProvider/useMaskSettings/useMaskSettings.test.tsx new file mode 100644 index 000000000..c5582fdc4 --- /dev/null +++ b/src/components/VideoProvider/useMaskSettings/useMaskSettings.test.tsx @@ -0,0 +1,150 @@ +import { act, renderHook } from '@testing-library/react-hooks'; +import { SELECTED_MASK_SETTINGS_KEY } from '../../../constants'; +import useMaskSettings, { MaskSettings } from './useMaskSettings'; + +const mockLoadModel = jest.fn(); +let mockIsSupported = true; +jest.mock('../../../processors/face-mask/MaskProcessor', () => { + return { + MaskProcessor: jest.fn().mockImplementation(() => { + return { + loadModel: mockLoadModel, + maskImage: '', + name: 'MaskProcessor', + }; + }), + get isSupported() { + return mockIsSupported; + }, + }; +}); + +const defaultSettings = { + type: 'none', + index: 0, +}; + +const imgSettings = { + type: 'image', + index: 2, +}; + +const globalAny: any = global; + +globalAny.Image = jest.fn().mockImplementation(() => { + return { + set src(newSrc: String) { + this.onload(); + }, + }; +}); + +let mockVideoTrack: any; +let mockRoom: any; +let maskSettings: any; +let setMaskSettings: any; +let renderResult: any; + +beforeEach(async () => { + mockVideoTrack = { + kind: 'video', + processor: '', + addProcessor: jest.fn(), + removeProcessor: jest.fn(), + }; + mockRoom = { + localParticipant: 'participant', + }; + const { result } = renderHook(() => useMaskSettings(mockVideoTrack as any, mockRoom)); + renderResult = result; + mockIsSupported = true; + [maskSettings, setMaskSettings] = renderResult.current; + await act(async () => { + setMaskSettings(defaultSettings); + }); +}); + +describe('The useMaskSettings hook ', () => { + it('should not call loadModel, addProcessor, or removeProcessor if isSupported is false', async () => { + mockIsSupported = false; + mockLoadModel.mockReset(); + // update maskSettings to trigger useEffect hook + await act(async () => { + setMaskSettings(defaultSettings); + }); + expect(mockLoadModel).not.toHaveBeenCalled(); + expect(mockVideoTrack.addProcessor).not.toHaveBeenCalled(); + expect(mockVideoTrack.removeProcessor).not.toHaveBeenCalled(); + }); + + it('should return the maskSettings and update function.', () => { + expect(renderResult.current).toEqual([defaultSettings, expect.any(Function)]); + }); + + it('should set the mask settings correctly and remove the video processor when "none" is selected', async () => { + await act(async () => { + setMaskSettings(imgSettings as MaskSettings); + }); + // set video processor to none + await act(async () => { + setMaskSettings(defaultSettings as MaskSettings); + }); + maskSettings = renderResult.current[0]; + expect(maskSettings.type).toEqual('none'); + }); + + it('should set the mask settings correctly and set the video processor when "image" is selected', async () => { + await act(async () => { + setMaskSettings(imgSettings as MaskSettings); + }); + maskSettings = renderResult.current[0]; + expect(maskSettings.type).toEqual('image'); + expect(maskSettings.index).toEqual(2); + expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({ + maskImage: expect.any(Object), + loadModel: mockLoadModel, + name: 'MaskProcessor', + }); + }); + + describe('The setMaskSettings function ', () => { + it('should call videoTrack.removeProcessor if videoTrack and videoTrack.processor exists', async () => { + mockVideoTrack = { + kind: 'video', + processor: 'processor', + addProcessor: jest.fn(), + removeProcessor: jest.fn(), + }; + const { result } = renderHook(() => useMaskSettings(mockVideoTrack as any, mockRoom)); + renderResult = result; + setMaskSettings = renderResult.current[1]; + await act(async () => { + setMaskSettings(defaultSettings); + }); + expect(mockVideoTrack.removeProcessor).toHaveBeenCalled(); + expect(window.localStorage.getItem(SELECTED_MASK_SETTINGS_KEY)).toEqual(JSON.stringify(defaultSettings)); + }); + + it("should not call videoTrack.addProcessor with a param of maskProcessor if maskSettings.type is not equal to 'image'", async () => { + mockVideoTrack.addProcessor.mockReset(); + await act(async () => { + setMaskSettings(imgSettings); + }); + expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith({ + loadModel: mockLoadModel, + name: 'MaskProcessor', + }); + expect(window.localStorage.getItem(SELECTED_MASK_SETTINGS_KEY)).toEqual(JSON.stringify(imgSettings)); + }); + + it('should not error when videoTrack does not exist and sets the local storage item', async () => { + const { result } = renderHook(() => useMaskSettings({} as any, mockRoom)); + renderResult = result; + setMaskSettings = renderResult.current[1]; + await act(async () => { + setMaskSettings(defaultSettings); + }); + expect(window.localStorage.getItem(SELECTED_MASK_SETTINGS_KEY)).toEqual(JSON.stringify(defaultSettings)); + }); + }); +}); diff --git a/src/components/VideoProvider/useMaskSettings/useMaskSettings.ts b/src/components/VideoProvider/useMaskSettings/useMaskSettings.ts new file mode 100644 index 000000000..3252d989b --- /dev/null +++ b/src/components/VideoProvider/useMaskSettings/useMaskSettings.ts @@ -0,0 +1,101 @@ +import { LocalVideoTrack, Room } from 'twilio-video'; +import { useEffect, useCallback } from 'react'; +import { SELECTED_MASK_SETTINGS_KEY } from '../../../constants'; +import { isSupported } from '@twilio/video-processors'; +import { Thumbnail } from '../../MaskSelectionDialog/MaskThumbnail/MaskThumbnail'; +import { useLocalStorageState } from '../../../hooks/useLocalStorageState/useLocalStorageState'; +import { MaskProcessor } from '../../../processors/face-mask/MaskProcessor'; + +import SantaImage from '../../../images/Santa.jpg'; +import Santa2Image from '../../../images/Santa2.jpg'; +import Santa3Image from '../../../images/Santa3.jpg'; +import Santa4Image from '../../../images/Santa4.jpg'; +import Scary1Image from '../../../images/Scary1.jpg'; +import Scary2Image from '../../../images/Scary2.jpg'; + +export interface MaskSettings { + type: Thumbnail; + index?: number; +} + +const imageNames: string[] = ['Santa', 'Santa 2', 'Santa 3', 'Santa 4', 'Scary 1', 'Scary 2']; + +const rawImagePaths = [SantaImage, Santa2Image, Santa3Image, Santa4Image, Scary1Image, Scary2Image]; + +let imageElements = new Map(); + +const getImage = (index: number): Promise => { + return new Promise((resolve, reject) => { + if (imageElements.has(index)) { + return resolve(imageElements.get(index)); + } + const img = new Image(); + img.onload = () => { + imageElements.set(index, img); + resolve(img); + }; + img.onerror = reject; + img.src = rawImagePaths[index]; + }); +}; + +export const maskConfig = { + imageNames, + images: rawImagePaths, +}; + +let maskProcessor: MaskProcessor; + +export default function useMaskSettings(videoTrack: LocalVideoTrack | undefined, room?: Room | null) { + const [maskSettings, setMaskSettings] = useLocalStorageState(SELECTED_MASK_SETTINGS_KEY, { + type: 'none', + index: 0, + }); + + const removeProcessor = useCallback(() => { + if (videoTrack && videoTrack.processor) { + videoTrack.removeProcessor(videoTrack.processor); + } + }, [videoTrack]); + + const addProcessor = useCallback( + (processor: MaskProcessor) => { + if (!videoTrack || videoTrack.processor === processor) { + return; + } + removeProcessor(); + videoTrack.addProcessor(processor); + }, + [videoTrack, removeProcessor] + ); + + useEffect(() => { + if (!isSupported) { + return; + } + // make sure localParticipant has joined room before applying video processors + // this ensures that the video processors are not applied on the LocalVideoPreview + const handleProcessorChange = async () => { + if (!maskProcessor) { + maskProcessor = new MaskProcessor({ + maskImage: await getImage(0), + }); + await maskProcessor.loadModel(); + } + + if (!room?.localParticipant) { + return; + } + + if (maskSettings.type === 'image' && typeof maskSettings.index === 'number') { + maskProcessor.maskImage = await getImage(maskSettings.index); + addProcessor(maskProcessor); + } else { + removeProcessor(); + } + }; + handleProcessorChange(); + }, [maskSettings, videoTrack, room, addProcessor, removeProcessor]); + + return [maskSettings, setMaskSettings] as const; +} diff --git a/src/constants.ts b/src/constants.ts index bfc833cfe..0f54beec6 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -9,8 +9,9 @@ export const SELECTED_AUDIO_INPUT_KEY = 'TwilioVideoApp-selectedAudioInput'; export const SELECTED_AUDIO_OUTPUT_KEY = 'TwilioVideoApp-selectedAudioOutput'; export const SELECTED_VIDEO_INPUT_KEY = 'TwilioVideoApp-selectedVideoInput'; -// This is used to store the current background settings in localStorage +// This is used to store the current background and face mask settings in localStorage export const SELECTED_BACKGROUND_SETTINGS_KEY = 'TwilioVideoApp-selectedBackgroundSettings'; +export const SELECTED_MASK_SETTINGS_KEY = 'TwilioVideoApp-selectedMaskSettings'; export const GALLERY_VIEW_ASPECT_RATIO = 9 / 16; // 16:9 export const GALLERY_VIEW_MARGIN = 3; diff --git a/src/icons/MaskIcon.tsx b/src/icons/MaskIcon.tsx new file mode 100644 index 000000000..73bd49cbe --- /dev/null +++ b/src/icons/MaskIcon.tsx @@ -0,0 +1,12 @@ +import React from 'react'; + +export default function MaskIcon() { + return ( + + + + ); +} diff --git a/src/images/Grinch.jpg b/src/images/Grinch.jpg new file mode 100644 index 000000000..d4ffff286 Binary files /dev/null and b/src/images/Grinch.jpg differ diff --git a/src/images/Santa.jpg b/src/images/Santa.jpg new file mode 100644 index 000000000..5bbe02fd7 Binary files /dev/null and b/src/images/Santa.jpg differ diff --git a/src/images/Santa2.jpg b/src/images/Santa2.jpg new file mode 100644 index 000000000..ab3e27fd1 Binary files /dev/null and b/src/images/Santa2.jpg differ diff --git a/src/images/Santa3.jpg b/src/images/Santa3.jpg new file mode 100644 index 000000000..7472d952c Binary files /dev/null and b/src/images/Santa3.jpg differ diff --git a/src/images/Santa4.jpg b/src/images/Santa4.jpg new file mode 100644 index 000000000..02a622cb9 Binary files /dev/null and b/src/images/Santa4.jpg differ diff --git a/src/images/Scary1.jpg b/src/images/Scary1.jpg new file mode 100644 index 000000000..0b2539782 Binary files /dev/null and b/src/images/Scary1.jpg differ diff --git a/src/images/Scary2.jpg b/src/images/Scary2.jpg new file mode 100644 index 000000000..4d6591d8d Binary files /dev/null and b/src/images/Scary2.jpg differ diff --git a/src/processors/face-mask/MaskProcessor.ts b/src/processors/face-mask/MaskProcessor.ts new file mode 100644 index 000000000..9ef5d5199 --- /dev/null +++ b/src/processors/face-mask/MaskProcessor.ts @@ -0,0 +1,211 @@ +import * as tfjsWasm from '@tensorflow/tfjs-backend-wasm'; +import '@tensorflow/tfjs-backend-webgl'; + +import { + createDetector, + SupportedModels, + Face, + FaceLandmarksDetector, +} from '@tensorflow-models/face-landmarks-detection'; + +import { createTextureFromImage, createTextureFromImageBitmap } from './utils/textures'; +import { Facemesh } from './utils/Facemesh'; +import { calcSizeToFit, Region, render2dScene } from './utils/webgl'; +import { Render2D } from './utils/Render2D'; + +// Initialize Tf.js +tfjsWasm.setWasmPaths(`https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tfjsWasm.version_wasm}/dist/`); + +/** + * Options passed to [[MaskProcessor]] constructor. + */ +export interface MaskProcessorOptions { + /** + * The HTMLImageElement to use for the face mask. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + maskImage: HTMLImageElement; +} + +/** + * The MaskProcessor, when added to a VideoTrack, + * place a mask in each video frame on top of the person's face, + * Each instance of MaskProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * + * let faceMask; + * const img = new Image(); + * + * img.onload = () => { + * faceMask = new MaskProcessor({ + * maskImage: img, + * }); + * + * faceMask.createDetector().then(() => { + * createLocalVideoTrack({ + * camWidth: 640, + * camHeight: 480, + * frameRate: 24 + * }).then(track => { + * track.addProcessor(faceMask); + * }); + * }); + * }; + * img.src = '/face-mask.jpg'; + * ``` + */ +export class MaskProcessor { + private _maskImage!: HTMLImageElement; + // tslint:disable-next-line no-unused-variable + private readonly _name: string = 'MaskProcessor'; + + private static _model: FaceLandmarksDetector | null = null; + + private _maskTex: WebGLTexture | null; + private _isMaskUpdated: boolean; + private _maskPredictions: Face[]; + private _facePredictions: Face[]; + + private _camRegion: Region | null; + + private _outputCanvas: HTMLCanvasElement; + private _outputContext: WebGL2RenderingContext; + + private _facemesh: Facemesh; + private _r2d: Render2D; + + /** + * Construct a MaskProcessor. Default values will be used for + * any missing optional properties in [[MaskProcessorOptions]], + * and invalid properties will be ignored. + */ + constructor(options: MaskProcessorOptions) { + this.maskImage = options.maskImage; + + this._maskTex = null; + this._isMaskUpdated = false; + this._maskPredictions = []; + this._facePredictions = []; + + this._camRegion = null; + + this._outputCanvas = document.createElement('canvas'); + this._outputContext = this._outputCanvas.getContext('webgl') as WebGL2RenderingContext; + + this._facemesh = new Facemesh(this._outputContext); + this._r2d = new Render2D(this._outputContext); + } + + /** + * The HTMLImageElement representing the current face mask image. + */ + get maskImage(): HTMLImageElement { + return this._maskImage; + } + + /** + * Set an HTMLImageElement as the new face mask image. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + set maskImage(image: HTMLImageElement) { + if (!image || !image.complete || !image.naturalHeight) { + throw new Error( + 'Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded' + ); + } + this._maskImage = image; + this._isMaskUpdated = false; + } + + /** + * Load the segmentation model. + * Call this method before attaching the processor to ensure + * video frames are processed correctly. + */ + async loadModel() { + try { + MaskProcessor._model = await createDetector(SupportedModels.MediaPipeFaceMesh, { + runtime: 'tfjs', + refineLandmarks: true, + }); + + console.log('Loaded face landmarks model successfully.'); + } catch (error) { + console.error('Unable to load face landmarks model.', error); + } + } + + async processFrame(inputFrameBuffer: OffscreenCanvas, outputFrameBuffer: HTMLCanvasElement): Promise { + // The WebGL Canvas Context + const gl = this._outputContext; + + // Configure viewport dimensions + const camWidth = inputFrameBuffer.width; + const camHeight = inputFrameBuffer.height; + + this._outputCanvas.width = camWidth; + this._outputCanvas.height = camHeight; + + // Handle viewport resizing + this._facemesh.resize_facemesh_render(camWidth, camHeight); + this._r2d.resize_viewport(camWidth, camHeight); + gl.viewport(0, 0, camWidth, camHeight); + + // Get image bitmap from input frame + const inputImageBitmap = inputFrameBuffer.transferToImageBitmap(); + + // Update mask if needed + if (!this._isMaskUpdated && MaskProcessor._model) { + this._maskTex = createTextureFromImage(gl, this._maskImage); + for (let i = 0; i < 2; i++) this._maskPredictions = await MaskProcessor._model?.estimateFaces(this._maskImage); + + this._isMaskUpdated = true; + } + + const camTex = createTextureFromImageBitmap(gl, inputImageBitmap); + + /* --------------------------------------- * + * invoke TF.js (Facemesh) + * --------------------------------------- */ + this._camRegion = calcSizeToFit(camWidth, camHeight, camWidth, camHeight); + + if (MaskProcessor._model) { + this._facePredictions = await MaskProcessor._model.estimateFaces(inputImageBitmap); + } + + /* --------------------------------------- * + * render scene + * --------------------------------------- */ + gl.clearColor(0, 0, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + + if (!camTex || !this._maskTex) return; + + render2dScene( + gl, + camTex, + this._facePredictions, + camWidth, + camHeight, + this._maskImage, + this._maskTex, + this._maskPredictions, + this._camRegion, + this._facemesh, + this._r2d + ); + + // Copy content to output frame + const ctx2D = outputFrameBuffer.getContext('2d'); + ctx2D?.drawImage(this._outputCanvas, 0, 0); + } +} diff --git a/src/processors/face-mask/constants.ts b/src/processors/face-mask/constants.ts new file mode 100644 index 000000000..92e8be0bd --- /dev/null +++ b/src/processors/face-mask/constants.ts @@ -0,0 +1,49 @@ +export const varray = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0]; + +export const tarray0 = [ + 0.0, + 0.0, // 0 +-----+ 2 + 0.0, + 1.0, // | | + 1.0, + 0.0, // | | + 1.0, + 1.0, +]; // 1 +-----+ 3 + +export const tarray1 = [ + 0.0, + 1.0, // 1 +-----+ 3 Vertical Flip + 0.0, + 0.0, // | | + 1.0, + 1.0, // | | + 1.0, + 0.0, +]; // 0 +-----+ 2 + +export const tarray2 = [ + 1.0, + 0.0, // 2 +-----+ 0 Horizontal Flip + 1.0, + 1.0, // | | + 0.0, + 0.0, // | | + 0.0, + 1.0, +]; // 3 +-----+ 1 + +export const tarray3 = [ + 1.0, + 1.0, // 3 +-----+ 1 Vertical & Horizontal Flip + 1.0, + 0.0, // | | + 0.0, + 1.0, // | | + 0.0, + 0.0, +]; // 2 +-----+ 0 + +export const SHADER_NUM = 2; + +export const M_PId180f = 3.1415926 / 180.0; diff --git a/src/processors/face-mask/data/face-contour-idx.json b/src/processors/face-mask/data/face-contour-idx.json new file mode 100644 index 000000000..b8f2f1136 --- /dev/null +++ b/src/processors/face-mask/data/face-contour-idx.json @@ -0,0 +1,38 @@ +[ + 10, + 338, + 297, + 332, + 284, + 251, + 389, + 356, + 454, + 323, + 361, + 288, + 397, + 365, + 379, + 378, + 400, + 377, + 152, + 148, + 176, + 149, + 150, + 136, + 172, + 58, + 132, + 93, + 234, + 127, + 162, + 21, + 54, + 103, + 67, + 109 +] diff --git a/src/processors/face-mask/data/s-face-tris.json b/src/processors/face-mask/data/s-face-tris.json new file mode 100644 index 000000000..434a7f805 --- /dev/null +++ b/src/processors/face-mask/data/s-face-tris.json @@ -0,0 +1,2642 @@ +[ + 127, + 34, + 139, + 11, + 0, + 37, + 232, + 231, + 120, + 72, + 37, + 39, + 128, + 121, + 47, + 232, + 121, + 128, + 104, + 69, + 67, + 175, + 171, + 148, + 157, + 154, + 155, + 118, + 50, + 101, + 73, + 39, + 40, + 9, + 151, + 108, + 48, + 115, + 131, + 194, + 204, + 211, + 74, + 40, + 185, + 80, + 42, + 183, + 40, + 92, + 186, + 230, + 229, + 118, + 202, + 212, + 214, + 83, + 18, + 17, + 76, + 61, + 146, + 160, + 29, + 30, + 56, + 157, + 173, + 106, + 204, + 194, + 135, + 214, + 192, + 203, + 165, + 98, + 21, + 71, + 68, + 51, + 45, + 4, + 144, + 24, + 23, + 77, + 146, + 91, + 205, + 50, + 187, + 201, + 200, + 18, + 91, + 106, + 182, + 90, + 91, + 181, + 85, + 84, + 17, + 206, + 203, + 36, + 148, + 171, + 140, + 92, + 40, + 39, + 193, + 189, + 244, + 159, + 158, + 28, + 247, + 246, + 161, + 236, + 3, + 196, + 54, + 68, + 104, + 193, + 168, + 8, + 117, + 228, + 31, + 189, + 193, + 55, + 98, + 97, + 99, + 126, + 47, + 100, + 166, + 79, + 218, + 155, + 154, + 26, + 209, + 49, + 131, + 135, + 136, + 150, + 47, + 126, + 217, + 223, + 52, + 53, + 45, + 51, + 134, + 211, + 170, + 140, + 67, + 69, + 108, + 43, + 106, + 91, + 230, + 119, + 120, + 226, + 130, + 247, + 63, + 53, + 52, + 238, + 20, + 242, + 46, + 70, + 156, + 78, + 62, + 96, + 46, + 53, + 63, + 143, + 34, + 227, + 173, + 155, + 133, + 123, + 117, + 111, + 44, + 125, + 19, + 236, + 134, + 51, + 216, + 206, + 205, + 154, + 153, + 22, + 39, + 37, + 167, + 200, + 201, + 208, + 36, + 142, + 100, + 57, + 212, + 202, + 20, + 60, + 99, + 28, + 158, + 157, + 35, + 226, + 113, + 160, + 159, + 27, + 204, + 202, + 210, + 113, + 225, + 46, + 43, + 202, + 204, + 62, + 76, + 77, + 137, + 123, + 116, + 41, + 38, + 72, + 203, + 129, + 142, + 64, + 98, + 240, + 49, + 102, + 64, + 41, + 73, + 74, + 212, + 216, + 207, + 42, + 74, + 184, + 169, + 170, + 211, + 170, + 149, + 176, + 105, + 66, + 69, + 122, + 6, + 168, + 123, + 147, + 187, + 96, + 77, + 90, + 65, + 55, + 107, + 89, + 90, + 180, + 101, + 100, + 120, + 63, + 105, + 104, + 93, + 137, + 227, + 15, + 86, + 85, + 129, + 102, + 49, + 14, + 87, + 86, + 55, + 8, + 9, + 100, + 47, + 121, + 145, + 23, + 22, + 88, + 89, + 179, + 6, + 122, + 196, + 88, + 95, + 96, + 138, + 172, + 136, + 215, + 58, + 172, + 115, + 48, + 219, + 42, + 80, + 81, + 195, + 3, + 51, + 43, + 146, + 61, + 171, + 175, + 199, + 81, + 82, + 38, + 53, + 46, + 225, + 144, + 163, + 110, + 246, + 33, + 7, + 52, + 65, + 66, + 229, + 228, + 117, + 34, + 127, + 234, + 107, + 108, + 69, + 109, + 108, + 151, + 48, + 64, + 235, + 62, + 78, + 191, + 129, + 209, + 126, + 111, + 35, + 143, + 163, + 161, + 246, + 117, + 123, + 50, + 222, + 65, + 52, + 19, + 125, + 141, + 221, + 55, + 65, + 3, + 195, + 197, + 25, + 7, + 33, + 220, + 237, + 44, + 70, + 71, + 139, + 122, + 193, + 245, + 247, + 130, + 33, + 71, + 21, + 162, + 153, + 158, + 159, + 170, + 169, + 150, + 188, + 174, + 196, + 216, + 186, + 92, + 144, + 160, + 161, + 2, + 97, + 167, + 141, + 125, + 241, + 164, + 167, + 37, + 72, + 38, + 12, + 145, + 159, + 160, + 38, + 82, + 13, + 63, + 68, + 71, + 226, + 35, + 111, + 158, + 153, + 154, + 101, + 50, + 205, + 206, + 92, + 165, + 209, + 198, + 217, + 165, + 167, + 97, + 220, + 115, + 218, + 133, + 112, + 243, + 239, + 238, + 241, + 214, + 135, + 169, + 190, + 173, + 133, + 171, + 208, + 32, + 125, + 44, + 237, + 86, + 87, + 178, + 85, + 86, + 179, + 84, + 85, + 180, + 83, + 84, + 181, + 201, + 83, + 182, + 137, + 93, + 132, + 76, + 62, + 183, + 61, + 76, + 184, + 57, + 61, + 185, + 212, + 57, + 186, + 214, + 207, + 187, + 34, + 143, + 156, + 79, + 239, + 237, + 123, + 137, + 177, + 44, + 1, + 4, + 201, + 194, + 32, + 64, + 102, + 129, + 213, + 215, + 138, + 59, + 166, + 219, + 242, + 99, + 97, + 2, + 94, + 141, + 75, + 59, + 235, + 24, + 110, + 228, + 25, + 130, + 226, + 23, + 24, + 229, + 22, + 23, + 230, + 26, + 22, + 231, + 112, + 26, + 232, + 189, + 190, + 243, + 221, + 56, + 190, + 28, + 56, + 221, + 27, + 28, + 222, + 29, + 27, + 223, + 30, + 29, + 224, + 247, + 30, + 225, + 238, + 79, + 20, + 166, + 59, + 75, + 60, + 75, + 240, + 147, + 177, + 215, + 20, + 79, + 166, + 187, + 147, + 213, + 112, + 233, + 244, + 233, + 128, + 245, + 128, + 114, + 188, + 114, + 217, + 174, + 131, + 115, + 220, + 217, + 198, + 236, + 198, + 131, + 134, + 177, + 132, + 58, + 143, + 35, + 124, + 110, + 163, + 7, + 228, + 110, + 25, + 356, + 389, + 368, + 11, + 302, + 267, + 452, + 350, + 349, + 302, + 303, + 269, + 357, + 343, + 277, + 452, + 453, + 357, + 333, + 332, + 297, + 175, + 152, + 377, + 384, + 398, + 382, + 347, + 348, + 330, + 303, + 304, + 270, + 9, + 336, + 337, + 278, + 279, + 360, + 418, + 262, + 431, + 304, + 408, + 409, + 310, + 415, + 407, + 270, + 409, + 410, + 450, + 348, + 347, + 422, + 430, + 434, + 313, + 314, + 17, + 306, + 307, + 375, + 387, + 388, + 260, + 286, + 414, + 398, + 335, + 406, + 418, + 364, + 367, + 416, + 423, + 358, + 327, + 251, + 284, + 298, + 281, + 5, + 4, + 373, + 374, + 253, + 307, + 320, + 321, + 425, + 427, + 411, + 421, + 313, + 18, + 321, + 405, + 406, + 320, + 404, + 405, + 315, + 16, + 17, + 426, + 425, + 266, + 377, + 400, + 369, + 322, + 391, + 269, + 417, + 465, + 464, + 386, + 257, + 258, + 466, + 260, + 388, + 456, + 399, + 419, + 284, + 332, + 333, + 417, + 285, + 8, + 346, + 340, + 261, + 413, + 441, + 285, + 327, + 460, + 328, + 355, + 371, + 329, + 392, + 439, + 438, + 382, + 341, + 256, + 429, + 420, + 360, + 364, + 394, + 379, + 277, + 343, + 437, + 443, + 444, + 283, + 275, + 440, + 363, + 431, + 262, + 369, + 297, + 338, + 337, + 273, + 375, + 321, + 450, + 451, + 349, + 446, + 342, + 467, + 293, + 334, + 282, + 458, + 461, + 462, + 276, + 353, + 383, + 308, + 324, + 325, + 276, + 300, + 293, + 372, + 345, + 447, + 382, + 398, + 362, + 352, + 345, + 340, + 274, + 1, + 19, + 456, + 248, + 281, + 436, + 427, + 425, + 381, + 256, + 252, + 269, + 391, + 393, + 200, + 199, + 428, + 266, + 330, + 329, + 287, + 273, + 422, + 250, + 462, + 328, + 258, + 286, + 384, + 265, + 353, + 342, + 387, + 259, + 257, + 424, + 431, + 430, + 342, + 353, + 276, + 273, + 335, + 424, + 292, + 325, + 307, + 366, + 447, + 345, + 271, + 303, + 302, + 423, + 266, + 371, + 294, + 455, + 460, + 279, + 278, + 294, + 271, + 272, + 304, + 432, + 434, + 427, + 272, + 407, + 408, + 394, + 430, + 431, + 395, + 369, + 400, + 334, + 333, + 299, + 351, + 417, + 168, + 352, + 280, + 411, + 325, + 319, + 320, + 295, + 296, + 336, + 319, + 403, + 404, + 330, + 348, + 349, + 293, + 298, + 333, + 323, + 454, + 447, + 15, + 16, + 315, + 358, + 429, + 279, + 14, + 15, + 316, + 285, + 336, + 9, + 329, + 349, + 350, + 374, + 380, + 252, + 318, + 402, + 403, + 6, + 197, + 419, + 318, + 319, + 325, + 367, + 364, + 365, + 435, + 367, + 397, + 344, + 438, + 439, + 272, + 271, + 311, + 195, + 5, + 281, + 273, + 287, + 291, + 396, + 428, + 199, + 311, + 271, + 268, + 283, + 444, + 445, + 373, + 254, + 339, + 263, + 466, + 249, + 282, + 334, + 296, + 449, + 347, + 346, + 264, + 447, + 454, + 336, + 296, + 299, + 338, + 10, + 151, + 278, + 439, + 455, + 292, + 407, + 415, + 358, + 371, + 355, + 340, + 345, + 372, + 390, + 249, + 466, + 346, + 347, + 280, + 442, + 443, + 282, + 19, + 94, + 370, + 441, + 442, + 295, + 248, + 419, + 197, + 263, + 255, + 359, + 440, + 275, + 274, + 300, + 383, + 368, + 351, + 412, + 465, + 263, + 467, + 466, + 301, + 368, + 389, + 380, + 374, + 386, + 395, + 378, + 379, + 412, + 351, + 419, + 436, + 426, + 322, + 373, + 390, + 388, + 2, + 164, + 393, + 370, + 462, + 461, + 164, + 0, + 267, + 302, + 11, + 12, + 374, + 373, + 387, + 268, + 12, + 13, + 293, + 300, + 301, + 446, + 261, + 340, + 385, + 384, + 381, + 330, + 266, + 425, + 426, + 423, + 391, + 429, + 355, + 437, + 391, + 327, + 326, + 440, + 457, + 438, + 341, + 382, + 362, + 459, + 457, + 461, + 434, + 430, + 394, + 414, + 463, + 362, + 396, + 369, + 262, + 354, + 461, + 457, + 316, + 403, + 402, + 315, + 404, + 403, + 314, + 405, + 404, + 313, + 406, + 405, + 421, + 418, + 406, + 366, + 401, + 361, + 306, + 408, + 407, + 291, + 409, + 408, + 287, + 410, + 409, + 432, + 436, + 410, + 434, + 416, + 411, + 264, + 368, + 383, + 309, + 438, + 457, + 352, + 376, + 401, + 274, + 275, + 4, + 421, + 428, + 262, + 294, + 327, + 358, + 433, + 416, + 367, + 289, + 455, + 439, + 462, + 370, + 326, + 2, + 326, + 370, + 305, + 460, + 455, + 254, + 449, + 448, + 255, + 261, + 446, + 253, + 450, + 449, + 252, + 451, + 450, + 256, + 452, + 451, + 341, + 453, + 452, + 413, + 464, + 463, + 441, + 413, + 414, + 258, + 442, + 441, + 257, + 443, + 442, + 259, + 444, + 443, + 260, + 445, + 444, + 467, + 342, + 445, + 459, + 458, + 250, + 289, + 392, + 290, + 290, + 328, + 460, + 376, + 433, + 435, + 250, + 290, + 392, + 411, + 416, + 433, + 341, + 463, + 464, + 453, + 464, + 465, + 357, + 465, + 412, + 343, + 412, + 399, + 360, + 363, + 440, + 437, + 399, + 456, + 420, + 456, + 363, + 401, + 435, + 288, + 372, + 383, + 353, + 339, + 255, + 249, + 448, + 261, + 255, + 133, + 243, + 190, + 133, + 155, + 112, + 33, + 246, + 247, + 33, + 130, + 25, + 398, + 384, + 286, + 362, + 398, + 414, + 362, + 463, + 341, + 263, + 359, + 467, + 263, + 249, + 255, + 466, + 467, + 260, + 75, + 60, + 166, + 238, + 239, + 79, + 162, + 127, + 139, + 72, + 11, + 37, + 121, + 232, + 120, + 73, + 72, + 39, + 114, + 128, + 47, + 233, + 232, + 128, + 103, + 104, + 67, + 152, + 175, + 148, + 173, + 157, + 155, + 119, + 118, + 101, + 74, + 73, + 40, + 107, + 9, + 108, + 49, + 48, + 131, + 32, + 194, + 211, + 184, + 74, + 185, + 191, + 80, + 183, + 185, + 40, + 186, + 119, + 230, + 118, + 210, + 202, + 214, + 84, + 83, + 17, + 77, + 76, + 146, + 161, + 160, + 30, + 190, + 56, + 173, + 182, + 106, + 194, + 138, + 135, + 192, + 129, + 203, + 98, + 54, + 21, + 68, + 5, + 51, + 4, + 145, + 144, + 23, + 90, + 77, + 91, + 207, + 205, + 187, + 83, + 201, + 18, + 181, + 91, + 182, + 180, + 90, + 181, + 16, + 85, + 17, + 205, + 206, + 36, + 176, + 148, + 140, + 165, + 92, + 39, + 245, + 193, + 244, + 27, + 159, + 28, + 30, + 247, + 161, + 174, + 236, + 196, + 103, + 54, + 104, + 55, + 193, + 8, + 111, + 117, + 31, + 221, + 189, + 55, + 240, + 98, + 99, + 142, + 126, + 100, + 219, + 166, + 218, + 112, + 155, + 26, + 198, + 209, + 131, + 169, + 135, + 150, + 114, + 47, + 217, + 224, + 223, + 53, + 220, + 45, + 134, + 32, + 211, + 140, + 109, + 67, + 108, + 146, + 43, + 91, + 231, + 230, + 120, + 113, + 226, + 247, + 105, + 63, + 52, + 241, + 238, + 242, + 124, + 46, + 156, + 95, + 78, + 96, + 70, + 46, + 63, + 116, + 143, + 227, + 116, + 123, + 111, + 1, + 44, + 19, + 3, + 236, + 51, + 207, + 216, + 205, + 26, + 154, + 22, + 165, + 39, + 167, + 199, + 200, + 208, + 101, + 36, + 100, + 43, + 57, + 202, + 242, + 20, + 99, + 56, + 28, + 157, + 124, + 35, + 113, + 29, + 160, + 27, + 211, + 204, + 210, + 124, + 113, + 46, + 106, + 43, + 204, + 96, + 62, + 77, + 227, + 137, + 116, + 73, + 41, + 72, + 36, + 203, + 142, + 235, + 64, + 240, + 48, + 49, + 64, + 42, + 41, + 74, + 214, + 212, + 207, + 183, + 42, + 184, + 210, + 169, + 211, + 140, + 170, + 176, + 104, + 105, + 69, + 193, + 122, + 168, + 50, + 123, + 187, + 89, + 96, + 90, + 66, + 65, + 107, + 179, + 89, + 180, + 119, + 101, + 120, + 68, + 63, + 104, + 234, + 93, + 227, + 16, + 15, + 85, + 209, + 129, + 49, + 15, + 14, + 86, + 107, + 55, + 9, + 120, + 100, + 121, + 153, + 145, + 22, + 178, + 88, + 179, + 197, + 6, + 196, + 89, + 88, + 96, + 135, + 138, + 136, + 138, + 215, + 172, + 218, + 115, + 219, + 41, + 42, + 81, + 5, + 195, + 51, + 57, + 43, + 61, + 208, + 171, + 199, + 41, + 81, + 38, + 224, + 53, + 225, + 24, + 144, + 110, + 105, + 52, + 66, + 118, + 229, + 117, + 227, + 34, + 234, + 66, + 107, + 69, + 10, + 109, + 151, + 219, + 48, + 235, + 183, + 62, + 191, + 142, + 129, + 126, + 116, + 111, + 143, + 7, + 163, + 246, + 118, + 117, + 50, + 223, + 222, + 52, + 94, + 19, + 141, + 222, + 221, + 65, + 196, + 3, + 197, + 45, + 220, + 44, + 156, + 70, + 139, + 188, + 122, + 245, + 139, + 71, + 162, + 145, + 153, + 159, + 149, + 170, + 150, + 122, + 188, + 196, + 206, + 216, + 92, + 163, + 144, + 161, + 164, + 2, + 167, + 242, + 141, + 241, + 0, + 164, + 37, + 11, + 72, + 12, + 144, + 145, + 160, + 12, + 38, + 13, + 70, + 63, + 71, + 31, + 226, + 111, + 157, + 158, + 154, + 36, + 101, + 205, + 203, + 206, + 165, + 126, + 209, + 217, + 98, + 165, + 97, + 237, + 220, + 218, + 237, + 239, + 241, + 210, + 214, + 169, + 140, + 171, + 32, + 241, + 125, + 237, + 179, + 86, + 178, + 180, + 85, + 179, + 181, + 84, + 180, + 182, + 83, + 181, + 194, + 201, + 182, + 177, + 137, + 132, + 184, + 76, + 183, + 185, + 61, + 184, + 186, + 57, + 185, + 216, + 212, + 186, + 192, + 214, + 187, + 139, + 34, + 156, + 218, + 79, + 237, + 147, + 123, + 177, + 45, + 44, + 4, + 208, + 201, + 32, + 98, + 64, + 129, + 192, + 213, + 138, + 235, + 59, + 219, + 141, + 242, + 97, + 97, + 2, + 141, + 240, + 75, + 235, + 229, + 24, + 228, + 31, + 25, + 226, + 230, + 23, + 229, + 231, + 22, + 230, + 232, + 26, + 231, + 233, + 112, + 232, + 244, + 189, + 243, + 189, + 221, + 190, + 222, + 28, + 221, + 223, + 27, + 222, + 224, + 29, + 223, + 225, + 30, + 224, + 113, + 247, + 225, + 99, + 60, + 240, + 213, + 147, + 215, + 60, + 20, + 166, + 192, + 187, + 213, + 243, + 112, + 244, + 244, + 233, + 245, + 245, + 128, + 188, + 188, + 114, + 174, + 134, + 131, + 220, + 174, + 217, + 236, + 236, + 198, + 134, + 215, + 177, + 58, + 156, + 143, + 124, + 25, + 110, + 7, + 31, + 228, + 25, + 264, + 356, + 368, + 0, + 11, + 267, + 451, + 452, + 349, + 267, + 302, + 269, + 350, + 357, + 277, + 350, + 452, + 357, + 299, + 333, + 297, + 396, + 175, + 377, + 381, + 384, + 382, + 280, + 347, + 330, + 269, + 303, + 270, + 151, + 9, + 337, + 344, + 278, + 360, + 424, + 418, + 431, + 270, + 304, + 409, + 272, + 310, + 407, + 322, + 270, + 410, + 449, + 450, + 347, + 432, + 422, + 434, + 18, + 313, + 17, + 291, + 306, + 375, + 259, + 387, + 260, + 424, + 335, + 418, + 434, + 364, + 416, + 391, + 423, + 327, + 301, + 251, + 298, + 275, + 281, + 4, + 254, + 373, + 253, + 375, + 307, + 321, + 280, + 425, + 411, + 200, + 421, + 18, + 335, + 321, + 406, + 321, + 320, + 405, + 314, + 315, + 17, + 423, + 426, + 266, + 396, + 377, + 369, + 270, + 322, + 269, + 413, + 417, + 464, + 385, + 386, + 258, + 248, + 456, + 419, + 298, + 284, + 333, + 168, + 417, + 8, + 448, + 346, + 261, + 417, + 413, + 285, + 326, + 327, + 328, + 277, + 355, + 329, + 309, + 392, + 438, + 381, + 382, + 256, + 279, + 429, + 360, + 365, + 364, + 379, + 355, + 277, + 437, + 282, + 443, + 283, + 281, + 275, + 363, + 395, + 431, + 369, + 299, + 297, + 337, + 335, + 273, + 321, + 348, + 450, + 349, + 359, + 446, + 467, + 283, + 293, + 282, + 250, + 458, + 462, + 300, + 276, + 383, + 292, + 308, + 325, + 283, + 276, + 293, + 264, + 372, + 447, + 346, + 352, + 340, + 354, + 274, + 19, + 363, + 456, + 281, + 426, + 436, + 425, + 380, + 381, + 252, + 267, + 269, + 393, + 421, + 200, + 428, + 371, + 266, + 329, + 432, + 287, + 422, + 290, + 250, + 328, + 385, + 258, + 384, + 446, + 265, + 342, + 386, + 387, + 257, + 422, + 424, + 430, + 445, + 342, + 276, + 422, + 273, + 424, + 306, + 292, + 307, + 352, + 366, + 345, + 268, + 271, + 302, + 358, + 423, + 371, + 327, + 294, + 460, + 331, + 279, + 294, + 303, + 271, + 304, + 436, + 432, + 427, + 304, + 272, + 408, + 395, + 394, + 431, + 378, + 395, + 400, + 296, + 334, + 299, + 6, + 351, + 168, + 376, + 352, + 411, + 307, + 325, + 320, + 285, + 295, + 336, + 320, + 319, + 404, + 329, + 330, + 349, + 334, + 293, + 333, + 366, + 323, + 447, + 316, + 15, + 315, + 331, + 358, + 279, + 317, + 14, + 316, + 8, + 285, + 9, + 277, + 329, + 350, + 253, + 374, + 252, + 319, + 318, + 403, + 351, + 6, + 419, + 324, + 318, + 325, + 397, + 367, + 365, + 288, + 435, + 397, + 278, + 344, + 439, + 310, + 272, + 311, + 248, + 195, + 281, + 375, + 273, + 291, + 175, + 396, + 199, + 312, + 311, + 268, + 276, + 283, + 445, + 390, + 373, + 339, + 295, + 282, + 296, + 448, + 449, + 346, + 356, + 264, + 454, + 337, + 336, + 299, + 337, + 338, + 151, + 294, + 278, + 455, + 308, + 292, + 415, + 429, + 358, + 355, + 265, + 340, + 372, + 388, + 390, + 466, + 352, + 346, + 280, + 295, + 442, + 282, + 354, + 19, + 370, + 285, + 441, + 295, + 195, + 248, + 197, + 457, + 440, + 274, + 301, + 300, + 368, + 417, + 351, + 465, + 251, + 301, + 389, + 385, + 380, + 386, + 394, + 395, + 379, + 399, + 412, + 419, + 410, + 436, + 322, + 387, + 373, + 388, + 326, + 2, + 393, + 354, + 370, + 461, + 393, + 164, + 267, + 268, + 302, + 12, + 386, + 374, + 387, + 312, + 268, + 13, + 298, + 293, + 301, + 265, + 446, + 340, + 380, + 385, + 381, + 280, + 330, + 425, + 322, + 426, + 391, + 420, + 429, + 437, + 393, + 391, + 326, + 344, + 440, + 438, + 458, + 459, + 461, + 364, + 434, + 394, + 428, + 396, + 262, + 274, + 354, + 457, + 317, + 316, + 402, + 316, + 315, + 403, + 315, + 314, + 404, + 314, + 313, + 405, + 313, + 421, + 406, + 323, + 366, + 361, + 292, + 306, + 407, + 306, + 291, + 408, + 291, + 287, + 409, + 287, + 432, + 410, + 427, + 434, + 411, + 372, + 264, + 383, + 459, + 309, + 457, + 366, + 352, + 401, + 1, + 274, + 4, + 418, + 421, + 262, + 331, + 294, + 358, + 435, + 433, + 367, + 392, + 289, + 439, + 328, + 462, + 326, + 94, + 2, + 370, + 289, + 305, + 455, + 339, + 254, + 448, + 359, + 255, + 446, + 254, + 253, + 449, + 253, + 252, + 450, + 252, + 256, + 451, + 256, + 341, + 452, + 414, + 413, + 463, + 286, + 441, + 414, + 286, + 258, + 441, + 258, + 257, + 442, + 257, + 259, + 443, + 259, + 260, + 444, + 260, + 467, + 445, + 309, + 459, + 250, + 305, + 289, + 290, + 305, + 290, + 460, + 401, + 376, + 435, + 309, + 250, + 392, + 376, + 411, + 433, + 453, + 341, + 464, + 357, + 453, + 465, + 343, + 357, + 412, + 437, + 343, + 399, + 344, + 360, + 440, + 420, + 437, + 456, + 360, + 420, + 363, + 361, + 401, + 288, + 265, + 372, + 353, + 390, + 339, + 249, + 339, + 448, + 255 +] diff --git a/src/processors/face-mask/data/s-face-wo-eyes-tris.json b/src/processors/face-mask/data/s-face-wo-eyes-tris.json new file mode 100644 index 000000000..d3940dcb6 --- /dev/null +++ b/src/processors/face-mask/data/s-face-wo-eyes-tris.json @@ -0,0 +1,2558 @@ +[ + 127, + 34, + 139, + 11, + 0, + 37, + 232, + 231, + 120, + 72, + 37, + 39, + 128, + 121, + 47, + 232, + 121, + 128, + 104, + 69, + 67, + 175, + 171, + 148, + 118, + 50, + 101, + 73, + 39, + 40, + 9, + 151, + 108, + 48, + 115, + 131, + 194, + 204, + 211, + 74, + 40, + 185, + 80, + 42, + 183, + 40, + 92, + 186, + 230, + 229, + 118, + 202, + 212, + 214, + 83, + 18, + 17, + 76, + 61, + 146, + 160, + 29, + 30, + 56, + 157, + 173, + 106, + 204, + 194, + 135, + 214, + 192, + 203, + 165, + 98, + 21, + 71, + 68, + 51, + 45, + 4, + 144, + 24, + 23, + 77, + 146, + 91, + 205, + 50, + 187, + 201, + 200, + 18, + 91, + 106, + 182, + 90, + 91, + 181, + 85, + 84, + 17, + 206, + 203, + 36, + 148, + 171, + 140, + 92, + 40, + 39, + 193, + 189, + 244, + 159, + 158, + 28, + 247, + 246, + 161, + 236, + 3, + 196, + 54, + 68, + 104, + 193, + 168, + 8, + 117, + 228, + 31, + 189, + 193, + 55, + 98, + 97, + 99, + 126, + 47, + 100, + 166, + 79, + 218, + 155, + 154, + 26, + 209, + 49, + 131, + 135, + 136, + 150, + 47, + 126, + 217, + 223, + 52, + 53, + 45, + 51, + 134, + 211, + 170, + 140, + 67, + 69, + 108, + 43, + 106, + 91, + 230, + 119, + 120, + 226, + 130, + 247, + 63, + 53, + 52, + 238, + 20, + 242, + 46, + 70, + 156, + 78, + 62, + 96, + 46, + 53, + 63, + 143, + 34, + 227, + 123, + 117, + 111, + 44, + 125, + 19, + 236, + 134, + 51, + 216, + 206, + 205, + 154, + 153, + 22, + 39, + 37, + 167, + 200, + 201, + 208, + 36, + 142, + 100, + 57, + 212, + 202, + 20, + 60, + 99, + 28, + 158, + 157, + 35, + 226, + 113, + 160, + 159, + 27, + 204, + 202, + 210, + 113, + 225, + 46, + 43, + 202, + 204, + 62, + 76, + 77, + 137, + 123, + 116, + 41, + 38, + 72, + 203, + 129, + 142, + 64, + 98, + 240, + 49, + 102, + 64, + 41, + 73, + 74, + 212, + 216, + 207, + 42, + 74, + 184, + 169, + 170, + 211, + 170, + 149, + 176, + 105, + 66, + 69, + 122, + 6, + 168, + 123, + 147, + 187, + 96, + 77, + 90, + 65, + 55, + 107, + 89, + 90, + 180, + 101, + 100, + 120, + 63, + 105, + 104, + 93, + 137, + 227, + 15, + 86, + 85, + 129, + 102, + 49, + 14, + 87, + 86, + 55, + 8, + 9, + 100, + 47, + 121, + 145, + 23, + 22, + 88, + 89, + 179, + 6, + 122, + 196, + 88, + 95, + 96, + 138, + 172, + 136, + 215, + 58, + 172, + 115, + 48, + 219, + 42, + 80, + 81, + 195, + 3, + 51, + 43, + 146, + 61, + 171, + 175, + 199, + 81, + 82, + 38, + 53, + 46, + 225, + 144, + 163, + 110, + 52, + 65, + 66, + 229, + 228, + 117, + 34, + 127, + 234, + 107, + 108, + 69, + 109, + 108, + 151, + 48, + 64, + 235, + 62, + 78, + 191, + 129, + 209, + 126, + 111, + 35, + 143, + 117, + 123, + 50, + 222, + 65, + 52, + 19, + 125, + 141, + 221, + 55, + 65, + 3, + 195, + 197, + 25, + 7, + 33, + 220, + 237, + 44, + 70, + 71, + 139, + 122, + 193, + 245, + 247, + 130, + 33, + 71, + 21, + 162, + 170, + 169, + 150, + 188, + 174, + 196, + 216, + 186, + 92, + 2, + 97, + 167, + 141, + 125, + 241, + 164, + 167, + 37, + 72, + 38, + 12, + 38, + 82, + 13, + 63, + 68, + 71, + 226, + 35, + 111, + 101, + 50, + 205, + 206, + 92, + 165, + 209, + 198, + 217, + 165, + 167, + 97, + 220, + 115, + 218, + 133, + 112, + 243, + 239, + 238, + 241, + 214, + 135, + 169, + 190, + 173, + 133, + 171, + 208, + 32, + 125, + 44, + 237, + 86, + 87, + 178, + 85, + 86, + 179, + 84, + 85, + 180, + 83, + 84, + 181, + 201, + 83, + 182, + 137, + 93, + 132, + 76, + 62, + 183, + 61, + 76, + 184, + 57, + 61, + 185, + 212, + 57, + 186, + 214, + 207, + 187, + 34, + 143, + 156, + 79, + 239, + 237, + 123, + 137, + 177, + 44, + 1, + 4, + 201, + 194, + 32, + 64, + 102, + 129, + 213, + 215, + 138, + 59, + 166, + 219, + 242, + 99, + 97, + 2, + 94, + 141, + 75, + 59, + 235, + 24, + 110, + 228, + 25, + 130, + 226, + 23, + 24, + 229, + 22, + 23, + 230, + 26, + 22, + 231, + 112, + 26, + 232, + 189, + 190, + 243, + 221, + 56, + 190, + 28, + 56, + 221, + 27, + 28, + 222, + 29, + 27, + 223, + 30, + 29, + 224, + 247, + 30, + 225, + 238, + 79, + 20, + 166, + 59, + 75, + 60, + 75, + 240, + 147, + 177, + 215, + 20, + 79, + 166, + 187, + 147, + 213, + 112, + 233, + 244, + 233, + 128, + 245, + 128, + 114, + 188, + 114, + 217, + 174, + 131, + 115, + 220, + 217, + 198, + 236, + 198, + 131, + 134, + 177, + 132, + 58, + 143, + 35, + 124, + 110, + 163, + 7, + 228, + 110, + 25, + 356, + 389, + 368, + 11, + 302, + 267, + 452, + 350, + 349, + 302, + 303, + 269, + 357, + 343, + 277, + 452, + 453, + 357, + 333, + 332, + 297, + 175, + 152, + 377, + 347, + 348, + 330, + 303, + 304, + 270, + 9, + 336, + 337, + 278, + 279, + 360, + 418, + 262, + 431, + 304, + 408, + 409, + 310, + 415, + 407, + 270, + 409, + 410, + 450, + 348, + 347, + 422, + 430, + 434, + 313, + 314, + 17, + 306, + 307, + 375, + 387, + 388, + 260, + 286, + 414, + 398, + 335, + 406, + 418, + 364, + 367, + 416, + 423, + 358, + 327, + 251, + 284, + 298, + 281, + 5, + 4, + 373, + 374, + 253, + 307, + 320, + 321, + 425, + 427, + 411, + 421, + 313, + 18, + 321, + 405, + 406, + 320, + 404, + 405, + 315, + 16, + 17, + 426, + 425, + 266, + 377, + 400, + 369, + 322, + 391, + 269, + 417, + 465, + 464, + 386, + 257, + 258, + 466, + 260, + 388, + 456, + 399, + 419, + 284, + 332, + 333, + 417, + 285, + 8, + 346, + 340, + 261, + 413, + 441, + 285, + 327, + 460, + 328, + 355, + 371, + 329, + 392, + 439, + 438, + 382, + 341, + 256, + 429, + 420, + 360, + 364, + 394, + 379, + 277, + 343, + 437, + 443, + 444, + 283, + 275, + 440, + 363, + 431, + 262, + 369, + 297, + 338, + 337, + 273, + 375, + 321, + 450, + 451, + 349, + 446, + 342, + 467, + 293, + 334, + 282, + 458, + 461, + 462, + 276, + 353, + 383, + 308, + 324, + 325, + 276, + 300, + 293, + 372, + 345, + 447, + 352, + 345, + 340, + 274, + 1, + 19, + 456, + 248, + 281, + 436, + 427, + 425, + 381, + 256, + 252, + 269, + 391, + 393, + 200, + 199, + 428, + 266, + 330, + 329, + 287, + 273, + 422, + 250, + 462, + 328, + 258, + 286, + 384, + 265, + 353, + 342, + 387, + 259, + 257, + 424, + 431, + 430, + 342, + 353, + 276, + 273, + 335, + 424, + 292, + 325, + 307, + 366, + 447, + 345, + 271, + 303, + 302, + 423, + 266, + 371, + 294, + 455, + 460, + 279, + 278, + 294, + 271, + 272, + 304, + 432, + 434, + 427, + 272, + 407, + 408, + 394, + 430, + 431, + 395, + 369, + 400, + 334, + 333, + 299, + 351, + 417, + 168, + 352, + 280, + 411, + 325, + 319, + 320, + 295, + 296, + 336, + 319, + 403, + 404, + 330, + 348, + 349, + 293, + 298, + 333, + 323, + 454, + 447, + 15, + 16, + 315, + 358, + 429, + 279, + 14, + 15, + 316, + 285, + 336, + 9, + 329, + 349, + 350, + 374, + 380, + 252, + 318, + 402, + 403, + 6, + 197, + 419, + 318, + 319, + 325, + 367, + 364, + 365, + 435, + 367, + 397, + 344, + 438, + 439, + 272, + 271, + 311, + 195, + 5, + 281, + 273, + 287, + 291, + 396, + 428, + 199, + 311, + 271, + 268, + 283, + 444, + 445, + 373, + 254, + 339, + 282, + 334, + 296, + 449, + 347, + 346, + 264, + 447, + 454, + 336, + 296, + 299, + 338, + 10, + 151, + 278, + 439, + 455, + 292, + 407, + 415, + 358, + 371, + 355, + 340, + 345, + 372, + 346, + 347, + 280, + 442, + 443, + 282, + 19, + 94, + 370, + 441, + 442, + 295, + 248, + 419, + 197, + 263, + 255, + 359, + 440, + 275, + 274, + 300, + 383, + 368, + 351, + 412, + 465, + 263, + 467, + 466, + 301, + 368, + 389, + 395, + 378, + 379, + 412, + 351, + 419, + 436, + 426, + 322, + 2, + 164, + 393, + 370, + 462, + 461, + 164, + 0, + 267, + 302, + 11, + 12, + 268, + 12, + 13, + 293, + 300, + 301, + 446, + 261, + 340, + 330, + 266, + 425, + 426, + 423, + 391, + 429, + 355, + 437, + 391, + 327, + 326, + 440, + 457, + 438, + 341, + 382, + 362, + 459, + 457, + 461, + 434, + 430, + 394, + 414, + 463, + 362, + 396, + 369, + 262, + 354, + 461, + 457, + 316, + 403, + 402, + 315, + 404, + 403, + 314, + 405, + 404, + 313, + 406, + 405, + 421, + 418, + 406, + 366, + 401, + 361, + 306, + 408, + 407, + 291, + 409, + 408, + 287, + 410, + 409, + 432, + 436, + 410, + 434, + 416, + 411, + 264, + 368, + 383, + 309, + 438, + 457, + 352, + 376, + 401, + 274, + 275, + 4, + 421, + 428, + 262, + 294, + 327, + 358, + 433, + 416, + 367, + 289, + 455, + 439, + 462, + 370, + 326, + 2, + 326, + 370, + 305, + 460, + 455, + 254, + 449, + 448, + 255, + 261, + 446, + 253, + 450, + 449, + 252, + 451, + 450, + 256, + 452, + 451, + 341, + 453, + 452, + 413, + 464, + 463, + 441, + 413, + 414, + 258, + 442, + 441, + 257, + 443, + 442, + 259, + 444, + 443, + 260, + 445, + 444, + 467, + 342, + 445, + 459, + 458, + 250, + 289, + 392, + 290, + 290, + 328, + 460, + 376, + 433, + 435, + 250, + 290, + 392, + 411, + 416, + 433, + 341, + 463, + 464, + 453, + 464, + 465, + 357, + 465, + 412, + 343, + 412, + 399, + 360, + 363, + 440, + 437, + 399, + 456, + 420, + 456, + 363, + 401, + 435, + 288, + 372, + 383, + 353, + 339, + 255, + 249, + 448, + 261, + 255, + 133, + 243, + 190, + 133, + 155, + 112, + 33, + 246, + 247, + 33, + 130, + 25, + 398, + 384, + 286, + 362, + 398, + 414, + 362, + 463, + 341, + 263, + 359, + 467, + 263, + 249, + 255, + 466, + 467, + 260, + 75, + 60, + 166, + 238, + 239, + 79, + 162, + 127, + 139, + 72, + 11, + 37, + 121, + 232, + 120, + 73, + 72, + 39, + 114, + 128, + 47, + 233, + 232, + 128, + 103, + 104, + 67, + 152, + 175, + 148, + 119, + 118, + 101, + 74, + 73, + 40, + 107, + 9, + 108, + 49, + 48, + 131, + 32, + 194, + 211, + 184, + 74, + 185, + 191, + 80, + 183, + 185, + 40, + 186, + 119, + 230, + 118, + 210, + 202, + 214, + 84, + 83, + 17, + 77, + 76, + 146, + 161, + 160, + 30, + 190, + 56, + 173, + 182, + 106, + 194, + 138, + 135, + 192, + 129, + 203, + 98, + 54, + 21, + 68, + 5, + 51, + 4, + 145, + 144, + 23, + 90, + 77, + 91, + 207, + 205, + 187, + 83, + 201, + 18, + 181, + 91, + 182, + 180, + 90, + 181, + 16, + 85, + 17, + 205, + 206, + 36, + 176, + 148, + 140, + 165, + 92, + 39, + 245, + 193, + 244, + 27, + 159, + 28, + 30, + 247, + 161, + 174, + 236, + 196, + 103, + 54, + 104, + 55, + 193, + 8, + 111, + 117, + 31, + 221, + 189, + 55, + 240, + 98, + 99, + 142, + 126, + 100, + 219, + 166, + 218, + 112, + 155, + 26, + 198, + 209, + 131, + 169, + 135, + 150, + 114, + 47, + 217, + 224, + 223, + 53, + 220, + 45, + 134, + 32, + 211, + 140, + 109, + 67, + 108, + 146, + 43, + 91, + 231, + 230, + 120, + 113, + 226, + 247, + 105, + 63, + 52, + 241, + 238, + 242, + 124, + 46, + 156, + 95, + 78, + 96, + 70, + 46, + 63, + 116, + 143, + 227, + 116, + 123, + 111, + 1, + 44, + 19, + 3, + 236, + 51, + 207, + 216, + 205, + 26, + 154, + 22, + 165, + 39, + 167, + 199, + 200, + 208, + 101, + 36, + 100, + 43, + 57, + 202, + 242, + 20, + 99, + 56, + 28, + 157, + 124, + 35, + 113, + 29, + 160, + 27, + 211, + 204, + 210, + 124, + 113, + 46, + 106, + 43, + 204, + 96, + 62, + 77, + 227, + 137, + 116, + 73, + 41, + 72, + 36, + 203, + 142, + 235, + 64, + 240, + 48, + 49, + 64, + 42, + 41, + 74, + 214, + 212, + 207, + 183, + 42, + 184, + 210, + 169, + 211, + 140, + 170, + 176, + 104, + 105, + 69, + 193, + 122, + 168, + 50, + 123, + 187, + 89, + 96, + 90, + 66, + 65, + 107, + 179, + 89, + 180, + 119, + 101, + 120, + 68, + 63, + 104, + 234, + 93, + 227, + 16, + 15, + 85, + 209, + 129, + 49, + 15, + 14, + 86, + 107, + 55, + 9, + 120, + 100, + 121, + 153, + 145, + 22, + 178, + 88, + 179, + 197, + 6, + 196, + 89, + 88, + 96, + 135, + 138, + 136, + 138, + 215, + 172, + 218, + 115, + 219, + 41, + 42, + 81, + 5, + 195, + 51, + 57, + 43, + 61, + 208, + 171, + 199, + 41, + 81, + 38, + 224, + 53, + 225, + 24, + 144, + 110, + 105, + 52, + 66, + 118, + 229, + 117, + 227, + 34, + 234, + 66, + 107, + 69, + 10, + 109, + 151, + 219, + 48, + 235, + 183, + 62, + 191, + 142, + 129, + 126, + 116, + 111, + 143, + 118, + 117, + 50, + 223, + 222, + 52, + 94, + 19, + 141, + 222, + 221, + 65, + 196, + 3, + 197, + 45, + 220, + 44, + 156, + 70, + 139, + 188, + 122, + 245, + 139, + 71, + 162, + 149, + 170, + 150, + 122, + 188, + 196, + 206, + 216, + 92, + 164, + 2, + 167, + 242, + 141, + 241, + 0, + 164, + 37, + 11, + 72, + 12, + 12, + 38, + 13, + 70, + 63, + 71, + 31, + 226, + 111, + 36, + 101, + 205, + 203, + 206, + 165, + 126, + 209, + 217, + 98, + 165, + 97, + 237, + 220, + 218, + 237, + 239, + 241, + 210, + 214, + 169, + 140, + 171, + 32, + 241, + 125, + 237, + 179, + 86, + 178, + 180, + 85, + 179, + 181, + 84, + 180, + 182, + 83, + 181, + 194, + 201, + 182, + 177, + 137, + 132, + 184, + 76, + 183, + 185, + 61, + 184, + 186, + 57, + 185, + 216, + 212, + 186, + 192, + 214, + 187, + 139, + 34, + 156, + 218, + 79, + 237, + 147, + 123, + 177, + 45, + 44, + 4, + 208, + 201, + 32, + 98, + 64, + 129, + 192, + 213, + 138, + 235, + 59, + 219, + 141, + 242, + 97, + 97, + 2, + 141, + 240, + 75, + 235, + 229, + 24, + 228, + 31, + 25, + 226, + 230, + 23, + 229, + 231, + 22, + 230, + 232, + 26, + 231, + 233, + 112, + 232, + 244, + 189, + 243, + 189, + 221, + 190, + 222, + 28, + 221, + 223, + 27, + 222, + 224, + 29, + 223, + 225, + 30, + 224, + 113, + 247, + 225, + 99, + 60, + 240, + 213, + 147, + 215, + 60, + 20, + 166, + 192, + 187, + 213, + 243, + 112, + 244, + 244, + 233, + 245, + 245, + 128, + 188, + 188, + 114, + 174, + 134, + 131, + 220, + 174, + 217, + 236, + 236, + 198, + 134, + 215, + 177, + 58, + 156, + 143, + 124, + 25, + 110, + 7, + 31, + 228, + 25, + 264, + 356, + 368, + 0, + 11, + 267, + 451, + 452, + 349, + 267, + 302, + 269, + 350, + 357, + 277, + 350, + 452, + 357, + 299, + 333, + 297, + 396, + 175, + 377, + 280, + 347, + 330, + 269, + 303, + 270, + 151, + 9, + 337, + 344, + 278, + 360, + 424, + 418, + 431, + 270, + 304, + 409, + 272, + 310, + 407, + 322, + 270, + 410, + 449, + 450, + 347, + 432, + 422, + 434, + 18, + 313, + 17, + 291, + 306, + 375, + 259, + 387, + 260, + 424, + 335, + 418, + 434, + 364, + 416, + 391, + 423, + 327, + 301, + 251, + 298, + 275, + 281, + 4, + 254, + 373, + 253, + 375, + 307, + 321, + 280, + 425, + 411, + 200, + 421, + 18, + 335, + 321, + 406, + 321, + 320, + 405, + 314, + 315, + 17, + 423, + 426, + 266, + 396, + 377, + 369, + 270, + 322, + 269, + 413, + 417, + 464, + 385, + 386, + 258, + 248, + 456, + 419, + 298, + 284, + 333, + 168, + 417, + 8, + 448, + 346, + 261, + 417, + 413, + 285, + 326, + 327, + 328, + 277, + 355, + 329, + 309, + 392, + 438, + 381, + 382, + 256, + 279, + 429, + 360, + 365, + 364, + 379, + 355, + 277, + 437, + 282, + 443, + 283, + 281, + 275, + 363, + 395, + 431, + 369, + 299, + 297, + 337, + 335, + 273, + 321, + 348, + 450, + 349, + 359, + 446, + 467, + 283, + 293, + 282, + 250, + 458, + 462, + 300, + 276, + 383, + 292, + 308, + 325, + 283, + 276, + 293, + 264, + 372, + 447, + 346, + 352, + 340, + 354, + 274, + 19, + 363, + 456, + 281, + 426, + 436, + 425, + 380, + 381, + 252, + 267, + 269, + 393, + 421, + 200, + 428, + 371, + 266, + 329, + 432, + 287, + 422, + 290, + 250, + 328, + 385, + 258, + 384, + 446, + 265, + 342, + 386, + 387, + 257, + 422, + 424, + 430, + 445, + 342, + 276, + 422, + 273, + 424, + 306, + 292, + 307, + 352, + 366, + 345, + 268, + 271, + 302, + 358, + 423, + 371, + 327, + 294, + 460, + 331, + 279, + 294, + 303, + 271, + 304, + 436, + 432, + 427, + 304, + 272, + 408, + 395, + 394, + 431, + 378, + 395, + 400, + 296, + 334, + 299, + 6, + 351, + 168, + 376, + 352, + 411, + 307, + 325, + 320, + 285, + 295, + 336, + 320, + 319, + 404, + 329, + 330, + 349, + 334, + 293, + 333, + 366, + 323, + 447, + 316, + 15, + 315, + 331, + 358, + 279, + 317, + 14, + 316, + 8, + 285, + 9, + 277, + 329, + 350, + 253, + 374, + 252, + 319, + 318, + 403, + 351, + 6, + 419, + 324, + 318, + 325, + 397, + 367, + 365, + 288, + 435, + 397, + 278, + 344, + 439, + 310, + 272, + 311, + 248, + 195, + 281, + 375, + 273, + 291, + 175, + 396, + 199, + 312, + 311, + 268, + 276, + 283, + 445, + 390, + 373, + 339, + 295, + 282, + 296, + 448, + 449, + 346, + 356, + 264, + 454, + 337, + 336, + 299, + 337, + 338, + 151, + 294, + 278, + 455, + 308, + 292, + 415, + 429, + 358, + 355, + 265, + 340, + 372, + 352, + 346, + 280, + 295, + 442, + 282, + 354, + 19, + 370, + 285, + 441, + 295, + 195, + 248, + 197, + 457, + 440, + 274, + 301, + 300, + 368, + 417, + 351, + 465, + 251, + 301, + 389, + 394, + 395, + 379, + 399, + 412, + 419, + 410, + 436, + 322, + 326, + 2, + 393, + 354, + 370, + 461, + 393, + 164, + 267, + 268, + 302, + 12, + 312, + 268, + 13, + 298, + 293, + 301, + 265, + 446, + 340, + 280, + 330, + 425, + 322, + 426, + 391, + 420, + 429, + 437, + 393, + 391, + 326, + 344, + 440, + 438, + 458, + 459, + 461, + 364, + 434, + 394, + 428, + 396, + 262, + 274, + 354, + 457, + 317, + 316, + 402, + 316, + 315, + 403, + 315, + 314, + 404, + 314, + 313, + 405, + 313, + 421, + 406, + 323, + 366, + 361, + 292, + 306, + 407, + 306, + 291, + 408, + 291, + 287, + 409, + 287, + 432, + 410, + 427, + 434, + 411, + 372, + 264, + 383, + 459, + 309, + 457, + 366, + 352, + 401, + 1, + 274, + 4, + 418, + 421, + 262, + 331, + 294, + 358, + 435, + 433, + 367, + 392, + 289, + 439, + 328, + 462, + 326, + 94, + 2, + 370, + 289, + 305, + 455, + 339, + 254, + 448, + 359, + 255, + 446, + 254, + 253, + 449, + 253, + 252, + 450, + 252, + 256, + 451, + 256, + 341, + 452, + 414, + 413, + 463, + 286, + 441, + 414, + 286, + 258, + 441, + 258, + 257, + 442, + 257, + 259, + 443, + 259, + 260, + 444, + 260, + 467, + 445, + 309, + 459, + 250, + 305, + 289, + 290, + 305, + 290, + 460, + 401, + 376, + 435, + 309, + 250, + 392, + 376, + 411, + 433, + 453, + 341, + 464, + 357, + 453, + 465, + 343, + 357, + 412, + 437, + 343, + 399, + 344, + 360, + 440, + 420, + 437, + 456, + 360, + 420, + 363, + 361, + 401, + 288, + 265, + 372, + 353, + 390, + 339, + 249, + 339, + 448, + 255 +] diff --git a/src/processors/face-mask/shaders.ts b/src/processors/face-mask/shaders.ts new file mode 100644 index 000000000..b0759a8fa --- /dev/null +++ b/src/processors/face-mask/shaders.ts @@ -0,0 +1,82 @@ +// Shaders for Fill Color +export const vs_fill = ` + attribute vec4 a_Vertex; + uniform mat4 u_PMVMatrix; + void main (void) + { + gl_Position = u_PMVMatrix * a_Vertex; + } + `; + +export const fs_fill = ` + precision mediump float; + uniform vec4 u_Color; + + void main (void) + { + gl_FragColor = u_Color; + } +`; + +// Shaders for Texture +export const vs_tex = ` + attribute vec4 a_Vertex; + attribute vec2 a_TexCoord; + varying vec2 v_TexCoord; + uniform mat4 u_PMVMatrix; + + void main (void) + { + gl_Position = u_PMVMatrix * a_Vertex; + v_TexCoord = a_TexCoord; + } +`; + +export const fs_tex = ` + precision mediump float; + varying vec2 v_TexCoord; + uniform sampler2D u_sampler; + uniform vec4 u_Color; + + void main (void) + { + gl_FragColor = texture2D (u_sampler, v_TexCoord); + gl_FragColor *= u_Color; + } +`; + +// Shaders for Face mesh + +export const strVS = ` +attribute vec4 a_Vertex; +attribute vec2 a_TexCoord; +attribute float a_vtxalpha; +uniform mat4 u_PMVMatrix; +varying vec2 v_texcoord; +varying float v_vtxalpha; + +void main(void) +{ + gl_Position = u_PMVMatrix * a_Vertex; + v_texcoord = a_TexCoord; + v_vtxalpha = a_vtxalpha; +} +`; + +export const strFS = ` +precision mediump float; + +uniform vec3 u_color; +uniform float u_alpha; +varying vec2 v_texcoord; +varying float v_vtxalpha; +uniform sampler2D u_sampler; + +void main(void) +{ + vec3 color; + color = vec3(texture2D(u_sampler, v_texcoord)); + color *= u_color; + gl_FragColor = vec4(color, v_vtxalpha * u_alpha); +} +`; diff --git a/src/processors/face-mask/utils/Facemesh.ts b/src/processors/face-mask/utils/Facemesh.ts new file mode 100644 index 000000000..c4ef7e5df --- /dev/null +++ b/src/processors/face-mask/utils/Facemesh.ts @@ -0,0 +1,163 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +import { matrix_identity, matrix_mult } from './matrix'; +import { generate_shader } from './shaders'; +import s_face_tris from '../data/s-face-tris.json'; +import s_face_wo_eyes_tris from '../data/s-face-wo-eyes-tris.json'; +import face_countour_idx from '../data/face-contour-idx.json'; +import { strVS, strFS } from '../shaders'; + +export class Facemesh { + private gl: WebGL2RenderingContext; + + // Shaders + private strVS: string; + private strFS: string; + + // Vertices + private s_face_tris: number[]; + private s_face_wo_eyes_tris: number[]; + + private program: WebGLProgram; + private loc_vtx: number; + private loc_clr: number; + private loc_nrm: number; + private loc_uv: number; + private loc_smp: WebGLUniformLocation | null; + private loc_vtxalpha: number; + private loc_mtx_pmv: WebGLUniformLocation | null; + private loc_color: WebGLUniformLocation | null; + private loc_alpha: WebGLUniformLocation | null; + private matPrj: number[]; + private vbo_vtx: WebGLBuffer | null; + private vbo_uv: WebGLBuffer | null; + private vbo_idx: WebGLBuffer | null; + private vbo_idx2: WebGLBuffer | null; + private vbo_alpha: WebGLBuffer | null; + private vbo_alpha2: WebGLBuffer | null; + + constructor(gl: WebGL2RenderingContext) { + this.gl = gl; + + this.strVS = strVS; + this.strFS = strFS; + + /* + * Vertex indices are from: + * https://github.com/tensorflow/tfjs-models/blob/master/facemesh/src/keypoints.ts + */ + + this.s_face_tris = s_face_tris; + this.s_face_wo_eyes_tris = s_face_wo_eyes_tris; + + const { program, loc_vtx, loc_clr, loc_nrm, loc_uv, loc_smp } = generate_shader(this.gl, this.strVS, this.strFS); + this.program = program; + this.loc_vtx = loc_vtx; + this.loc_clr = loc_clr; + this.loc_nrm = loc_nrm; + this.loc_uv = loc_uv; + this.loc_smp = loc_smp; + + this.loc_vtxalpha = this.gl.getAttribLocation(this.program, 'a_vtxalpha'); + this.loc_mtx_pmv = this.gl.getUniformLocation(this.program, 'u_PMVMatrix'); + this.loc_color = this.gl.getUniformLocation(this.program, 'u_color'); + this.loc_alpha = this.gl.getUniformLocation(this.program, 'u_alpha'); + + this.matPrj = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 1, 0, 1]; + + this.vbo_vtx = this.gl.createBuffer(); + this.vbo_uv = this.gl.createBuffer(); + + this.vbo_idx = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.vbo_idx); + this.gl.bufferData(this.gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(this.s_face_tris), this.gl.STATIC_DRAW); + + this.vbo_idx2 = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.vbo_idx2); + this.gl.bufferData(this.gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(this.s_face_wo_eyes_tris), this.gl.STATIC_DRAW); + + this.vbo_alpha = this.create_vbo_alpha_array(this.s_face_tris); + this.vbo_alpha2 = this.create_vbo_alpha_array(this.s_face_wo_eyes_tris); + } + + create_vbo_alpha_array(tris: number[]): WebGLBuffer | null { + let vtx_counts = tris.length; + let alpha_array = new Array(vtx_counts); + + for (let i = 0; i < vtx_counts; i++) { + let alpha = 1.0; + for (let j = 0; j < face_countour_idx.length; j++) { + if (i === face_countour_idx[j]) { + alpha = 0.0; + break; + } + } + alpha_array[i] = alpha; + } + + let vbo_alpha = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vbo_alpha); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array(alpha_array), this.gl.STATIC_DRAW); + + return vbo_alpha; + } + + resize_facemesh_render(w: number, h: number) { + this.matPrj[0] = 2.0 / w; + this.matPrj[5] = -2.0 / h; + } + + draw_facemesh_tri_tex(texture: WebGLTexture, vtx: number[], uv: number[], color: number[], drill_eye_hole: boolean) { + let matMV = new Array(16); + let matPMV = new Array(16); + + this.gl.enable(this.gl.CULL_FACE); + + this.gl.useProgram(this.program); + + this.gl.enableVertexAttribArray(this.loc_vtx); + this.gl.enableVertexAttribArray(this.loc_uv); + this.gl.enableVertexAttribArray(this.loc_vtxalpha); + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vbo_vtx); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array(vtx), this.gl.STATIC_DRAW); + this.gl.vertexAttribPointer(this.loc_vtx, 3, this.gl.FLOAT, false, 0, 0); + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vbo_uv); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array(uv), this.gl.STATIC_DRAW); + this.gl.vertexAttribPointer(this.loc_uv, 2, this.gl.FLOAT, false, 0, 0); + + let vtx_counts; + if (drill_eye_hole) { + this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.vbo_idx2); + vtx_counts = this.s_face_wo_eyes_tris.length; + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vbo_alpha2); + this.gl.vertexAttribPointer(this.loc_vtxalpha, 1, this.gl.FLOAT, false, 0, 0); + } else { + this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.vbo_idx); + vtx_counts = this.s_face_tris.length; + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.vbo_alpha); + this.gl.vertexAttribPointer(this.loc_vtxalpha, 1, this.gl.FLOAT, false, 0, 0); + } + + matrix_identity(matMV); + matrix_mult(matPMV, this.matPrj, matMV); + + this.gl.uniformMatrix4fv(this.loc_mtx_pmv, false, matPMV); + this.gl.uniform3f(this.loc_color, color[0], color[1], color[2]); + this.gl.uniform1f(this.loc_alpha, color[3]); + + this.gl.enable(this.gl.BLEND); + + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + + this.gl.drawElements(this.gl.TRIANGLES, vtx_counts, this.gl.UNSIGNED_SHORT, 0); + + this.gl.disable(this.gl.BLEND); + this.gl.frontFace(this.gl.CCW); + } +} diff --git a/src/processors/face-mask/utils/Render2D.ts b/src/processors/face-mask/utils/Render2D.ts new file mode 100644 index 000000000..7fa28e195 --- /dev/null +++ b/src/processors/face-mask/utils/Render2D.ts @@ -0,0 +1,199 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +import { SHADER_NUM, varray, tarray0, tarray1, tarray2, tarray3 } from '../constants'; +import { fs_fill, fs_tex, vs_fill, vs_tex } from '../shaders'; +import { matrix_identity, matrix_mult, matrix_rotate, matrix_scale, matrix_translate } from './matrix'; +import { generate_shader, ShaderObject } from './shaders'; + +type TextureParams = { + type: number; + texture: WebGLTexture | null; + rot: number; + upsidedown: number; + px: number; + py: number; + blendfunc: number[]; + blendfunc_en: number; + user_texcoord?: number[]; + x: number; + y: number; + w: number; + h: number; + color: number[]; +}; + +export class Render2D { + private gl: WebGL2RenderingContext; + + private _textureParams: TextureParams; + + private _FLIP_V: number; + private _FLIP_H: number; + private _shaders: string[]; + + private _s_matprj: number[]; + + private _sobj: ShaderObject[]; + private _loc_mtx: (WebGLUniformLocation | null)[]; + private _loc_color: (WebGLUniformLocation | null)[]; + + private _vbo_vtx: WebGLBuffer | null; + private _vbo_uv: WebGLBuffer | null; + + constructor(gl: WebGL2RenderingContext) { + this.gl = gl; + + this._textureParams = { + type: 0, + rot: 0, + upsidedown: 0, + texture: null, + px: 0, + py: 0, + blendfunc: [], + blendfunc_en: 0, + color: [0, 0, 0, 0], + x: 0, + y: 0, + w: 0, + h: 0, + }; + + this._FLIP_V = 1; + this._FLIP_H = 2; + + this._s_matprj = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, 1.0, 0.0, 1.0]; + this._shaders = [vs_fill, fs_fill, vs_tex, fs_tex]; + + this._sobj = new Array(SHADER_NUM); + this._loc_mtx = new Array(SHADER_NUM); + this._loc_color = new Array(SHADER_NUM); + + for (let i = 0; i < SHADER_NUM; i++) { + const _sobj = generate_shader(this.gl, this._shaders[2 * i], this._shaders[2 * i + 1]); + this._loc_mtx[i] = this.gl.getUniformLocation(_sobj.program, 'u_PMVMatrix'); + this._loc_color[i] = this.gl.getUniformLocation(_sobj.program, 'u_Color'); + this._sobj[i] = _sobj; + } + + this._vbo_vtx = this.gl.createBuffer(); + this._vbo_uv = this.gl.createBuffer(); + } + + set_projection_matrix(w: number, h: number) { + this._s_matprj[0] = 2.0 / w; + this._s_matprj[5] = -2.0 / h; + } + + resize_viewport(w: number, h: number) { + this.set_projection_matrix(w, h); + } + + draw_2d_texture_in() { + const ttype = this._textureParams.type; + const texture = this._textureParams.texture; + const x = this._textureParams.x; + const y = this._textureParams.y; + const w = this._textureParams.w; + const h = this._textureParams.h; + const rot = this._textureParams.rot; + const _sobj = this._sobj[ttype]; + const matrix = new Array(16); + let uv: number[] = tarray0; + + this.gl.useProgram(_sobj.program); + this.gl.uniform1i(_sobj.loc_smp, 0); + + switch (ttype) { + case 0 /* fill */: + break; + case 1 /* tex */: + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + switch (this._textureParams.upsidedown) { + case this._FLIP_V: + uv = tarray1; + break; + case this._FLIP_H: + uv = tarray2; + break; + case this._FLIP_V | this._FLIP_H: + uv = tarray3; + break; + default: + uv = tarray0; + break; + } + break; + default: + break; + } + + if (this._textureParams.user_texcoord) { + uv = this._textureParams.user_texcoord; + } + + if (_sobj.loc_uv >= 0) { + this.gl.enableVertexAttribArray(_sobj.loc_uv); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this._vbo_uv); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array(uv), this.gl.STATIC_DRAW); + this.gl.vertexAttribPointer(_sobj.loc_uv, 2, this.gl.FLOAT, false, 0, 0); + } + + this.gl.enable(this.gl.BLEND); + + if (this._textureParams.blendfunc_en) { + this.gl.blendFuncSeparate( + this._textureParams.blendfunc[0], + this._textureParams.blendfunc[1], + this._textureParams.blendfunc[2], + this._textureParams.blendfunc[3] + ); + } else { + this.gl.blendFuncSeparate( + this.gl.SRC_ALPHA, + this.gl.ONE_MINUS_SRC_ALPHA, + this.gl.ONE, + this.gl.ONE_MINUS_SRC_ALPHA + ); + } + + matrix_identity(matrix); + matrix_translate(matrix, x, y, 0.0); + if (rot !== 0) { + let px = this._textureParams.px; + let py = this._textureParams.py; + matrix_translate(matrix, px, py, 0.0); + matrix_rotate(matrix, rot, 0.0, 0.0, 1.0); + matrix_translate(matrix, -px, -py, 0.0); + } + matrix_scale(matrix, w, h, 1.0); + matrix_mult(matrix, this._s_matprj, matrix); + + this.gl.uniformMatrix4fv(this._loc_mtx[ttype], false, matrix); + this.gl.uniform4fv(this._loc_color[ttype], this._textureParams.color); + + this.gl.enableVertexAttribArray(_sobj.loc_vtx); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this._vbo_vtx); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array(varray), this.gl.STATIC_DRAW); + this.gl.vertexAttribPointer(_sobj.loc_vtx, 2, this.gl.FLOAT, false, 0, 0); + + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + + this.gl.disable(this.gl.BLEND); + } + + draw_2d_texture(texture: WebGLTexture, x: number, y: number, w: number, h: number, upsidedown: any) { + this._textureParams.x = x; + this._textureParams.y = y; + this._textureParams.w = w; + this._textureParams.h = h; + this._textureParams.texture = texture; + this._textureParams.type = 1; + this._textureParams.color = [1.0, 1.0, 1.0, 1.0]; + this._textureParams.upsidedown = upsidedown; + + this.draw_2d_texture_in(); + } +} diff --git a/src/processors/face-mask/utils/matrix.ts b/src/processors/face-mask/utils/matrix.ts new file mode 100644 index 000000000..a46dbe9f2 --- /dev/null +++ b/src/processors/face-mask/utils/matrix.ts @@ -0,0 +1,700 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +import { M_PId180f } from '../constants'; + +export function RAD_TO_DEG(rad: number) { + return (rad * 180.0) / Math.PI; +} + +export function vec3_length(v: number[]) { + return Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); +} + +export function vec3_normalize(v: number[]) { + let len, invLen; + + len = vec3_length(v); + if (len === 0.0) { + return; + } + invLen = 1.0 / len; + + v[0] *= invLen; + v[1] *= invLen; + v[2] *= invLen; + + return len; +} + +export function matrix_turn_x(m: number[], cosA: number, sinA: number) { + let m01, m02; + let m11, m12; + let m21, m22; + let m31, m32; + let mx01, mx02; + let mx11, mx12; + let mx21, mx22; + let mx31, mx32; + + m01 = m[4]; //m->m01; + m02 = m[8]; //m->m02; + m11 = m[5]; //m->m11; + m12 = m[9]; //m->m12; + m21 = m[6]; //m->m21; + m22 = m[10]; //m->m22; + m31 = m[7]; //m->m31; + m32 = m[11]; //m->m32; + + mx01 = cosA * m01; + mx02 = sinA * m01; + + mx11 = cosA * m11; + mx12 = sinA * m11; + + mx21 = cosA * m21; + mx22 = sinA * m21; + + mx31 = cosA * m31; + mx32 = sinA * m31; + + mx01 = sinA * m02 + mx01; + mx02 = cosA * m02 - mx02; + + mx11 = sinA * m12 + mx11; + mx12 = cosA * m12 - mx12; + + mx21 = sinA * m22 + mx21; + mx22 = cosA * m22 - mx22; + + mx31 = sinA * m32 + mx31; + mx32 = cosA * m32 - mx32; + + m[4] = mx01; + m[8] = mx02; + + m[5] = mx11; + m[9] = mx12; + + m[6] = mx21; + m[10] = mx22; + + m[7] = mx31; + m[11] = mx32; +} + +/* + * void turn_y(float *m, float cosA, float cosB) + * local rotation around Y-axis + * M = M * Ry + * + * | m00 m01 m02 m03 | | m00 m01 m02 m03 | | cosA 0 sinA 0 | + * | m10 m11 m12 m13 | = | m10 m11 m12 m13 | * | 0 1 0 0 | + * | m20 m21 m22 m23 | | m20 m21 m22 m23 | | -sinA 0 cosA 0 | + * | m30 m31 m32 m33 | | m30 m31 m32 m33 | | 0 0 0 1 | + */ + +export function matrix_turn_y(m: number[], cosA: number, sinA: number) { + let m00, m02; + let m10, m12; + let m20, m22; + let m30, m32; + let mx00, mx02; + let mx10, mx12; + let mx20, mx22; + let mx30, mx32; + + m00 = m[0]; //m->m00; + m02 = m[8]; //m->m02; + m10 = m[1]; //m->m10; + m12 = m[9]; //m->m12; + m20 = m[2]; //m->m20; + m22 = m[10]; //m->m22; + m30 = m[3]; //m->m30; + m32 = m[11]; //m->m32; + + mx00 = cosA * m00; + mx02 = sinA * m00; + + mx10 = cosA * m10; + mx12 = sinA * m10; + + mx20 = cosA * m20; + mx22 = sinA * m20; + + mx30 = cosA * m30; + mx32 = sinA * m30; + + mx00 = -sinA * m02 + mx00; + mx02 = cosA * m02 + mx02; + + mx10 = -sinA * m12 + mx10; + mx12 = cosA * m12 + mx12; + + mx20 = -sinA * m22 + mx20; + mx22 = cosA * m22 + mx22; + + mx30 = -sinA * m32 + mx30; + mx32 = cosA * m32 + mx32; + + m[0] = mx00; + m[8] = mx02; + + m[1] = mx10; + m[9] = mx12; + + m[2] = mx20; + m[10] = mx22; + + m[3] = mx30; + m[11] = mx32; +} + +/* + * void turn_z(float *m, float cosA, float sinA) + * local rotation around Z-axis + * M = M * Rz + * + * | m00 m01 m02 m03 | | m00 m01 m02 m03 | | cosA -sinA 0 0 | + * | m10 m11 m12 m13 | = | m10 m11 m12 m13 | * | sinA cosA 0 0 | + * | m20 m21 m22 m23 | | m20 m21 m22 m23 | | 0 0 1 0 | + * | m30 m31 m32 m33 | | m30 m31 m32 m33 | | 0 0 0 1 | + * + */ + +export function matrix_turn_z(m: number[], cosA: number, sinA: number) { + let m00, m01; + let m10, m11; + let m20, m21; + let m30, m31; + let mx00, mx01; + let mx10, mx11; + let mx20, mx21; + let mx30, mx31; + + m00 = m[0]; //m->m00; + m01 = m[4]; //m->m01; + m10 = m[1]; //m->m10; + m11 = m[5]; //m->m11; + m20 = m[2]; //m->m20; + m21 = m[6]; //m->m21; + m30 = m[3]; //m->m30; + m31 = m[7]; //m->m31; + + mx00 = cosA * m00; + mx01 = sinA * m00; + + mx10 = cosA * m10; + mx11 = sinA * m10; + + mx20 = cosA * m20; + mx21 = sinA * m20; + + mx30 = cosA * m30; + mx31 = sinA * m30; + + mx00 = sinA * m01 + mx00; + mx01 = cosA * m01 - mx01; + + mx10 = sinA * m11 + mx10; + mx11 = cosA * m11 - mx11; + + mx20 = sinA * m21 + mx20; + mx21 = cosA * m21 - mx21; + + mx30 = sinA * m31 + mx30; + mx31 = cosA * m31 - mx31; + + m[0] = mx00; + m[4] = mx01; + m[1] = mx10; + m[5] = mx11; + m[2] = mx20; + m[6] = mx21; + m[3] = mx30; + m[7] = mx31; +} + +/************************************************************ + Translate Matrix + M = M * T + + | m00 m04 m08 m12 | | 1 0 0 x | | m00 m04 m08 (m00*x + m04*y + m08*z + m12) | + | m01 m05 m09 m13 | * | 0 1 0 y | = | m01 m05 m09 (m01*x + m05*y + m09*z + m13) | + | m02 m06 m10 m14 | | 0 0 1 z | | m02 m06 m10 (m02*x + m06*y + m10*z + m14) | + | m03 m07 m11 m15 | | 0 0 0 1 | | m03 m07 m11 (m03*x + m07*y + m11*z + m15) | +***********************************************************/ +export function matrix_translate(m: number[], x: number, y: number, z: number) { + let m00, m01, m02, m03; + let m04, m05, m06, m07; + let m08, m09, m10, m11; + let m12, m13, m14, m15; + + m00 = m[0]; + m04 = m[4]; + m08 = m[8]; /* m12 = m[12]; */ + m01 = m[1]; + m05 = m[5]; + m09 = m[9]; /* m13 = m[13]; */ + m02 = m[2]; + m06 = m[6]; + m10 = m[10]; /* m14 = m[14]; */ + m03 = m[3]; + m07 = m[7]; + m11 = m[11]; /* m15 = m[15]; */ + + m12 = m[12]; + m13 = m[13]; + m14 = m[14]; + m15 = m[15]; + + m12 += m08 * z; + m13 += m09 * z; + m14 += m10 * z; + m15 += m11 * z; + + m12 += m04 * y; + m13 += m05 * y; + m14 += m06 * y; + m15 += m07 * y; + + m12 += m00 * x; + m13 += m01 * x; + m14 += m02 * x; + m15 += m03 * x; + + m[12] = m12; + m[13] = m13; + m[14] = m14; + m[15] = m15; +} + +/************************************************************ + Rotate Matrix + | m00 m04 m08 m12 | | r00 r04 r08 0 | + | m01 m05 m09 m13 | * | r01 r05 r09 0 | + | m02 m06 m10 m14 | | r02 r06 r10 0 | + | m03 m07 m11 m15 | | 0 0 0 1 | + + m00 = m00*r00 + m04*r01 + m08*r02 + m01 = m01*r00 + m05*r01 + m09*r02 + m02 = m02*r00 + m06*r01 + m10*r02 + m03 = m03*r00 + m07*r01 + m11*r02 + + m04 = m00*r04 + m04*r05 + m08*r06 + m05 = m01*r04 + m05*r05 + m09*r06 + m06 = m02*r04 + m06*r05 + m10*r06 + m07 = m03*r04 + m07*r05 + m11*r06 + + m08 = m00*r08 + m04*r09 + m08*r10 + m09 = m01*r08 + m05*r09 + m09*r10 + m10 = m02*r08 + m06*r09 + m10*r10 + m11 = m03*r08 + m07*r09 + m11*r10 + + m12 = m12 + m13 = m13 + m14 = m14 + m15 = m15 +***********************************************************/ +export function matrix_rotate(m: number[], angle: number, x: number, y: number, z: number) { + let v = [0, 0, 0], + angleRadian; + let sinA, cosA, cosA2; + let xcosA2, ycosA2, zcosA2; + let xsinA, ysinA, zsinA; + + angleRadian = angle * M_PId180f; + sinA = Math.sin(angleRadian); + cosA = Math.cos(angleRadian); + + /* for fast rotation around X-Axis/Y-Axis,and Z-Axis */ + if (x === 0.0 && y === 0.0 && z !== 0.0) { + if (z < 0.0) { + /* If the Axis of the Rotation is minus, Rotate Backwords */ + sinA = -sinA; + } + /* Z Axis Rotateion */ + matrix_turn_z(m, cosA, sinA); + return; + } else if (x === 0.0 && y !== 0.0 && z === 0.0) { + if (y < 0.0) { + /* If the Axis of the Rotation is minus, Rotate Backwords */ + sinA = -sinA; + } + /* Y Axis Rotation */ + matrix_turn_y(m, cosA, sinA); + return; + } else if (x !== 0.0 && y === 0.0 && z === 0.0) { + if (x < 0.0) { + /* If the Axis of the Rotation is minus, Rotate Backwords */ + sinA = -sinA; + } + /* X Axis Rotation */ + matrix_turn_x(m, cosA, sinA); + return; + } + + { + let r00, r01, r02; + let r10, r11, r12; + let r20, r21, r22; + + /* normalization of 3D-vector */ + v[0] = x; + v[1] = y; + v[2] = z; + vec3_normalize(v); + + x = v[0]; + y = v[1]; + z = v[2]; + + /* making rotation matrix */ + cosA2 = 1.0 - cosA; + xsinA = x * sinA; + ysinA = y * sinA; + zsinA = z * sinA; + xcosA2 = x * cosA2; + ycosA2 = y * cosA2; + zcosA2 = z * cosA2; + + r00 = x * xcosA2 + cosA; + r10 = y * xcosA2 + zsinA; + r20 = z * xcosA2 - ysinA; + + r01 = x * ycosA2 - zsinA; + r11 = y * ycosA2 + cosA; + r21 = z * ycosA2 + xsinA; + + r02 = x * zcosA2 + ysinA; + r12 = y * zcosA2 - xsinA; + r22 = z * zcosA2 + cosA; + + /* multing with 3x3 rotating matrix. */ + { + let fm0, fm1, fm2; + let mx, my, mz; + + /* load 0th low of "m" */ + fm0 = m[0]; + fm1 = m[4]; + fm2 = m[8]; /* fm3 = m[12]; */ + + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[1]; + fm1 = m[5]; + fm2 = m[9]; /* fm3 = m[13]; */ + + m[0] = mx; + m[4] = my; + m[8] = mz; + + /* *************************** */ + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[2]; + fm1 = m[6]; + fm2 = m[10]; /* m23 = m[14]; */ + + m[1] = mx; + m[5] = my; + m[9] = mz; + + /* *************************** */ + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + fm0 = m[3]; + fm1 = m[7]; + fm2 = m[11]; /* m33 = m[15]; */ + + m[2] = mx; + m[6] = my; + m[10] = mz; + + /* *************************** */ + mx = fm0 * r00; + my = fm0 * r01; + mz = fm0 * r02; + + mx += fm1 * r10; + my += fm1 * r11; + mz += fm1 * r12; + + mx += fm2 * r20; + my += fm2 * r21; + mz += fm2 * r22; + + m[3] = mx; + m[7] = my; + m[11] = mz; + } + } +} + +/****************************************** + Scale Matrix + | m00 m04 m08 m12 | | x 0 0 0 | + | m01 m05 m09 m13 | * | 0 y 0 0 | + | m02 m06 m10 m14 | | 0 0 z 0 | + | m03 m07 m11 m15 | | 0 0 0 1 | +*******************************************/ +export function matrix_scale(m: number[], x: number, y: number, z: number) { + let m00, m01, m02, m03; + let m04, m05, m06, m07; + let m08, m09, m10, m11; + /* let m12, m13, m14, m15; */ + + m00 = m[0]; + m04 = m[4]; + m08 = m[8]; /* m12 = m[12]; */ + m01 = m[1]; + m05 = m[5]; + m09 = m[9]; /* m13 = m[13]; */ + m02 = m[2]; + m06 = m[6]; + m10 = m[10]; /* m14 = m[14]; */ + m03 = m[3]; + m07 = m[7]; + m11 = m[11]; /* m15 = m[15]; */ + + m00 = m00 * x; + m04 = m04 * y; + m08 = m08 * z; + + m01 = m01 * x; + m05 = m05 * y; + m09 = m09 * z; + + m02 = m02 * x; + m06 = m06 * y; + m10 = m10 * z; + + m03 = m03 * x; + m07 = m07 * y; + m11 = m11 * z; + + m[0] = m00; + m[4] = m04; + m[8] = m08; + + m[1] = m01; + m[5] = m05; + m[9] = m09; + + m[2] = m02; + m[6] = m06; + m[10] = m10; + + m[3] = m03; + m[7] = m07; + m[11] = m11; +} + +/****************************************** + Multiply Matrix + M = M1 * M2 +*******************************************/ +export function matrix_mult(m: number[], m1: number[], m2: number[]) { + let fm0, fm1, fm2, fm3; + let fpm00, fpm01, fpm02, fpm03; + let fpm10, fpm11, fpm12, fpm13; + let fpm20, fpm21, fpm22, fpm23; + let fpm30, fpm31, fpm32, fpm33; + let x, y, z, w; + + /* load pMb */ + fpm00 = m2[0]; + fpm01 = m2[4]; + fpm02 = m2[8]; + fpm03 = m2[12]; + + fpm10 = m2[1]; + fpm11 = m2[5]; + fpm12 = m2[9]; + fpm13 = m2[13]; + + fpm20 = m2[2]; + fpm21 = m2[6]; + fpm22 = m2[10]; + fpm23 = m2[14]; + + fpm30 = m2[3]; + fpm31 = m2[7]; + fpm32 = m2[11]; + fpm33 = m2[15]; + + /* process 0-line of "m1" */ + fm0 = m1[0]; + fm1 = m1[4]; + fm2 = m1[8]; + fm3 = m1[12]; + + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[1]; + fm1 = m1[5]; + fm2 = m1[9]; + fm3 = m1[13]; + + m[0] = x; + m[4] = y; + m[8] = z; + m[12] = w; + + /* *************************** */ + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[2]; + fm1 = m1[6]; + fm2 = m1[10]; + fm3 = m1[14]; + + m[1] = x; + m[5] = y; + m[9] = z; + m[13] = w; + + /* *************************** */ + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + fm0 = m1[3]; + fm1 = m1[7]; + fm2 = m1[11]; + fm3 = m1[15]; + + m[2] = x; + m[6] = y; + m[10] = z; + m[14] = w; + + /* *************************** */ + x = fm0 * fpm00; + y = fm0 * fpm01; + z = fm0 * fpm02; + w = fm0 * fpm03; + + x += fm1 * fpm10; + y += fm1 * fpm11; + z += fm1 * fpm12; + w += fm1 * fpm13; + + x += fm2 * fpm20; + y += fm2 * fpm21; + z += fm2 * fpm22; + w += fm2 * fpm23; + + x += fm3 * fpm30; + y += fm3 * fpm31; + z += fm3 * fpm32; + w += fm3 * fpm33; + + m[3] = x; + m[7] = y; + m[11] = z; + m[15] = w; +} + +export function matrix_identity(m: number[]) { + m[0] = 1.0; + m[4] = 0.0; + m[8] = 0.0; + m[12] = 0.0; + m[1] = 0.0; + m[5] = 1.0; + m[9] = 0.0; + m[13] = 0.0; + m[2] = 0.0; + m[6] = 0.0; + m[10] = 1.0; + m[14] = 0.0; + m[3] = 0.0; + m[7] = 0.0; + m[11] = 0.0; + m[15] = 1.0; +} diff --git a/src/processors/face-mask/utils/shaders.ts b/src/processors/face-mask/utils/shaders.ts new file mode 100644 index 000000000..db2138adf --- /dev/null +++ b/src/processors/face-mask/utils/shaders.ts @@ -0,0 +1,58 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +const compile_shader_text = function(gl: WebGL2RenderingContext, shader_type: number, text: string) { + const shader = gl.createShader(shader_type) as WebGLShader; + gl.shaderSource(shader, text); + + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader)); + gl.deleteShader(shader); + return null; + } + return shader; +}; + +const link_shaders = function(gl: WebGL2RenderingContext, vertShader: WebGLShader, fragShader: WebGLShader) { + const program = gl.createProgram() as WebGLProgram; + + gl.attachShader(program, vertShader); + gl.attachShader(program, fragShader); + + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + alert('Could not initialise shaders'); + } + return program; +}; + +export type ShaderObject = { + program: WebGLProgram; + loc_vtx: number; + loc_clr: number; + loc_nrm: number; + loc_uv: number; + loc_smp: WebGLUniformLocation | null; +}; + +export const generate_shader = function(gl: WebGL2RenderingContext, str_vs: string, str_fs: string): ShaderObject { + const vs = compile_shader_text(gl, gl.VERTEX_SHADER, str_vs); + const fs = compile_shader_text(gl, gl.FRAGMENT_SHADER, str_fs); + + if (!vs || !fs) throw new Error('Failed to compile shaders for face masks effect.'); + const program = link_shaders(gl, vs, fs); + + gl.deleteShader(vs); + gl.deleteShader(fs); + + return { + program, + loc_vtx: gl.getAttribLocation(program, `a_Vertex`), + loc_clr: gl.getAttribLocation(program, `a_Color`), + loc_nrm: gl.getAttribLocation(program, `a_Normal`), + loc_uv: gl.getAttribLocation(program, `a_TexCoord`), + loc_smp: gl.getUniformLocation(program, `u_sampler`), + }; +}; diff --git a/src/processors/face-mask/utils/textures.ts b/src/processors/face-mask/utils/textures.ts new file mode 100644 index 000000000..99f9e866e --- /dev/null +++ b/src/processors/face-mask/utils/textures.ts @@ -0,0 +1,33 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +const createTexture = function(gl: WebGL2RenderingContext) { + let texid = gl.createTexture(); + + gl.bindTexture(gl.TEXTURE_2D, texid); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + + return texid; +}; + +export const createTextureFromImage = function(gl: WebGL2RenderingContext, image: HTMLImageElement) { + const texture = createTexture(gl); + + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); + + return texture; +}; + +export const createTextureFromImageBitmap = function(gl: WebGL2RenderingContext, image: ImageBitmap) { + const texture = createTexture(gl); + + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); + + return texture; +}; diff --git a/src/processors/face-mask/utils/webgl.ts b/src/processors/face-mask/utils/webgl.ts new file mode 100644 index 000000000..377715ca0 --- /dev/null +++ b/src/processors/face-mask/utils/webgl.ts @@ -0,0 +1,115 @@ +/* ------------------------------------------------ * + * Inspired by: https://github.com/terryky/tfjs_webgl_app/tree/master/face_landmark + * ------------------------------------------------ */ + +import { Face, Keypoint } from '@tensorflow-models/face-landmarks-detection'; +import { Facemesh } from './Facemesh'; +import { Render2D } from './Render2D'; + +/* Adjust the texture size to fit the window size + * + * Portrait + * Landscape +------+ + * +-+------+-+ +------+ + * | | | | | | + * | | | | | | + * +-+------+-+ +------+ + * +------+ + */ + +export type Region = { + width: number; + height: number; + offsetX: number; + offsetY: number; + scaledW: number; + scaledH: number; + scale: number; +}; + +export function calcSizeToFit(srcW: number, srcH: number, winW: number, winH: number): Region { + const winAspect = winW / winH; + const texAspect = srcW / srcH; + let scale; + let scaledW, scaledH; + let offsetX, offsetY; + + if (winAspect > texAspect) { + scale = winH / srcH; + scaledW = scale * srcW; + scaledH = scale * srcH; + offsetX = (winW - scaledW) * 0.5; + offsetY = 0; + } else { + scale = winW / srcW; + scaledW = scale * srcW; + scaledH = scale * srcH; + offsetX = 0; + offsetY = (winH - scaledH) * 0.5; + } + + return { + width: winW /* full rect width with margin */, + height: winH /* full rect height with margin */, + offsetX /* start position of valid texture */, + offsetY /* start position of valid texture */, + scaledW /* width of valid texture */, + scaledH /* height of valid texture */, + scale, + }; +} + +export function render2dScene( + gl: WebGL2RenderingContext, + camTex: WebGLTexture, + facePredictions: Face[], + camWidth: number, + camHeight: number, + maskImage: HTMLImageElement, + maskTex: WebGLTexture, + maskPredictions: Face[], + camRegion: Region, + facemesh: Facemesh, + r2d: Render2D +) { + const tx = camRegion.offsetX; + const ty = camRegion.offsetY; + const tw = camRegion.scaledW; + const th = camRegion.scaledH; + const scale = camRegion.scale; + + gl.disable(gl.DEPTH_TEST); + + // Render the camera texture + r2d.draw_2d_texture(camTex, tx, ty, tw, th, 0); + + for (let i = 0; i < facePredictions.length; i++) { + const faceKeypoints: number[][] = facePredictions[i].keypoints.map( + ({ x, y, z }: Keypoint) => [x, y, z] as number[] + ); + + /* render the deformed mask image onto the camera image */ + if (maskPredictions.length > 0) { + const maskKeypoints: number[][] = maskPredictions[0].keypoints.map( + ({ x, y, z }: Keypoint) => [x, y, z] as number[] + ); + + let faceVtx = new Array(faceKeypoints.length * 3); + let faceUv = new Array(faceKeypoints.length * 2); + for (let j = 0; j < faceKeypoints.length; j++) { + let p = faceKeypoints[j]; + faceVtx[3 * j + 0] = p[0] * scale + tx; + faceVtx[3 * j + 1] = p[1] * scale + ty; + faceVtx[3 * j + 2] = p[2]; + + let q = maskKeypoints[j]; + faceUv[2 * j + 0] = q[0] / maskImage.width; + faceUv[2 * j + 1] = q[1] / maskImage.height; + } + + const eyeHole = true; + const maskColor = [1.0, 1.0, 1.0, 0.7]; + facemesh.draw_facemesh_tri_tex(maskTex, faceVtx, faceUv, maskColor, eyeHole); + } + } +}