Add 3D Face Effects Twilio Video Processor and Menu

package.json

@@ -7,6 +7,9 @@
     "@material-ui/core": "^4.12.3",
     "@material-ui/icons": "^4.11.2",
     "@material-ui/lab": "^4.0.0-alpha.60",
+    "@tensorflow-models/face-landmarks-detection": "^1.0.2",
+    "@tensorflow/tfjs-backend-wasm": "^4.0.0",
+    "@tensorflow/tfjs-core": "^4.0.0",
     "@twilio-labs/plugin-rtc": "^0.8.4",
     "@twilio/conversations": "^2.1.0",
     "@twilio/video-processors": "^1.0.1",

src/components/MaskSelectionDialog/MaskSelectionDialog.tsx

@@ -0,0 +1,51 @@
+import React from 'react';
+import MaskSelectionHeader from './MaskSelectionHeader/MaskSelectionHeader';
+import MaskThumbnail from './MaskThumbnail/MaskThumbnail';
+import Drawer from '@material-ui/core/Drawer';
+import { makeStyles, Theme } from '@material-ui/core/styles';
+import { maskConfig } from '../VideoProvider/useMaskSettings/useMaskSettings';
+import useVideoContext from '../../hooks/useVideoContext/useVideoContext';
+
+const useStyles = makeStyles((theme: Theme) => ({
+  drawer: {
+    display: 'flex',
+    width: theme.rightDrawerWidth,
+    height: `calc(100% - ${theme.footerHeight}px)`,
+  },
+  thumbnailContainer: {
+    display: 'flex',
+    flexWrap: 'wrap',
+    padding: '5px',
+    overflowY: 'auto',
+  },
+}));
+
+function MaskSelectionDialog() {
+  const classes = useStyles();
+  const { isMaskSelectionOpen, setIsMaskSelectionOpen } = useVideoContext();
+
+  const imageNames = maskConfig.imageNames;
+  const images = maskConfig.images;
+
+  return (
+    <Drawer
+      variant="persistent"
+      anchor="right"
+      open={isMaskSelectionOpen}
+      transitionDuration={0}
+      classes={{
+        paper: classes.drawer,
+      }}
+    >
+      <MaskSelectionHeader onClose={() => setIsMaskSelectionOpen(false)} />
+      <div className={classes.thumbnailContainer}>
+        <MaskThumbnail thumbnail={'none'} name={'None'} />
+        {images.map((image, index) => (
+          <MaskThumbnail thumbnail={'image'} name={imageNames[index]} index={index} imagePath={image} key={image} />
+        ))}
+      </div>
+    </Drawer>
+  );
+}
+
+export default MaskSelectionDialog;

src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.test.tsx

@@ -0,0 +1,17 @@
+import React from 'react';
+import { shallow } from 'enzyme';
+import CloseIcon from '../../../icons/CloseIcon';
+import MaskSelectionHeader from './MaskSelectionHeader';
+
+const mockCloseDialog = jest.fn();
+
+describe('The Background Selection Header Component', () => {
+  it('should close the selection dialog when "X" is clicked', () => {
+    const wrapper = shallow(<MaskSelectionHeader onClose={mockCloseDialog} />);
+    wrapper
+      .find(CloseIcon)
+      .parent()
+      .simulate('click');
+    expect(mockCloseDialog).toHaveBeenCalled();
+  });
+});

src/components/MaskSelectionDialog/MaskSelectionHeader/MaskSelectionHeader.tsx

@@ -0,0 +1,43 @@
+import React from 'react';
+import { makeStyles, createStyles } from '@material-ui/core/styles';
+import CloseIcon from '../../../icons/CloseIcon';
+
+const useStyles = makeStyles(() =>
+  createStyles({
+    container: {
+      minHeight: '56px',
+      background: '#F4F4F6',
+      borderBottom: '1px solid #E4E7E9',
+      display: 'flex',
+      justifyContent: 'space-between',
+      alignItems: 'center',
+      padding: '0 1em',
+    },
+    text: {
+      fontWeight: 'bold',
+    },
+    closeMaskSelection: {
+      cursor: 'pointer',
+      display: 'flex',
+      background: 'transparent',
+      border: '0',
+      padding: '0.4em',
+    },
+  })
+);
+
+interface MaskSelectionHeaderProps {
+  onClose: () => void;
+}
+
+export default function MaskSelectionHeader({ onClose }: MaskSelectionHeaderProps) {
+  const classes = useStyles();
+  return (
+    <div className={classes.container}>
+      <div className={classes.text}>Mask Effects</div>
+      <button className={classes.closeMaskSelection} onClick={onClose}>
+        <CloseIcon />
+      </button>
+    </div>
+  );
+}

src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.test.tsx

@@ -0,0 +1,65 @@
+import React from 'react';
+import MaskThumbnail from './MaskThumbnail';
+import BlurIcon from '@material-ui/icons/BlurOnOutlined';
+import NoneIcon from '@material-ui/icons/NotInterestedOutlined';
+import { shallow } from 'enzyme';
+import useVideoContext from '../../../hooks/useVideoContext/useVideoContext';
+
+jest.mock('../../../hooks/useVideoContext/useVideoContext');
+const mockUseVideoContext = useVideoContext as jest.Mock<any>;
+const mockSetMaskSettings = jest.fn();
+mockUseVideoContext.mockImplementation(() => ({
+  maskSettings: {
+    type: 'blur',
+    index: 0,
+  },
+  setMaskSettings: mockSetMaskSettings,
+}));
+
+describe('The MaskThumbnail component', () => {
+  it('should update the mask settings when clicked', () => {
+    const wrapper = shallow(<MaskThumbnail thumbnail={'none'} index={5} />);
+    wrapper.simulate('click');
+    expect(mockSetMaskSettings).toHaveBeenCalledWith({ index: 5, type: 'none' });
+  });
+
+  it('should not be selected when thumbnail prop and maskSettings type are not equivalent (icon)', () => {
+    const wrapper = shallow(<MaskThumbnail thumbnail={'none'} />);
+    expect(wrapper.find('.selected').exists()).toBe(false);
+  });
+
+  it('should be selected when thumbnail prop and maskSettings type are equivalent (image)', () => {
+    mockUseVideoContext.mockImplementationOnce(() => ({
+      maskSettings: {
+        type: 'image',
+        index: 1,
+      },
+      setMaskSettings: mockSetMaskSettings,
+    }));
+    const wrapper = shallow(<MaskThumbnail thumbnail={'image'} index={1} />);
+    expect(wrapper.find('.selected').exists()).toBe(true);
+  });
+
+  it('should not be selected when thumbnail and maskSettings type are not equivlanet (image)', () => {
+    mockUseVideoContext.mockImplementationOnce(() => ({
+      maskSettings: {
+        type: 'image',
+        index: 1,
+      },
+      setMaskSettings: mockSetMaskSettings,
+    }));
+    const wrapper = shallow(<MaskThumbnail thumbnail={'image'} index={5} />);
+    expect(wrapper.find('.selected').exists()).toBe(false);
+  });
+
+  it("should contain the NoneIcon when thumbnail is set to 'none'", () => {
+    const wrapper = shallow(<MaskThumbnail thumbnail={'none'} />);
+    expect(wrapper.containsMatchingElement(<NoneIcon />)).toBe(true);
+  });
+
+  it("should not have any icons when thumbnail is set to 'image'", () => {
+    const wrapper = shallow(<MaskThumbnail thumbnail={'image'} />);
+    expect(wrapper.containsMatchingElement(<BlurIcon />)).toBe(false);
+    expect(wrapper.containsMatchingElement(<NoneIcon />)).toBe(false);
+  });
+});

src/components/MaskSelectionDialog/MaskThumbnail/MaskThumbnail.tsx

@@ -0,0 +1,132 @@
+import React from 'react';
+import clsx from 'clsx';
+import BlurIcon from '@material-ui/icons/BlurOnOutlined';
+import { makeStyles, Theme, createStyles } from '@material-ui/core/styles';
+import NoneIcon from '@material-ui/icons/NotInterestedOutlined';
+import useVideoContext from '../../../hooks/useVideoContext/useVideoContext';
+
+export type Thumbnail = 'none' | 'image';
+
+interface MaskThumbnailProps {
+  thumbnail: Thumbnail;
+  imagePath?: string;
+  name?: string;
+  index?: number;
+}
+
+const useStyles = makeStyles((theme: Theme) =>
+  createStyles({
+    thumbContainer: {
+      margin: '5px',
+      width: 'calc(50% - 10px)',
+      display: 'flex',
+      position: 'relative',
+      '&::after': {
+        content: '""',
+        paddingBottom: '55.5%',
+      },
+    },
+    thumbIconContainer: {
+      width: '100%',
+      display: 'flex',
+      justifyContent: 'center',
+      alignItems: 'center',
+      borderRadius: '10px',
+      border: `solid ${theme.palette.grey[400]}`,
+      '&.selected': {
+        border: `solid ${theme.palette.primary.main}`,
+        '& svg': {
+          color: `${theme.palette.primary.main}`,
+        },
+      },
+    },
+    thumbIcon: {
+      height: 50,
+      width: 50,
+      color: `${theme.palette.grey[400]}`,
+      '&.selected': {
+        color: `${theme.palette.primary.main}`,
+      },
+    },
+    thumbImage: {
+      width: '100%',
+      height: '100%',
+      position: 'absolute',
+      top: 0,
+      bottom: 0,
+      left: 0,
+      right: 0,
+      objectFit: 'cover',
+      borderRadius: '10px',
+      border: `solid ${theme.palette.grey[400]}`,
+      '&:hover': {
+        cursor: 'pointer',
+        '& svg': {
+          color: `${theme.palette.primary.main}`,
+        },
+        '& $thumbOverlay': {
+          visibility: 'visible',
+        },
+      },
+      '&.selected': {
+        border: `solid ${theme.palette.primary.main}`,
+        '& svg': {
+          color: `${theme.palette.primary.main}`,
+        },
+      },
+    },
+    thumbOverlay: {
+      position: 'absolute',
+      color: 'transparent',
+      padding: '20px',
+      fontSize: '14px',
+      fontWeight: 'bold',
+      width: '100%',
+      height: '100%',
+      borderRadius: '10px',
+      display: 'flex',
+      justifyContent: 'center',
+      alignItems: 'center',
+      '&:hover': {
+        background: 'rgba(95, 93, 128, 0.6)',
+        color: 'white',
+      },
+    },
+  })
+);
+
+export default function MaskThumbnail({ thumbnail, imagePath, name, index }: MaskThumbnailProps) {
+  const classes = useStyles();
+  const { maskSettings, setMaskSettings } = useVideoContext();
+  const isImage = thumbnail === 'image';
+  const thumbnailSelected = isImage
+    ? maskSettings.index === index && maskSettings.type === 'image'
+    : maskSettings.type === thumbnail;
+  const icons = {
+    none: NoneIcon,
+    blur: BlurIcon,
+    image: null,
+  };
+  const ThumbnailIcon = icons[thumbnail];
+
+  return (
+    <div
+      className={classes.thumbContainer}
+      onClick={() =>
+        setMaskSettings({
+          type: thumbnail,
+          index: index,
+        })
+      }
+    >
+      {ThumbnailIcon ? (
+        <div className={clsx(classes.thumbIconContainer, { selected: thumbnailSelected })}>
+          <ThumbnailIcon className={classes.thumbIcon} />
+        </div>
+      ) : (
+        <img className={clsx(classes.thumbImage, { selected: thumbnailSelected })} src={imagePath} alt={name} />
+      )}
+      <div className={classes.thumbOverlay}>{name}</div>
+    </div>
+  );
+}

src/components/MenuBar/Menu/Menu.tsx

@@ -1,6 +1,7 @@
 import React, { useState, useRef } from 'react';
 import AboutDialog from '../../AboutDialog/AboutDialog';
 import BackgroundIcon from '../../../icons/BackgroundIcon';
+import MaskIcon from '../../../icons/MaskIcon';
 import CollaborationViewIcon from '@material-ui/icons/AccountBox';
 import DeviceSelectionDialog from '../../DeviceSelectionDialog/DeviceSelectionDialog';
 import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
@@ -39,7 +40,7 @@ export default function Menu(props: { buttonClassName?: string }) {
   const { isFetching, updateRecordingRules, roomType, setIsGalleryViewActive, isGalleryViewActive } = useAppState();
   const { setIsChatWindowOpen } = useChatContext();
   const isRecording = useIsRecording();
-  const { room, setIsBackgroundSelectionOpen } = useVideoContext();
+  const { room, setIsBackgroundSelectionOpen, setIsMaskSelectionOpen } = useVideoContext();
 
   const anchorRef = useRef<HTMLButtonElement>(null);
   const { flipCameraDisabled, toggleFacingMode, flipCameraSupported } = useFlipCameraToggle();
@@ -85,6 +86,7 @@ export default function Menu(props: { buttonClassName?: string }) {
           <MenuItem
             onClick={() => {
               setIsBackgroundSelectionOpen(true);
+              setIsMaskSelectionOpen(false);
               setIsChatWindowOpen(false);
               setMenuOpen(false);
             }}
@@ -96,6 +98,22 @@ export default function Menu(props: { buttonClassName?: string }) {
           </MenuItem>
         )}
 
+        {isSupported && (
+          <MenuItem
+            onClick={() => {
+              setIsBackgroundSelectionOpen(false);
+              setIsMaskSelectionOpen(true);
+              setIsChatWindowOpen(false);
+              setMenuOpen(false);
+            }}
+          >
+            <IconContainer>
+              <MaskIcon />
+            </IconContainer>
+            <Typography variant="body1">Face Effects</Typography>
+          </MenuItem>
+        )}
+
         {flipCameraSupported && (
           <MenuItem disabled={flipCameraDisabled} onClick={toggleFacingMode}>
             <IconContainer>