Merge pull request #16 from paywteam/feature/notifier

Regular merge to master

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: Node CI
 
 on:
   pull_request:
-    branches: 
+    branches:
       - master
   push:
     branches:

package.json

@@ -39,10 +39,13 @@
     "iconv-lite": "^0.5.0",
     "jsdom": "^15.1.1",
     "lru-cache": "^5.1.1",
+    "moment": "^2.24.0",
     "mongoose": "^5.7.3",
     "nanoid": "^2.1.3",
+    "p-limit": "^2.2.1",
     "passport": "^0.4.0",
     "passport-google-oauth20": "^2.0.0",
+    "puppeteer": "^2.0.0",
     "request": "^2.88.0",
     "uuid": "^3.3.3",
     "winston": "^3.2.1"
@@ -63,6 +66,7 @@
     "@types/nanoid": "^2.1.0",
     "@types/passport": "^1.0.1",
     "@types/passport-google-oauth20": "^2.0.2",
+    "@types/puppeteer": "^2.0.0",
     "@types/request": "^2.48.3",
     "@types/request-promise-native": "^1.0.17",
     "@types/uuid": "^3.4.5",

src/app/http/controllers/FragmentController.ts

@@ -1,38 +1,27 @@
 import { WGRequest, WGResponse, SimpleHandler } from '@/http/RequestHandler'
 import { GlueBoardDoc } from '@@/migrate/schemas/glue-board'
 import GlueBoard from '@@/migrate/models/glue-board'
-import { FragmentDoc } from '@@/migrate/schemas/fragment'
 import Fragment from '@@/migrate/models/fragment'
+import { FragmentDoc, FragmentJSON } from '@@/migrate/schemas/fragment'
 import { checkSchema, ValidationChain } from 'express-validator'
 import { UserDoc } from '@@/migrate/schemas/user'
 import UniformURL from '@/modules/webglue-api/UniformURL'
 import UIDGenerator from '@/modules/UIDGenerator'
+import Snappy from '@/modules/webglue-api/Snappy'
 
 interface IndexResponseBody {
-  fragments: Array<{
-    id: string
-    url: string
-    selector: string
-    xPos: number
-    yPos: number
-    scale: number
-  }>
+  fragments: FragmentJSON[]
 }
 
-interface GetResponseBody {
-  id: string
-  url: string
-  selector: string
-  xPos: number
-  yPos: number
-  scale: number
-}
+type GetResponseBody = FragmentJSON
 
 interface CreateResponseBody {
   createdID: string
 }
 
 export default class FragmentController {
+  public static readonly DEFAULT_WATCH_CYCLE = 60
+
   /**
    * Get all fragments of the GlueBoard.
    */
@@ -45,7 +34,8 @@ export default class FragmentController {
       })
         .lean()
         .populate({
-          path: 'fragments'
+          path: 'fragments',
+          select: '-_id -__v -glueBoard'
         })) as GlueBoardDoc
 
       const fragments = glueBoard.fragments as FragmentDoc[]
@@ -56,14 +46,7 @@ export default class FragmentController {
 
       // compose response body
       for (const fragment of fragments) {
-        responseBody.fragments.push({
-          id: fragment.id,
-          url: fragment.url,
-          selector: fragment.selector,
-          xPos: fragment.xPos,
-          yPos: fragment.yPos,
-          scale: fragment.scale
-        })
+        responseBody.fragments.push(fragment)
       }
 
       return res.status(200).json(responseBody)
@@ -79,6 +62,8 @@ export default class FragmentController {
         trim: true,
         customSanitizer: {
           options: async (url: string): Promise<string> => {
+            url = encodeURI(url)
+
             // check if the url protocol is set
             // if not, add the default protocol `http`
             if (!url.startsWith('http')) {
@@ -97,14 +82,25 @@ export default class FragmentController {
         },
         errorMessage: '`url` must be a url format.'
       },
-      selector: {
+      'selector.name': {
         exists: {
           options: { checkFalsy: true }
         },
         in: 'body',
         isString: true,
         trim: true,
-        errorMessage: '`selector` must be a string.'
+        errorMessage: '`selector.name` must be a string.'
+      },
+      'selector.offset': {
+        optional: true,
+        in: 'body',
+        isInt: {
+          options: {
+            min: 0
+          }
+        },
+        errorMessage:
+          '`selector.offset` must be a integer which is greater than 0'
       },
       xPos: {
         exists: true,
@@ -133,13 +129,13 @@ export default class FragmentController {
   public static create(): SimpleHandler {
     return async (req, res): Promise<WGResponse> => {
       // create a fragment
-      const fragment = (await Fragment.create({
+      const fragment = await Fragment.create({
         id: UIDGenerator.alphaNumericUID(16), // url id
         url: await req.body.url,
         selector: req.body.selector,
         xPos: req.body.xPos,
         yPos: req.body.yPos
-      })) as FragmentDoc
+      })
 
       // if scale is set, apply to document.
       if (req.body.scale) {
@@ -165,16 +161,11 @@ export default class FragmentController {
    */
   public static get(): SimpleHandler {
     return (req, res): WGResponse => {
-      const fragment = res.locals.fragment as FragmentDoc
+      const fragment = (res.locals.fragment as FragmentDoc).toJSON()
+      delete fragment._id
+      delete fragment.__v
 
-      const responseBody: GetResponseBody = {
-        id: fragment.id,
-        url: fragment.url,
-        selector: fragment.selector,
-        xPos: fragment.xPos,
-        yPos: fragment.yPos,
-        scale: fragment.scale
-      }
+      const responseBody: GetResponseBody = fragment
 
       return res.status(200).json(responseBody)
     }
@@ -200,6 +191,23 @@ export default class FragmentController {
         isNumeric: true,
         errorMessage: '`scale` must be a numeric.'
       },
+      subscription: {
+        optional: true,
+        in: 'body',
+        isBoolean: true,
+        errorMessage: '`subscription` must be a boolean.'
+      },
+      watchCycle: {
+        optional: true,
+        in: 'body',
+        isInt: {
+          options: {
+            min: 30
+          }
+        },
+        errorMessage:
+          '`watchCycle` must be a integer which is greater than equal 30'
+      },
       transferGlueBoardID: {
         optional: {
           options: { checkFalsy: true }
@@ -258,12 +266,50 @@ export default class FragmentController {
         fragment.scale = req.body.scale
       }
 
+      // update subscription
+      if (req.body.subscription !== undefined) {
+        if (req.body.subscription === true && fragment.subscription === false) {
+          // subscribe
+          fragment.subscription = true
+          fragment.headers.userAgent = req.headers['user-agent']
+          fragment.headers.accept = req.headers.accept
+          fragment.headers.acceptLanguage = req.headers['accept-language']
+          fragment.snapshot = (
+            await Snappy.Instance.snapshotElement(
+              fragment.url,
+              fragment.headers,
+              fragment.selector.name,
+              fragment.selector.offset
+            )
+          ).outerHTML
+          fragment.watchCycle = this.DEFAULT_WATCH_CYCLE
+          fragment.lastWatchedAt = new Date()
+        } else if (
+          req.body.subscription === false &&
+          fragment.subscription === true
+        ) {
+          // unsubscribe
+          fragment.subscription = false
+          fragment.headers = undefined
+          fragment.snapshot = undefined
+          fragment.watchCycle = undefined
+          fragment.lastWatchedAt = undefined
+        }
+      }
+
+      // update watch cycle
+      if (req.body.watchCycle) {
+        if (fragment.subscription) {
+          fragment.watchCycle = req.body.watchCycle
+        }
+      }
+
       await fragment.save()
 
       if (req.body.transferGlueBoardID) {
-        const newGlueBoard = (await GlueBoard.findOne({
+        const transferGlueBoard = await GlueBoard.findOne({
           id: req.body.transferGlueBoardID
-        })) as GlueBoardDoc
+        })
         const currGlueBoard = res.locals.glueBoard as GlueBoardDoc
 
         // unlink from current GlueBoard
@@ -274,8 +320,8 @@ export default class FragmentController {
         await currGlueBoard.save()
 
         // link to new GlueBoard
-        newGlueBoard.fragments.push(fragment._id)
-        await newGlueBoard.save()
+        transferGlueBoard.fragments.push(fragment._id)
+        await transferGlueBoard.save()
       }
 
       return res.status(204).json()

src/app/http/controllers/GlueBoardController.ts

@@ -1,30 +1,16 @@
 import { WGRequest, WGResponse, SimpleHandler } from '@/http/RequestHandler'
 import { UserDoc } from '@@/migrate/schemas/user'
 import User from '@@/migrate/models/user'
-import { GlueBoardDoc } from '@@/migrate/schemas/glue-board'
+import { GlueBoardDoc, GlueBoardJSON } from '@@/migrate/schemas/glue-board'
 import { checkSchema, ValidationChain } from 'express-validator'
 import GlueBoard from '@@/migrate/models/glue-board'
 import UIDGenerator from '@/modules/UIDGenerator'
 
 interface IndexResponseBody {
-  glueBoards: Array<{
-    id: string
-    category: {
-      name: string
-      color: string
-    }
-    sharing: boolean
-  }>
+  glueBoards: GlueBoardJSON[]
 }
 
-interface GetResponseBody {
-  id: string
-  category: {
-    name: string
-    color: string
-  }
-  sharing: boolean
-}
+type GetResponseBody = GlueBoardJSON
 
 interface CreateResponseBody {
   createdID: string
@@ -44,7 +30,7 @@ export default class GlueBoardController {
         .lean()
         .populate({
           path: 'glueBoards',
-          select: '-_id -category._id'
+          select: '-_id -__v -user -fragments'
         })) as UserDoc
 
       const glueBoards = user.glueBoards as GlueBoardDoc[]
@@ -55,11 +41,7 @@ export default class GlueBoardController {
 
       // compose response body
       for (const glueBoard of glueBoards) {
-        responseBody.glueBoards.push({
-          id: glueBoard.id,
-          category: glueBoard.category,
-          sharing: glueBoard.sharing
-        })
+        responseBody.glueBoards.push(glueBoard)
       }
 
       return res.status(200).json(responseBody)
@@ -118,14 +100,14 @@ export default class GlueBoardController {
       const user = req.user as UserDoc
 
       // create a GlueBoard
-      const glueBoard = (await GlueBoard.create({
+      const glueBoard = await GlueBoard.create({
         user: user._id,
         id: UIDGenerator.alphaNumericUID(14), // url id
         category: {
           name: req.body.name,
           color: req.body.color
         }
-      })) as GlueBoardDoc
+      })
 
       // Add new GlueBoard to user
       user.glueBoards.push(glueBoard._id)
@@ -144,16 +126,13 @@ export default class GlueBoardController {
    */
   public static get(): SimpleHandler {
     return (req, res): WGResponse => {
-      const glueBoard = res.locals.glueBoard as GlueBoardDoc
+      const glueBoard = (res.locals.glueBoard as GlueBoardDoc).toJSON()
+      delete glueBoard._id
+      delete glueBoard.__v
+      delete glueBoard.user
+      delete glueBoard.fragments
 
-      const responseBody: GetResponseBody = {
-        id: glueBoard.id,
-        category: {
-          name: glueBoard.category.name,
-          color: glueBoard.category.color
-        },
-        sharing: glueBoard.sharing
-      }
+      const responseBody: GetResponseBody = glueBoard
 
       return res.status(200).json(responseBody)
     }