diff --git a/api/config/config.json b/api/config/config.json index d5da991..28e9952 100644 --- a/api/config/config.json +++ b/api/config/config.json @@ -1,10 +1,6 @@ { "sso": { "cookiename": "gruppe-adler-sso-token", - "domain": "https://sso.gruppe-adler.de" - }, - "twitter": { - "base64-bearer-token-credentials": "", - "screen-name": "gruppe_adler" + "domain": "https://sso.gruppe-adler.de" } } \ No newline at end of file diff --git a/api/package-lock.json b/api/package-lock.json index 430bda6..2f61675 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -3557,12 +3557,6 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", "optional": true }, - "twitter-d": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/twitter-d/-/twitter-d-0.4.0.tgz", - "integrity": "sha512-t8z7qqq0yt1dD7BUISRvASw9mtYRMSTTVzmkvjz8oM9ZN++LEC6Ix8WEXZS0RtswnGgwdSPTxY9qyHcYgPexdA==", - "dev": true - }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/api/package.json b/api/package.json index c3d1b45..4ab1fe5 100644 --- a/api/package.json +++ b/api/package.json @@ -43,7 +43,6 @@ "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.1.0", "ts-node-dev": "^1.0.0", - "twitter-d": "^0.4.0", "typescript": "^3.9.3" } } diff --git a/api/src/database.ts b/api/src/database.ts index 0489225..3f64d6c 100644 --- a/api/src/database.ts +++ b/api/src/database.ts @@ -1,7 +1,7 @@ import 'reflect-metadata'; import { join } from 'path'; import { Sequelize } from 'sequelize-typescript'; -import { Container, Page, HiddenTweet } from './models'; +import { Container, Page } from './models'; const sequelize = new Sequelize({ dialect: 'sqlite', @@ -9,7 +9,7 @@ const sequelize = new Sequelize({ logging: false }); -sequelize.addModels([Container, Page, HiddenTweet]); +sequelize.addModels([Container, Page]); sequelize.sync(); export default sequelize; diff --git a/api/src/models/hiddentweet.model.ts b/api/src/models/hiddentweet.model.ts deleted file mode 100644 index f52f0ba..0000000 --- a/api/src/models/hiddentweet.model.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { - Table, - Column, - Model, - DataType, - PrimaryKey -} from 'sequelize-typescript'; - -@Table -export default class HiddenTweet extends Model { - @PrimaryKey - @Column(DataType.NUMBER) - public id: number; -} diff --git a/api/src/models/index.ts b/api/src/models/index.ts index d4249df..d9b2d86 100644 --- a/api/src/models/index.ts +++ b/api/src/models/index.ts @@ -1,9 +1,4 @@ import Page from './page.model'; import Container from './container.model'; -import HiddenTweet from './hiddentweet.model'; -export { - Page, - Container, - HiddenTweet -}; +export { Page, Container }; diff --git a/api/src/utils/TwitterService.ts b/api/src/utils/TwitterService.ts deleted file mode 100644 index d50b9f3..0000000 --- a/api/src/utils/TwitterService.ts +++ /dev/null @@ -1,156 +0,0 @@ -import fetch, { Response } from 'node-fetch'; -import ReponseError from './ResponseError'; - -import { Status as Tweet } from 'twitter-d/types/status'; -import * as equals from 'fast-deep-equal'; - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const config = require('../../config/config.json'); - -export class TwitterService { - private twitterBearerToken: string|null = null; - private twitterBearerTokenCreated = 0; - private cachedTweets: Tweet[] = []; - private cachedDate: Date = new Date(0); - private cachePromise: Promise; - private lastModified: Date = new Date(0); - private static instance: TwitterService|null = null; - - // this constructor is actually important to make sure it is private (singleton pattern) - // eslint-disable-next-line no-useless-constructor, @typescript-eslint/no-empty-function - private constructor() { - this.cachedDate = new Date(); - this.cachePromise = this.cacheAllTweets(); - } - - public static getInstance(): TwitterService { - if (this.instance === null) { - this.instance = new TwitterService(); - } - - return this.instance; - } - - public async getTweets(maxId?: string, count = 20, excludeReplies = false): Promise<{ tweets: Tweet[], lastModified: Date }> { - // fetch new tweets if last fetch is more than 15min ago - const now = new Date(); - if (now.getTime() > this.cachedDate.getTime() + 900000) { - this.cachedDate = now; - this.cachePromise = this.cacheAllTweets(); - } - - await this.cachePromise; - - let startIndex = 0; - - if (maxId !== undefined) { - const maxIdIndex = this.cachedTweets.findIndex(t => t.id_str === maxId); - if (maxIdIndex > -1) startIndex = maxIdIndex; - } - - let tweets: Tweet[] = []; - if (excludeReplies) { - let i = startIndex; - - while (tweets.length < count && i < this.cachedTweets.length) { - const tweet = this.cachedTweets[i]; - if (tweet.in_reply_to_status_id === null) tweets.push(tweet); - i++; - } - } else { - tweets = this.cachedTweets.slice(startIndex, startIndex + count); - } - - return { tweets, lastModified: this.lastModified }; - } - - public async cacheAllTweets(): Promise { - const allTweets: Tweet[] = []; - let maxId: string|undefined; - - while (true) { - let tweets = await this.fetchTweets(maxId); - - if (tweets.length > 0 && tweets[0].id_str === maxId) tweets = tweets.slice(1); - - if (tweets.length === 0) break; - - allTweets.push(...tweets); - - maxId = tweets[tweets.length - 1].id_str; - } - - if (!equals(this.cachedTweets, allTweets)) { - this.cachedTweets = allTweets; - this.lastModified = new Date(); - } - } - - private async fetchTweets(maxId?: string): Promise { - const url = new URL('https://api.twitter.com/1.1/statuses/user_timeline.json'); - - url.searchParams.append('screen_name', config.twitter['screen-name']); - url.searchParams.append('tweet_mode', 'extended'); - url.searchParams.append('count', '200'); - - if (maxId !== undefined) url.searchParams.append('max_id', maxId); - - const bearerToken = await this.getToken(); - - let response: Response; - try { - response = await fetch(url.toString(), { headers: { Authorization: `Bearer ${bearerToken}` } }); - } catch (err) { - console.error(err); - throw new ReponseError(504); - } - - if (!response.ok) { - console.error(`Error while trying to fetch tweets. API responded with status code ${response.status}.`); - throw new ReponseError(502); - } - - return response.json() as Promise; - } - - private async getToken(): Promise { - if (this.twitterBearerToken == null) { - return this.requestNewToken(); - } else if ((new Date()).getTime() > this.twitterBearerTokenCreated + 900000) { - // get new token if last token is older than 15 minutes - return this.requestNewToken(); - } - - return this.twitterBearerToken; - } - - private async requestNewToken(): Promise { - let response: Response; - try { - response = await fetch('https://api.twitter.com/oauth2/token?grant_type=client_credentials', { - method: 'POST', - headers: { - Authorization: `Basic ${config.twitter['base64-bearer-token-credentials']}`, - 'Content-type': 'application/x-www-form-urlencoded;charset=UTF-8.' - }, - body: 'grant_type=client_credentials' - }); - } catch (err) { - console.error(err); - throw new ReponseError(504); - } - - if (!response.ok) { - console.error(`Error while trying to request new token for twitter API. API responded with status code ${response.status}.`); - throw new ReponseError(502); - } - - // eslint-disable-next-line camelcase - const json = await response.json() as { access_token: string }; - - this.twitterBearerToken = json.access_token; - this.twitterBearerTokenCreated = (new Date()).getTime(); - - return this.twitterBearerToken; - } -} diff --git a/api/src/v1/index.ts b/api/src/v1/index.ts index 932ebdf..2c85869 100644 --- a/api/src/v1/index.ts +++ b/api/src/v1/index.ts @@ -3,7 +3,6 @@ import { globalErrorHandler } from '../utils/express'; import pageRouter from './routes/page.router'; import containerRouter from './routes/container.router'; import uploadRouter from './routes/upload.router'; -import twitterRouter from './routes/twitter.router'; import eventsRouter from './routes/events.router'; const v1Router = Router(); @@ -11,7 +10,6 @@ const v1Router = Router(); v1Router.use('/page', pageRouter); v1Router.use('/container', containerRouter); v1Router.use('/upload', uploadRouter); -v1Router.use('/twitter', twitterRouter); v1Router.use('/events', eventsRouter); v1Router.use(globalErrorHandler); diff --git a/api/src/v1/routes/twitter.router.ts b/api/src/v1/routes/twitter.router.ts deleted file mode 100644 index 8e3a690..0000000 --- a/api/src/v1/routes/twitter.router.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Router } from 'express'; -import { globalErrorHandler, wrapAsync, return422 } from '../../utils/express'; -import { TwitterService } from '../../utils/TwitterService'; -import { matchedData, query, param } from 'express-validator'; -import { HiddenTweet } from '../../models'; -import { ssoCheckAuthorized } from '../../utils/sso'; - -const twitterService = TwitterService.getInstance(); - -const twitterRouter = Router(); - -twitterRouter.get('/', [ - query('count').optional().isInt({ gt: 0 }).toInt(), - query('max_id').optional().isInt(), - query('exclude_replies').optional().isBoolean().toBoolean(), - return422 -], wrapAsync(async (req, res) => { - // eslint-disable-next-line camelcase - const { count, max_id, exclude_replies } = matchedData(req) as { count?: number, max_id?: string, exclude_replies?: boolean }; - - const { tweets, lastModified } = await twitterService.getTweets(max_id, count, exclude_replies); - - res.header('Cache-Control', 'no-cache'); - res.header('Last-Modified', lastModified.toUTCString()); - - res.json(tweets); -})); - -// get hidden tweets -twitterRouter.get('/hidden', wrapAsync(async (req, res) => { - const tweets = await HiddenTweet.findAll(); - - res.json(tweets.map(x => x.id)); -})); - -// hide tweet -twitterRouter.post('/hidden/:id', [ - ssoCheckAuthorized, - param('id').isInt({ gt: 0 }), - return422 -], wrapAsync(async (req, res) => { - const { id } = matchedData(req) as { id: number }; - - await HiddenTweet.findOrCreate({ where: { id } }); - - res.send(200).end(); -})); - -// unhide tweet -twitterRouter.delete('/hidden/:id', [ - ssoCheckAuthorized, - param('id').isInt({ gt: 0 }), - return422 -], wrapAsync(async (req, res) => { - const { id } = matchedData(req) as { id: number }; - - await HiddenTweet.destroy({ where: { id } }); - - res.send(200).end(); -})); - -twitterRouter.use(globalErrorHandler); - -export default twitterRouter; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index adc959a..d1ce8c6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -40,7 +40,6 @@ "eslint-plugin-vue": "^7.1.0", "sass": "^1.26.3", "sass-loader": "^8.0.2", - "twitter-d": "^0.4.0", "typescript": "~3.8.3", "vue-template-compiler": "^2.6.12", "webpack-assets-manifest": "^4.0.0" @@ -13653,12 +13652,6 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", "dev": true }, - "node_modules/twitter-d": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/twitter-d/-/twitter-d-0.4.0.tgz", - "integrity": "sha512-t8z7qqq0yt1dD7BUISRvASw9mtYRMSTTVzmkvjz8oM9ZN++LEC6Ix8WEXZS0RtswnGgwdSPTxY9qyHcYgPexdA==", - "dev": true - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -27117,12 +27110,6 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", "dev": true }, - "twitter-d": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/twitter-d/-/twitter-d-0.4.0.tgz", - "integrity": "sha512-t8z7qqq0yt1dD7BUISRvASw9mtYRMSTTVzmkvjz8oM9ZN++LEC6Ix8WEXZS0RtswnGgwdSPTxY9qyHcYgPexdA==", - "dev": true - }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 7f7a414..62b01a7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -40,7 +40,6 @@ "eslint-plugin-vue": "^7.1.0", "sass": "^1.26.3", "sass-loader": "^8.0.2", - "twitter-d": "^0.4.0", "typescript": "~3.8.3", "vue-template-compiler": "^2.6.12", "webpack-assets-manifest": "^4.0.0" diff --git a/frontend/src/components/Home/Tweet.vue b/frontend/src/components/Home/Tweet.vue deleted file mode 100644 index 95bcec7..0000000 --- a/frontend/src/components/Home/Tweet.vue +++ /dev/null @@ -1,223 +0,0 @@ - - - - diff --git a/frontend/src/components/Home/TweetMedia.vue b/frontend/src/components/Home/TweetMedia.vue deleted file mode 100644 index caa1a78..0000000 --- a/frontend/src/components/Home/TweetMedia.vue +++ /dev/null @@ -1,74 +0,0 @@ - - - - - diff --git a/frontend/src/services/twitter.ts b/frontend/src/services/twitter.ts deleted file mode 100644 index 20e5265..0000000 --- a/frontend/src/services/twitter.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { - Status as ApiResTweet, - Entities as TweetEntities, - HashtagEntity, - MediaEntity, - UserMentionEntity, - UrlEntity, - FullUser as FullTwitterUser -} from 'twitter-d'; - -import { fetchJSON } from './utils'; -import { API_URI } from '.'; -import ResponseError from './utils/ResponseError'; - -export const TWEET_TYPE = 'tweet'; -export const RETWEET_TYPE = 'retweet'; - -interface TwitterUser { - username: string; - displayName: string; - picture: string; -} - -export interface TweetMedia { - id: number; - url: string; - target: string; - sizes: Array<{ name: string, w: number, h: number }> -} - -interface TweetConstructorArguments { - date: Date; - caption: string; - author: TwitterUser; - media: TweetMedia[]; - id: string; - hidden: boolean; -} - -export class Tweet { - public type: string = TWEET_TYPE; - public date: Date; - public id: string; - public caption: string; - public author: TwitterUser; - public media: TweetMedia[]; - public hidden: boolean; - - constructor ({ date, caption, author, media, id, hidden }: TweetConstructorArguments) { - this.id = id; - this.date = date; - this.caption = caption; - this.author = author; - this.media = media; - this.hidden = hidden; - } -} - -interface RetweetConstructorArguments extends TweetConstructorArguments { - tweet: Tweet; -} - -export class Retweet extends Tweet { - public type: string = RETWEET_TYPE; - public retweetedTweet: Tweet; - - constructor ({ date, caption, author, tweet, media, id, hidden }: RetweetConstructorArguments) { - super({ date, caption, author, media, id, hidden }); - this.retweetedTweet = tweet; - } -} - -const fields: Array<'hashtags'|'media'|'urls'|'user_mentions'> = ['hashtags', 'media', 'urls', 'user_mentions']; - -/** - * @description Enriches text with links to user mentions or hashtags - * @author DerZade - * @param {string} text Text - * @param {TweetEntities} entries Entries to add to the text - * @returns {string} Enriched text - */ -const enrichTwitterCaption = (text: string, entities: TweetEntities): string => { - // Each entity has a start and a end position. We have to insert the entities back to front - // to ensure that we won't fuck up the index-positions of following entities. - // This means we have to somehow sort all entities, therefore we need an array with all Entities - - // this will contain all entities in the correct order (first entity is last in text) - const allEntities: Array<{ - type: string; - end: number; - start: number; - entity: HashtagEntity|MediaEntity|UserMentionEntity|UrlEntity; - }> = []; - - // add all entities to the array - fields.forEach(field => { - const arr = entities[field]; - - if (!arr) return; - - arr.forEach((entity: HashtagEntity|MediaEntity|UserMentionEntity|UrlEntity) => { - if (!entity.indices) return; - allEntities.push({ type: field, start: entity.indices[0], end: entity.indices[1], entity }); - }); - }); - - const sortedEntities = allEntities.sort((a, b) => b.end - a.end); - - // we won't use text.split(''), because twitter uses indices based on unicode chars - // see https://stackoverflow.com/a/35223207 - const graphemes = Array.from(text); - - // now we can start inserting into the text - sortedEntities.forEach(e => { - let insertText = ''; - - if (e.type === 'hashtags') { - const entity = e.entity as HashtagEntity; - const mid = graphemes.slice(e.start, e.end).join(''); - insertText = `${mid}`; - } else if (e.type === 'media') { - // media url will be just discarded - } else if (e.type === 'urls') { - const entity = e.entity as UrlEntity; - insertText = `${entity.display_url}`; - } else if (e.type === 'user_mentions') { - const entity = e.entity as UserMentionEntity; - const mid = graphemes.slice(e.start, e.end).join(''); - insertText = `${mid}`; - } - graphemes.splice(e.start, e.end - e.start, insertText); - }); - - return graphemes.join('').replace(/\n/g, '
'); -}; - -let hiddenTweets: number[]|null = null; -let hiddenTweetsPromise: Promise|null = null; -const getHiddenTweets = async (): Promise => { - if (hiddenTweets !== null) return hiddenTweets; - - if (hiddenTweetsPromise === null) { - hiddenTweetsPromise = fetchJSON(`${API_URI}/api/v1/twitter/hidden`).then(arr => { hiddenTweets = arr; return arr; }); - } - - return hiddenTweetsPromise; -}; - -export async function hideTweet (id: string, hidden: boolean): Promise { - const response = await fetch(`${API_URI}/api/v1/twitter/hidden/${id}`, { credentials: 'include', method: hidden ? 'POST' : 'DELETE' }); - - if (!response.ok) throw new ResponseError(response); -} - -function entityToTweetMedia (entity: MediaEntity): TweetMedia { - const sizes = []; - for (const name of ['small', 'medium', 'large'] as Array<'small'|'medium'|'large'>) { - if (!Object.prototype.hasOwnProperty.call(entity.sizes, name)) continue; - - const { h, w } = entity.sizes[name]; - sizes.push({ name, h, w }); - } - - return { - id: entity.id, - url: entity.media_url_https, - target: entity.expanded_url, - sizes - }; -} - -/** - * @async - * @description Retrieves tweets - * @author DerZade - * @param {string} maxId MaxId to pass th twitter api (see twitter api GET statuses/user_timeline parameters max_id) - * (https://developer.twitter.com/en/docs/tweets/timelines/api-reference/get-statuses-user_timeline.html) - * @returns {Promise} Tweets - */ -export async function fetchTweets (maxId?: string): Promise { - const params = maxId !== undefined ? `?max_id=${maxId}&count=20&exclude_replies=true` : '?count=20&exclude_replies=true'; - - const responseTweetsProm = fetchJSON(`${API_URI}/api/v1/twitter${params}`); - - const hiddenTweets = await getHiddenTweets(); - const responseTweets = await responseTweetsProm; - - const tweets = responseTweets.map(mainTweet => { - const mainId: string = mainTweet.id_str; - - // author of main tweet - const mainAuthor: TwitterUser = { - username: (mainTweet.user as FullTwitterUser).screen_name, - displayName: (mainTweet.user as FullTwitterUser).name, - picture: (mainTweet.user as FullTwitterUser).profile_image_url_https - }; - - // media of main tweet - let mainMedia: TweetMedia[] = []; - if (mainTweet.extended_entities && mainTweet.extended_entities.media) { - mainMedia = mainTweet.extended_entities.media.map(entityToTweetMedia); - } - - // content of main tweet - let mainCaption; - - // quoted_status and retweeted_status both contain the tweet that was retweeted with the - // only difference being that the parent tweet of quoted_status will contain a own comment - // and parent of retweeted_status is just a retweet of that status without any extra comment - let retweetedStatus: ApiResTweet|null = null; - if (mainTweet.quoted_status) { - retweetedStatus = mainTweet.quoted_status; - - if (Object.prototype.hasOwnProperty.call(mainTweet, 'quoted_status_permalink')) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const url = mainTweet.quoted_status_permalink!.url; - - // the full_text of the mainTweet will include a link of the quoted tweet, and we dont want that - mainTweet.full_text = mainTweet.full_text.replace(url, ''); - - mainTweet.entities.urls = (mainTweet.entities.urls || []).filter(e => e.url !== url); - } - mainCaption = enrichTwitterCaption(mainTweet.full_text, mainTweet.entities); - } else if (mainTweet.retweeted_status) { - retweetedStatus = mainTweet.retweeted_status; - - mainCaption = ''; // full_text of main tweet will just contain part of the retweeted full_text - } else { - mainCaption = enrichTwitterCaption(mainTweet.full_text, mainTweet.entities); - } - - if (retweetedStatus !== null) { - const retweetedId = retweetedStatus.id_str; - - // retweeted media - let retweetedMedia: TweetMedia[] = []; - if (retweetedStatus.extended_entities && retweetedStatus.extended_entities.media) { - retweetedMedia = retweetedStatus.extended_entities.media.map(entityToTweetMedia); - } - - const retweetedTweet = new Tweet({ - id: retweetedId, - media: retweetedMedia, - date: new Date(retweetedStatus.created_at), - caption: enrichTwitterCaption(retweetedStatus.full_text, retweetedStatus.entities), - author: { - username: (retweetedStatus.user as FullTwitterUser).screen_name, - displayName: (retweetedStatus.user as FullTwitterUser).name, - picture: (retweetedStatus.user as FullTwitterUser).profile_image_url_https - }, - hidden: false - }); - - // media of retweeted tweet will also appear in media of main tweet and we don't like that shit ^^ - // so remove everything from main tweet media, which already is in retweeted status - mainMedia = mainMedia.filter(photo => { - return retweetedMedia.findIndex(retweetedPhoto => retweetedPhoto.id === photo.id) === -1; - }); - - return new Retweet({ - date: new Date(mainTweet.created_at), - caption: mainCaption, - media: mainMedia, - id: mainId, - author: mainAuthor, - tweet: retweetedTweet, - hidden: hiddenTweets.includes(Number.parseInt(mainId, 10)) - }); - } - - // tweet is just regular tweet - return new Tweet({ - date: new Date(mainTweet.created_at), - caption: mainCaption, - media: mainMedia, - id: mainId, - author: mainAuthor, - hidden: hiddenTweets.includes(Number.parseInt(mainId, 10)) - }); - }); - - // return tweets without the one specified in maxId - return tweets.filter(t => t.id !== maxId).sort((a, b) => b.date.getTime() - a.date.getTime()); -}