Merge branch 'sql'

database/dump_database.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ $# -lt 3 ];
+then
+  echo "Usage: $(basename $0) <DB_HOST> <DB_USER> <DB_NAME> [<DB_PASS>]" && exit 1
+fi
+
+if [ $# -eq 3 ];
+then
+  echo -n "DB password: "
+  read -s DB_pass
+else
+  DB_pass=$4
+fi
+
+DB_host=$1
+DB_user=$2
+DB=$3
+
+test -d $DB || mkdir -p $DB
+
+echo
+echo "Dumping tables into separate SQL command files for database '$DB'"
+
+tbl_count=0
+
+for t in $(mysql -NBA -h $DB_host -u $DB_user -p$DB_pass -D $DB -e 'show tables')
+do
+  echo "DUMPING TABLE: $t"
+  mysqldump --compact --no-data -h $DB_host -u $DB_user -p$DB_pass $DB $t | sed 's/ AUTO_INCREMENT=[0-9]*//g' | sed -e 's/DEFINER[ ]*=[ ]*[^*]*\*/\*/' > $DB/$t.sql
+  (( tbl_count++ ))
+done
+
+echo "$tbl_count tables dumped from database '$DB'"

database/make_fresh_deploy.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+[ $# -lt 1 ] && echo "Usage: $(basename $0) <DB_NAME>" && exit 1
+
+DB=$1
+
+echo "Creating initial dump script for '$DB'"
+
+cat <(echo "SET FOREIGN_KEY_CHECKS=0;") > fresh_deploy_$DB.sql
+
+cat <(echo "CREATE DATABASE $DB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;;") >> fresh_deploy_$DB.sql
+
+cat <(echo "USE $DB;") >> fresh_deploy_$DB.sql
+
+FILES=$DB/*.sql
+for f in $FILES
+do
+  if [ "$f" == "$DB/RefData.sql" ]
+  then
+    continue;
+  fi
+  echo "Processing $f..."
+  cat $f >> fresh_deploy_$DB.sql
+done

database/once_tweet/Feeds.sql

@@ -0,0 +1,12 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `Feeds` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `Name` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `Handle` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `ScreenNames` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `Key` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `Secret` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;

database/once_tweet/Items.sql

@@ -0,0 +1,14 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `Items` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `Hash` char(40) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `ScreenName` varchar(15) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `id_str` varchar(24) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `Url` text COLLATE utf8mb4_unicode_ci,
+  `created_at` int(10) unsigned NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `Hash` (`Hash`),
+  KEY `created_at` (`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;

database/once_tweet/Meta.sql

@@ -0,0 +1,8 @@
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `Meta` (
+  `Name` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `Value` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  PRIMARY KEY (`Name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;

index.js

@@ -3,19 +3,22 @@
 const Feeds = require('./src/Feeds');
 
 const run = function() {
-  return Feeds.get().then((feeds) => {
-    return feeds.run();
-  }).catch((err) => {
-    console.log(err);
+  const db = require('./src/db')();
+  return Feeds.get(db).then((feeds) => {
+    return feeds.run().then(() => {
+      return feeds.cleanup();
+    });
+  }).then(() => {
+    return db.end();
   });
 };
 
 module.exports = {
   handler: (event, context, callback) => {
     run().then(() => {
-      callback();
+      context.done();
     }).catch((err) => {
-      callback(err);
+      context.done(err);
     });
   }
 };

package.json

@@ -1,12 +1,19 @@
 {
   "name": "once-tweet",
   "main": "index.js",
+  "private": true,
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sedenardi/once-tweet.git"
+  },
+  "author": "Sanders DeNardi",
+  "homepage": "https://github.com/sedenardi/once-tweet#readme",
   "dependencies": {
-    "aws-sdk": "^2.7.27",
     "bignumber.js": "^4.0.0",
     "cheerio": "^0.22.0",
     "lodash": "^4.17.4",
     "moment": "^2.17.1",
+    "mysql2": "^1.5.1",
     "request": "^2.79.0",
     "twitter": "^1.7.0"
   },

src/Feed.js

@@ -16,6 +16,7 @@ class Feed {
     this.Secret = feed.Secret;
 
     this.Since = feed.Since;
+    this.db = feed.db;
   }
   run() {
     const twit = twitter(this);
@@ -25,7 +26,7 @@ class Feed {
     return Promise.all(fetches).then((res) => {
       const itemActions = _(res)
         .flatten()
-        .map((t) => { return FeedItem.parse(t); })
+        .map((t) => { return FeedItem.parse(t, this.db); })
         .value();
       return Promise.all(itemActions);
     }).then((res) => {
@@ -52,8 +53,10 @@ class Feed {
     });
   }
 }
-Feed.parse = function(feed, meta) {
+Feed.parse = function(feed, meta, db) {
   feed.Since = _.find(meta, { Name: 'since_id' }).Value;
+  feed.ScreenNames = feed.ScreenNames.split(',');
+  feed.db = db;
   return new Feed(feed);
 };
 

src/FeedItem.js

@@ -3,30 +3,31 @@
 const moment = require('moment');
 const _ = require('lodash');
 const req = require('./req');
-const dynamo = require('./dynamo')();
+const crypto = require('crypto');
 
 class FeedItem {
   constructor(item) {
     this.id_str = item.id_str;
+    this.handle = item.handle;
     this.Url = item.Url;
+    this.Hash = item.Hash;
     this.created_at = item.created_at;
+    this.db = item.db;
   }
   save() {
-    return dynamo.put({
-      TableName: 'once_Items',
-      Item: {
-        Url: this.Url,
-        created_at: this.created_at
-      }
-    });
+    const sql = 'insert into once_tweet.Items(`Hash`,ScreenName,id_str,Url,created_at) value(?,?,?,?,?);';
+    return this.db.query(sql, [
+      this.Hash,
+      this.handle,
+      this.id_str,
+      this.Url,
+      this.created_at.toString().slice(0, -3)
+    ]);
   }
   run(twit, last) {
-    return dynamo.get({
-      TableName: 'once_Items',
-      Key: { Url: this.Url },
-      ConsistentRead: true
-    }).then((res) => {
-      if (res.Item) {
+    const sql = 'select * from once_tweet.Items where `Hash` = ?;';
+    return this.db.query(sql, [this.Hash]).then((res) => {
+      if (res[0]) {
         return Promise.resolve();
       }
       return this.save().then(() => {
@@ -37,10 +38,12 @@ class FeedItem {
     });
   }
 }
-FeedItem.parse = function(rawItem) {
+FeedItem.parse = function(rawItem, db) {
   const item = {
     id_str: rawItem.id_str,
-    created_at: moment(rawItem.created_at, 'dd MMM DD HH:mm:ss ZZ YYYY').valueOf()
+    handle: rawItem.user.screen_name,
+    created_at: moment(rawItem.created_at, 'dd MMM DD HH:mm:ss ZZ YYYY').valueOf(),
+    db: db
   };
 
   if (rawItem.entities.urls.length) {
@@ -53,6 +56,7 @@ FeedItem.parse = function(rawItem) {
       if (!url) { return Promise.resolve(null); }
       return req.head(url).then((resolvedUrl) => {
         item.Url = resolvedUrl.split(/[?#]/)[0].toLowerCase();
+        item.Hash = crypto.createHash('sha1').update(item.Url).digest('hex');
         return Promise.resolve(new FeedItem(item));
       });
     });

src/Feeds.js

@@ -4,10 +4,17 @@ const moment = require('moment');
 const _ = require('lodash');
 const BigNumber = require('bignumber.js');
 const Feed = require('./Feed');
-const dynamo = require('./dynamo')();
+
+const feedsQuery = (db) => { return db.query('select * from once_tweet.Feeds;'); };
+const metaQuery = (db) => { return db.query('select * from once_tweet.Meta;'); };
+const metaUpdate = (db, max) => {
+  const sql = 'update once_tweet.Meta set `Value` = ? where `Name` = \'since_id\'';
+  return db.query(sql, [max]);
+};
 
 class Feeds {
   constructor(feeds) {
+    this.db = feeds.db;
     this.Feeds = feeds.Feeds;
   }
   save(maxes) {
@@ -17,13 +24,7 @@ class Feeds {
       .value();
     if (compacted.length) {
       const max = BigNumber.max(compacted);
-      return dynamo.put({
-        TableName: 'once_Meta',
-        Item: {
-          Name: 'since_id',
-          Value: max.toString()
-        }
-      });
+      return metaUpdate(this.db, max.toString());
     } else {
       return Promise.resolve();
     }
@@ -34,16 +35,24 @@ class Feeds {
       return this.save(maxes);
     });
   }
+  cleanup() {
+    const threshold = moment().subtract(3, 'months').unix();
+    const sql = 'delete from once_tweet.Items where created_at < ?';
+    return this.db.query(sql, [threshold]).then((res) => {
+      console.log(`${res.affectedRows} rows cleaned up.`);
+      return Promise.resolve();
+    });
+  }
 }
-Feeds.get = function() {
+Feeds.get = function(db) {
   return Promise.all([
-    dynamo.scan({ TableName: 'once_Feeds' }),
-    dynamo.scan({ TableName: 'once_Meta' })
+    feedsQuery(db),
+    metaQuery(db)
   ]).then((res) => {
-    const feeds = res[0].Items.map((v) => {
-      return Feed.parse(v, res[1].Items);
+    const feeds = res[0].map((v) => {
+      return Feed.parse(v, res[1], db);
     });
-    return Promise.resolve(new Feeds({ Feeds: feeds }));
+    return Promise.resolve(new Feeds({ db: db, Feeds: feeds }));
   });
 };
 

src/db.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const config = require('../config');
+const mysql = require('mysql2');
+
+module.exports = function() {
+  const pool = mysql.createPool(config.mysql);
+  return {
+    query: function(sql, params) {
+      return new Promise((resolve, reject) => {
+        if (typeof sql !== 'string') {
+          return reject(new Error('Missing query string'));
+        }
+        if (params !== undefined && params.constructor !== Array) {
+          return reject(new Error('Params must be an array'));
+        }
+        pool.getConnection((err, conn) => {
+          if (err) { return reject(err); }
+          conn.query(sql, params, (error, res) => {
+            conn.release();
+            if (error) { return reject(error); }
+            return resolve(res);
+          });
+        });
+      });
+    },
+    end: function() {
+      return new Promise((resolve, reject) => {
+        pool.end((err) => {
+          if (err) { return reject(err); }
+          console.log('Closing database.');
+          return resolve();
+        });
+      });
+    }
+  };
+};