analysis_tests.py

# -*- coding: utf-8 -*-

from google.cloud.language.entity import Entity
from os import getenv
from pytest import fixture

from analysis import Analysis
from analysis import MID_TO_TICKER_QUERY
from twitter import Twitter


@fixture
def analysis():
    return Analysis(logs_to_cloud=False)


def get_tweet(tweet_id):
    """Looks up data for a single tweet."""

    twitter = Twitter(logs_to_cloud=False)
    return twitter.get_tweets([tweet_id])[0]


def get_tweet_text(tweet_id):
    """Looks up the text for a single tweet."""

    tweet = get_tweet(tweet_id)
    analysis = Analysis(logs_to_cloud=False)
    return analysis.get_expanded_text(tweet)


def test_environment_variables():
    assert getenv("GOOGLE_APPLICATION_CREDENTIALS")


def test_get_company_data(analysis):
    assert analysis.get_company_data("/m/035nm") == [{
        "exchange": "New York Stock Exchange",
        "name": "General Motors",
        "ticker": "GM"}]
    assert analysis.get_company_data("/m/04n3_w4") == [{
        "exchange": "New York Stock Exchange",
        "name": "Fiat",
        "root": "Fiat Chrysler Automobiles",
        "ticker": "FCAU"}]
    assert analysis.get_company_data("/m/0d8c4") == [{
        "exchange": "New York Stock Exchange",
        "name": "Lockheed Martin",
        "ticker": "LMT"}]
    assert analysis.get_company_data("/m/0hkqn") == [{
        "exchange": "New York Stock Exchange",
        "name": "Lockheed Martin",
        "ticker": "LMT"}]
    assert analysis.get_company_data("/m/09jcvs") == [{
        "exchange": "NASDAQ",
        "name": "YouTube",
        "root": "Google",
        "ticker": "GOOG"}, {
        "exchange": "NASDAQ",
        "name": "YouTube",
        "root": "Google",
        "ticker": "GOOGL"}, {
        "exchange": "NASDAQ",
        "name": "YouTube",
        "root": "Alphabet Inc.",
        "ticker": "GOOG"}, {
        "exchange": "NASDAQ",
        "name": "YouTube",
        "root": "Alphabet Inc.",
        "ticker": "GOOGL"}]
    assert analysis.get_company_data("/m/045c7b") == [{
        "exchange": "NASDAQ",
        "name": "Google",
        "ticker": "GOOG"}, {
        "exchange": "NASDAQ",
        "name": "Google",
        "ticker": "GOOGL"}, {
        "exchange": "NASDAQ",
        "name": "Google",
        "root": "Alphabet Inc.",
        "ticker": "GOOG"}, {
        "exchange": "NASDAQ",
        "name": "Google",
        "root": "Alphabet Inc.",
        "ticker": "GOOGL"}]
    assert analysis.get_company_data("/m/01snr1") == [{
        "exchange": "New York Stock Exchange",
        "name": "Bayer",
        "root": "BlackRock",
        "ticker": "BLK"}, {
        "exchange": "New York Stock Exchange",
        "name": "Bayer",
        "root": "PNC Financial Services",
        "ticker": "PNC"}]
    assert analysis.get_company_data("/m/02zs4") == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "ticker": "F"}]
    assert analysis.get_company_data("/m/0841v") == [{
        "exchange": "New York Stock Exchange",
        "name": "Walmart",
        "ticker": "WMT"}, {
        "exchange": "New York Stock Exchange",
        "name": "Walmart",
        "root": "State Street Corporation",
        "ticker": "STT"}]
    assert analysis.get_company_data("/m/07mb6") == [{
        "exchange": "New York Stock Exchange",
        "name": "Toyota",
        "ticker": "TM"}]
    assert analysis.get_company_data("/m/0178g") == [{
        "exchange": "New York Stock Exchange",
        "name": "Boeing",
        "ticker": "BA"}]
    assert analysis.get_company_data("/m/07_dc0") == [{
        "exchange": "New York Stock Exchange",
        "name": "Carrier Corporation",
        "root": "United Technologies Corporation",
        "ticker": "UTX"}]
    assert analysis.get_company_data("/m/01pkxd") == [{
        "exchange": "New York Stock Exchange",
        "name": "Macy's",
        "root": "Macy's, Inc.",
        "ticker": "M"}]
    assert analysis.get_company_data("/m/02rnkmh") == [{
        "exchange": "New York Stock Exchange",
        "name": "Keystone Pipeline",
        "root": "TransCanada Corporation",
        "ticker": "TRP"}]
    assert analysis.get_company_data("/m/0k9ts") == [{
        "exchange": "New York Stock Exchange",
        "name": "Delta Air Lines",
        "ticker": "DAL"}]
    assert analysis.get_company_data("/m/017b3j") == [{
        "exchange": "NASDAQ",
        "name": "The Wall Street Journal",
        "root": "21st Century Fox",
        "ticker": "FOXA"}, {
        "exchange": "NASDAQ",
        "name": "The Wall Street Journal",
        "root": "News Corp",
        "ticker": "NWSA"}]
    assert analysis.get_company_data("/m/07k2d") == [{
        "exchange": "New York Stock Exchange",
        "name": "The New York Times",
        "root": "The New York Times Company",
        "ticker": "NYT"}]
    assert analysis.get_company_data("/m/02z_b") == [{
        "exchange": "NASDAQ",
        "name": "Fox News Channel",
        "root": "21st Century Fox",
        "ticker": "FOXA"}, {
        "exchange": "NASDAQ",
        "name": "Fox News Channel",
        "root": "News Corp",
        "ticker": "NWSA"}]
    assert analysis.get_company_data("/m/0d6lp") == None
    assert analysis.get_company_data("xyz") == None
    assert analysis.get_company_data("") == None


def test_entity_tostring(analysis):
    assert analysis.entity_tostring(Entity(
        name="General Motors",
        entity_type="ORGANIZATION",
        metadata={
            "mid": "/m/035nm",
            "wikipedia_url": "http://en.wikipedia.org/wiki/General_Motors"},
        salience=0.33838183,
        mentions=["General Motors"])) == (
            '{name: "General Motors",'
            ' entity_type: "ORGANIZATION",'
            ' wikipedia_url: "http://en.wikipedia.org/wiki/General_Motors",'
            ' metadata: {"mid": "/m/035nm"},'
            ' salience: 0.33838183,'
            ' mentions: ["General Motors"]}')
    assert analysis.entity_tostring(Entity(
        name="jobs",
        entity_type="OTHER",
        metadata={},
        salience=0.31634554,
        mentions=["jobs"])) == (
        '{name: "jobs",'
        ' entity_type: "OTHER",'
        ' wikipedia_url: None,'
        ' metadata: {},'
        ' salience: 0.31634554,'
        ' mentions: ["jobs"]}')


def test_entities_tostring(analysis):
    assert analysis.entities_tostring([Entity(
        name="General Motors",
        entity_type="ORGANIZATION",
        metadata={
            "mid": "/m/035nm",
            "wikipedia_url": "http://en.wikipedia.org/wiki/General_Motors"},
        salience=0.33838183,
        mentions=["General Motors"]), Entity(
        name="jobs",
        entity_type="OTHER",
        metadata={"wikipedia_url": None},
        salience=0.31634554,
        mentions=["jobs"])]) == (
        '[{name: "General Motors",'
        ' entity_type: "ORGANIZATION",'
        ' wikipedia_url: "http://en.wikipedia.org/wiki/General_Motors",'
        ' metadata: {"mid": "/m/035nm"},'
        ' salience: 0.33838183,'
        ' mentions: ["General Motors"]}, '
        '{name: "jobs",'
        ' entity_type: "OTHER",'
        ' wikipedia_url: None,'
        ' metadata: {},'
        ' salience: 0.31634554,'
        ' mentions: ["jobs"]}]')
    assert analysis.entities_tostring([]) == "[]"


def test_get_sentiment(analysis):
    assert analysis.get_sentiment(get_tweet_text("806134244384899072")) < 0
    # assert analysis.get_sentiment(get_tweet_text("812061677160202240")) > 0
    assert analysis.get_sentiment(get_tweet_text("816260343391514624")) < 0
    assert analysis.get_sentiment(get_tweet_text("816324295781740544")) > 0
    assert analysis.get_sentiment(get_tweet_text("816635078067490816")) > 0
    assert analysis.get_sentiment(get_tweet_text("817071792711942145")) < 0
    assert analysis.get_sentiment(get_tweet_text("818460862675558400")) > 0
    assert analysis.get_sentiment(get_tweet_text("818461467766824961")) > 0
    assert analysis.get_sentiment(get_tweet_text("821415698278875137")) > 0
    # assert analysis.get_sentiment(get_tweet_text("821697182235496450")) > 0
    assert analysis.get_sentiment(get_tweet_text("821703902940827648")) > 0
    assert analysis.get_sentiment(get_tweet_text("803808454620094465")) > 0
    assert analysis.get_sentiment(get_tweet_text("621669173534584833")) < 0
    assert analysis.get_sentiment(get_tweet_text("664911913831301123")) < 0
    # assert analysis.get_sentiment(get_tweet_text("823950814163140609")) > 0
    assert analysis.get_sentiment(get_tweet_text("824055927200423936")) > 0
    assert analysis.get_sentiment(get_tweet_text("826041397232943104")) < 0
    assert analysis.get_sentiment(get_tweet_text("824765229527605248")) > 0
    assert analysis.get_sentiment(get_tweet_text("827874208021639168")) < 0
    assert analysis.get_sentiment(get_tweet_text("828642511698669569")) < 0
    assert analysis.get_sentiment(get_tweet_text("828793887275761665")) < 0
    assert analysis.get_sentiment(get_tweet_text("829410107406614534")) > 0
    assert analysis.get_sentiment(get_tweet_text("829356871848951809")) < 0
    assert analysis.get_sentiment("") == 0


def test_find_companies(analysis):
    assert analysis.find_companies(get_tweet("806134244384899072")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Boeing",
        "sentiment": -0.1,
        "ticker": "BA"}]
    assert analysis.find_companies(get_tweet("812061677160202240")) == [{
        # "exchange": "New York Stock Exchange",
        # "name": "Lockheed Martin",
        # "sentiment": -0.1,
        # "ticker": "LMT"}, {
        "exchange": "New York Stock Exchange",
        "name": "Boeing",
        "sentiment": 0,  # 0.1,
        "ticker": "BA"}]
    assert analysis.find_companies(get_tweet("816260343391514624")) == [{
        "exchange": "New York Stock Exchange",
        "name": "General Motors",
        "sentiment": -0.1,
        "ticker": "GM"}]
    assert analysis.find_companies(get_tweet("816324295781740544")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "sentiment": 0.5,
        "ticker": "F"}]
    assert analysis.find_companies(get_tweet("816635078067490816")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "sentiment": 0.3,
        "ticker": "F"}]
    assert analysis.find_companies(get_tweet("817071792711942145")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Toyota",
        "sentiment": -0.2,
        "ticker": "TM"}]
    assert analysis.find_companies(get_tweet("818460862675558400")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Fiat",
        "root": "Fiat Chrysler Automobiles",
        "sentiment": 0.2,
        "ticker": "FCAU"}]
    assert analysis.find_companies(get_tweet("818461467766824961")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "sentiment": 0.3,
        "ticker": "F"}, {
        "exchange": "New York Stock Exchange",
        "name": "Fiat",
        "root": "Fiat Chrysler Automobiles",
        "sentiment": 0.3,
        "ticker": "FCAU"}]
    assert analysis.find_companies(get_tweet("821415698278875137")) == [{
        "exchange": "New York Stock Exchange",
        "name": "General Motors",
        "sentiment": 0.4,
        "ticker": "GM"}, {
        "exchange": "New York Stock Exchange",
        "name": "Walmart",
        "sentiment": 0.4,
        "ticker": "WMT"}, {
        "exchange": "New York Stock Exchange",
        "name": "Walmart",
        "root": "State Street Corporation",
        "sentiment": 0.4,
        "ticker": "STT"}]
    assert analysis.find_companies(get_tweet("821697182235496450")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "sentiment": -0.6,  # 0,
        "ticker": "F"}, {
        "exchange": "New York Stock Exchange",
        "name": "General Motors",
        "sentiment": -0.6,  # 0,
        "ticker": "GM"}, {
        "exchange": "New York Stock Exchange",
        "name": "Lockheed Martin",
        "sentiment": -0.6,  # 0,
        "ticker": "LMT"}]
    assert analysis.find_companies(get_tweet("821703902940827648")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Bayer",
        "sentiment": 0.4,
        "root": "BlackRock",
        "ticker": "BLK"}, {
        "exchange": "New York Stock Exchange",
        "name": "Bayer",
        "sentiment": 0.4,
        "root": "PNC Financial Services",
        "exchange": "New York Stock Exchange",
        "ticker": "PNC"}, {
        "exchange": "NASDAQ",
        "name": "The Wall Street Journal",
        "sentiment": 0.4,
        "root": "21st Century Fox",
        "ticker": "FOXA"}, {
        "exchange": "NASDAQ",
        "name": "The Wall Street Journal",
        "sentiment": 0.4,
        "root": "News Corp",
        "ticker": "NWSA"}]
    # assert analysis.find_companies(get_tweet("803808454620094465")) == [{
    #     "exchange": "New York Stock Exchange",
    #     "name": "Carrier Corporation",
    #     "sentiment": 0.5,
    #     "root": "United Technologies Corporation",
    #     "ticker": "UTX"}]
    assert analysis.find_companies(get_tweet("621669173534584833")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Macy's",
        "root": "Macy's, Inc.",
        "sentiment": -0.5,
        "ticker": "M"}]
    assert analysis.find_companies(get_tweet("664911913831301123")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Macy's",
        "root": "Macy's, Inc.",
        "sentiment": -0.3,
        "ticker": "M"}]
    assert analysis.find_companies(get_tweet("823950814163140609")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Keystone Pipeline",
        "root": "TransCanada Corporation",
        "sentiment": -0.1,  # 0.1,
        "ticker": "TRP"}]
    assert analysis.find_companies(get_tweet("824055927200423936")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Ford",
        "sentiment": 0.5,
        "ticker": "F"}, {
        "exchange": "New York Stock Exchange",
        "name": "General Motors",
        "sentiment": 0.5,
        "ticker": "GM"}]
    assert analysis.find_companies(get_tweet("826041397232943104")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Delta Air Lines",
        "sentiment": -0.4,
        "ticker": "DAL"}]
    assert analysis.find_companies(get_tweet("824765229527605248")) == [{
        "exchange": "NASDAQ",
        "name": "Fox News Channel",
        "root": "21st Century Fox",
        "sentiment": 0.5,
        "ticker": "FOXA"}, {
        "exchange": "NASDAQ",
        "name": "Fox News Channel",
        "root": "News Corp",
        "sentiment": 0.5,
        "ticker": "NWSA"}]
    assert analysis.find_companies(get_tweet("827874208021639168")) == [{
        "exchange": "New York Stock Exchange",
        "name": "The New York Times",
        "root": "The New York Times Company",
        "sentiment": -0.9,
        "ticker": "NYT"}]
    assert analysis.find_companies(get_tweet("828642511698669569")) == [{
        "exchange": "New York Stock Exchange",
        "name": "The New York Times",
        "root": "The New York Times Company",
        "sentiment": -0.6,
        "ticker": "NYT"}]
    assert analysis.find_companies(get_tweet("828793887275761665")) == [{
        "exchange": "New York Stock Exchange",
        "name": "The New York Times",
        "root": "The New York Times Company",
        "sentiment": -0.8,
        "ticker": "NYT"}]
    assert analysis.find_companies(get_tweet("829410107406614534")) == [{
        "exchange": "NASDAQ",
        "name": "Intel",
        "sentiment": 0.8,
        "ticker": "INTC"}]
    assert analysis.find_companies(get_tweet("829356871848951809")) == [{
        "exchange": "New York Stock Exchange",
        "name": "Nordstrom",
        "sentiment": -0.2,
        "ticker": "JWN"}]
    assert analysis.find_companies("") == []


def test_get_expanded_text(analysis):
    assert analysis.get_expanded_text(get_tweet("829410107406614534")) == (
        u"Thank you Brian Krzanich, CEO of Intel. A great investment ($7 BILLI"
        u"ON) in American INNOVATION and JOBS!\u2026 https://t.co/oicfDsPKHQ")
    assert analysis.get_expanded_text(get_tweet("828574430800539648")) == (
        "Any negative polls are fake news, just like the CNN, ABC, NBC polls i"
        "n the election. Sorry, people want border security and extreme vettin"
        "g.")
    assert analysis.get_expanded_text(get_tweet("828642511698669569")) == (
        "The failing The New York Times writes total fiction concerning me. Th"
        "ey have gotten it wrong for two years, and now are making up stories "
        "&amp; sources!")
    assert analysis.get_expanded_text("") == None


def test_make_wikidata_request(analysis):
    assert analysis.make_wikidata_request(
        MID_TO_TICKER_QUERY % "/m/07k2d") == [{
            "companyLabel": {
                "type": "literal",
                "value": "The New York Times",
                "xml:lang": "en"},
            "rootLabel": {
                "type": "literal",
                "value": "The New York Times Company",
                "xml:lang": "en"},
            "exchangeNameLabel": {
                "type": "literal",
                "value": "New York Stock Exchange",
                "xml:lang": "en"},
            "tickerLabel": {
                "type": "literal",
                "value": "NYT"}}]
    assert analysis.make_wikidata_request("") == None