utilities.py

import pandas as pd
import numpy as np
import itertools
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt


def SimpleVectorizer(documents):
    cv = CountVectorizer(lowercase=False, min_df=1)
    term_matrix = cv.fit_transform(documents).todense()
    labels = cv.get_feature_names()
    results_df = pd.DataFrame(term_matrix, columns=labels, index=documents)
    return results_df

def DisplayTFMatrix(CVinstance, documents):
    term_matrix = CVinstance.fit_transform(documents).todense()
    labels = CVinstance.get_feature_names()
    return pd.DataFrame(term_matrix, columns=labels, index=documents)


# Courtesy of the Scikit learn examples:
# http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('Actual')
    plt.xlabel('Predicted')