Using Real Movie Data From Amazon

# Importing
import feather
import pandas as pd
import numpy as np
import mpld3 as mpld3
from scipy.sparse import coo_matrix
from sklearn.decomposition import TruncatedSVD
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.utils import shuffle

# Constants:
REDUCED_NUM_REVIEWERS = 1000
REDUCED_NUM_PRODUCTS = 1000
MIN_NUM_REVIEWS_MOVIE = 250
MIN_NUM_REVIEWS_PERSON = 10

# Reading data
df = feather.read_dataframe("../results/movie_df.feather")

# Find number of reviewers per movie and
# the number of movies reviewed by each person
n_reviews_movie_df = df.groupby('asin').reviewer_id.nunique()
n_reviews_person_df = df.groupby('reviewer_id').asin.nunique()

# Get list of movies that have been reviewed
# more than MIN_NUM_REVIEWS_MOVIE times and
# users that have reviewed more than
# MIN_NUM_REVIEWS_PERSON
popular_movies = list(n_reviews_movie_df[n_reviews_movie_df > MIN_NUM_REVIEWS_MOVIE].index)
critical_people = list(n_reviews_person_df[n_reviews_person_df > MIN_NUM_REVIEWS_PERSON].index)

# Filter dataframe to use only popular movies
# and critical people:
popular_df = df[df['asin'].isin(popular_movies)]
popular_df = popular_df[popular_df['reviewer_id'].isin(critical_people)]

# Peak at data:
popular_df[popular_df.reviewer_id == "A1006V961PBMKA"]

# Get ID's to be used later as sanity checks (this is the Beavis and Butt-head review above)
person_of_interest = "A1006V961PBMKA"
product_of_interest = "157330686X"

# Only take relevant columns:
rating_df = popular_df[['reviewer_id', 'asin', 'overall']]

# Obtaining number of reviewers and products reviewed:
n_reviewers = len(rating_df.reviewer_id.unique())
n_products = len(rating_df.asin.unique())

# Print results:
print("There are", n_reviewers, "reviewers")
print("There are", n_products, "products reviewed")

# Creating map to map from reviewer id/asin(product id) to a number and vice versa:
reviewerID_to_index_map = dict(zip(np.unique(rating_df.reviewer_id), list(range(n_reviewers))))
productID_to_index_map = dict(zip(np.unique(rating_df.asin), list(range(n_products))))
productID_to_title_map = dict(zip(popular_df.asin, popular_df.title))
index_to_productID_map = dict((reversed(item) for item in productID_to_index_map.items()))

# Creating dataframe with reviewer_id. This is done because we need to shuffle
# an array made below and need to keep track of how we shuffled it.
tracker_df = pd.DataFrame({ 'reviewer_id': np.unique(rating_df.reviewer_id)})

# Obtain indicies associated with each id.
# This is used to create a sparse matrix later.
reviewer_index = np.array([reviewerID_to_index_map[reviewer_id] for reviewer_id in rating_df.reviewer_id])
product_index = np.array([productID_to_index_map[asin] for asin in rating_df.asin])

# Obtain ratings to be put into sparse matrix:
ratings = np.array(rating_df.overall)

# Create sparse X matrix:
X = coo_matrix((ratings, (reviewer_index, product_index)), shape=(n_reviewers, n_products))

# Performing sanity check:
person_of_interest_index = reviewerID_to_index_map[person_of_interest]
product_of_interest_index = productID_to_index_map[product_of_interest]

X.toarray()[person_of_interest_index, product_of_interest_index]

# Using dense array for now:
X_array_all = X.toarray()

# Reduce size of dataset (REMOVE LATER):
# The main issue occurs below during gradient descent.
# np.dot() takes a long time. This is discussed further below:
X_array = X_array_all[0:REDUCED_NUM_REVIEWERS, 0:REDUCED_NUM_PRODUCTS]

# Set up parameters:
n_iters = 2000
alpha = 0.0005
num_latent_features = 2

# Randomly initialize:
U = np.random.randn(REDUCED_NUM_REVIEWERS, num_latent_features) * 1e-5
V = np.random.randn(num_latent_features, REDUCED_NUM_PRODUCTS) * 1e-5

# Perform gradient descent:
for i in range(n_iters):

    # Obtain predictions:
    X_hat = np.dot(U, V)     

    # Make sure the person of interest didn't blow up:
    if np.isnan(X_hat[person_of_interest_index, product_of_interest_index]):
        print("ERROR")
        break

    # Obtain residual
    resid = X_hat - X_array
    resid[np.isnan(resid)] = 0

    # Calculate gradients:
    dU = np.dot(resid, V.T)
    dV = np.dot(U.T, resid)

    # Update values:
    U = U - dU*alpha
    V = V - dV*alpha

    # Output every 10% to make sure on the right track:
    if (i%(n_iters/10) == 0):
        print("Iteration:", i,
              "   Cost:", np.sum(resid**2),
              "   Rating of interest:", X_hat[person_of_interest_index, product_of_interest_index])

X_pred = np.dot(U, V)

predicted_val = X_pred[person_of_interest_index, product_of_interest_index]
actual_val = X_array[person_of_interest_index, product_of_interest_index]
print("Actual Rating: ", actual_val)
print("Predicted Rating: ", predicted_val)   

# Replacing zeros with nan:
# (THIS IS THE FIX)
X_array[X_array==0] = np.nan

# Set up parameters:
n_iters = 2000
alpha = 0.0005
num_latent_features = 2

# Initialize U and V randomly
U = np.random.randn(REDUCED_NUM_REVIEWERS, num_latent_features) * 1e-5
V = np.random.randn(num_latent_features, REDUCED_NUM_PRODUCTS) * 1e-5

# Perform gradient descent:
for i in range(n_iters):        
    # Obtain predictions:
    X_hat = np.dot(U, V)     

    if np.isnan(X_hat[person_of_interest_index, product_of_interest_index]):
        print("ERROR")
        break

    # Obtain residual
    resid = X_hat - X_array
    resid[np.isnan(resid)] = 0

    # Calculate gradients:
    dU = np.dot(resid, V.T)
    dV = np.dot(U.T, resid)

    # Update values:
    U = U - dU*alpha
    V = V - dV*alpha

    # Output every 10% to make sure on the right track:
    if (i%(n_iters/10) == 0):
        print("Iteration:", i,
              "   Cost:", np.sum(resid**2),
              "   Rating of interest:", X_hat[person_of_interest_index, product_of_interest_index])

# Make prediction:
X_pred = np.dot(U, V)

predicted_val = X_pred[person_of_interest_index, product_of_interest_index]
actual_val = X_array[person_of_interest_index, product_of_interest_index]
print("Actual Rating: ", actual_val)
print("Predicted Rating: ", predicted_val)

# Replacing zeros with nan:
X_array_all[X_array_all==0] = np.nan

# Randomly initialize:
num_latent_features = 2
U = np.random.randn(n_reviewers, num_latent_features) * 1e-5
V = np.random.randn(num_latent_features, n_products) * 1e-5
k = 1

# Initialize X_hat:
X_hat = U@V

batch_size = 200
n_iters = 2000

alpha = 0.0005

# Perform gradient descent:
for i in range(n_iters):

    # Shuffle users:
    (X_array_all, U, tracker_df) = shuffle(X_array_all, U, tracker_df)
    person_of_interest_index = np.where(tracker_df.reviewer_id == person_of_interest)[0][0]

    for j in range(0, n_reviewers, batch_size):
        # Make sure doesn't pass array size
        if j + batch_size > n_reviewers:
            break

        # Select mini batches of data:
        mini_X_array = X_array_all[j:j+batch_size, :]
        mini_U = U[j:j+batch_size, :]

        # Calculate predicted values
        mini_X_hat = mini_U@V

        # Calculate error
        resid = mini_X_hat - mini_X_array
        resid[np.isnan(resid)] = 0

        mini_dU = np.dot(resid, V.T)
        dV = np.dot(mini_U.T, resid)

        U[j:j+batch_size] = U[j:j+batch_size] - mini_dU*alpha
        V = V - dV*alpha

    if (i%(n_iters/10) == 0) or (i >= (n_iters-1)):
        print( "Iteration:", i,  
               "   Batch Cost:", np.sum(resid**2),
               "   Person of Interest:", np.dot(U[person_of_interest_index,:], V[:, product_of_interest_index]))


X_pred = U@V
predicted_val = X_pred[person_of_interest_index, product_of_interest_index]
actual_val = X_array_all[person_of_interest_index, product_of_interest_index]

print("Actual Rating: ", actual_val)
print("Predicted Rating: ", predicted_val)   

# Initializing parameters:
n_reviews = rating_df.shape[0]
prediction_list = list()
title_list = list()

# Matching ratings with proper reviewer and product:
for i in range(n_reviews):
    rev_id = rating_df.iloc[i].reviewer_id
    prod_id = rating_df.iloc[i].asin

    rev_index = reviewerID_to_index_map[rev_id]
    prod_index = productID_to_index_map[prod_id]

    prediction = X_pred[rev_index, prod_index]
    title = productID_to_title_map[prod_id]

    title_list.append(title)
    prediction_list.append(prediction)

# Making dataframe:
result_df = pd.DataFrame({"reviewer_id": rating_df.reviewer_id,
                          "product_name": title_list,
                          "actual_rating": rating_df.overall,
                          "predicted_rating": prediction_list
                         })

# Peaking at results:
shuffle(result_df, random_state=4242).head(10)

# Getting the two latent features of each movie
param1 = V.T[:,0]
param2 = V.T[:,1]

# Matching latent features to appropriate movie:
movie_title_list = list()
for index, productID in index_to_productID_map.items():
    movie_title_list.append(productID_to_title_map[productID])

# Plotting the products:
plt.figure(figsize=(12,8))
plt.plot(param1, param2, 'ro')
plt.show()

# Performing k-means:
kmeans = KMeans(n_clusters=10, random_state=0).fit(V.T)
groups = kmeans.labels_

# Making plot:
fig = plt.figure(figsize=(12,8))

scatter = plt.scatter(param1, param2, c=groups)

tooltip = mpld3.plugins.PointLabelTooltip(scatter, labels=movie_title_list)
mpld3.plugins.connect(fig, tooltip)

mpld3.display()