Skip to content
Extraits de code Groupes Projets
Valider e78fa579 rédigé par Nathanaël Kindidi's avatar Nathanaël Kindidi
Parcourir les fichiers

corrected evaluator assignment 2 , data tiny

parent 92ed75ad
Aucune branche associée trouvée
Aucune étiquette associée trouvée
Aucune requête de fusion associée trouvée
Masquer les modifications d'espaces
En ligne Côte à côte
2_Evaluator_Block/configs.py
+ 1
− 1
Voir le fichier @ e78fa579
......@@ -26,7 +26,7 @@ class EvalConfig:
full_metrics = ["novelty"]
# Split parameters
test_size = 0.3 # -- configure the test_size (from 0 to 1) --
test_size = 0.25 # -- configure the test_size (from 0 to 1) --
# Loo parameters
top_n_value = 10 # -- configure the numer of recommendations (> 1) --
2_Evaluator_Block/constants.py
+ 1
− 1
Voir le fichier @ e78fa579
......@@ -6,7 +6,7 @@ class Constant:
"""Constants for dataset paths and column names."""
DATA_PATH = Path('../data/small') # -- fill here the dataset size to use
DATA_PATH = Path('../data/tiny') # -- fill here the dataset size to use
# Content
CONTENT_PATH = DATA_PATH / 'content' # Path to content data
......
2_Evaluator_Block/evaluator.ipynb
+ 99
− 25
Voir le fichier @ e78fa579
%% Cell type:markdown id:a665885b tags:
# Evaluator Module
The Evaluator module creates evaluation reports.
Reports contain evaluation metrics depending on models specified in the evaluation config.
%% Cell type:code id:6aaf9140 tags:
``` python
# reloads modules automatically before entering the execution of code
%load_ext autoreload
%autoreload 2
# third parties imports
import numpy as np
import pandas as pd
# -- add new imports here --
# local imports
from configs import EvalConfig
from constants import Constant as C
from loaders import export_evaluation_report
from loaders import load_ratings
# -- add new imports here --
from surprise.model_selection import train_test_split
from surprise import accuracy
from surprise.model_selection import LeaveOneOut
from collections import Counter
```
%% Output
The autoreload extension is already loaded. To reload it, use:
%reload_ext autoreload
%% Cell type:markdown id:d47c24a4 tags:
# 1. Model validation functions
Validation functions are a way to perform crossvalidation on recommender system models.
%% Cell type:code id:d6d82188 tags:
``` python
def generate_split_predictions(algo, ratings_dataset, eval_config):
"""Generate predictions on a random test set specified in eval_config"""
# -- implement the function generate_split_predictions --
# Spliting the data into train and test sets
trainset, testset = train_test_split(ratings_dataset, test_size=eval_config.test_size)
# Training the algorithm on the train data set
algo.fit(trainset)
# Predict ratings for the testset
predictions = algo.test(testset)
return predictions
def generate_loo_top_n(algo, ratings_dataset, eval_config):
"""Generate top-n recommendations for each user on a random Leave-one-out split (LOO)"""
# -- implement the function generate_loo_top_n --
# Create a LeaveOneOut split
loo = LeaveOneOut(n_splits=1)
for trainset, testset in loo.split(ratings_dataset):
algo.fit(trainset) # Train the algorithm on the training set
anti_testset = trainset.build_anti_testset() # Build the anti test-set
predictions = algo.test(anti_testset) # Get predictions on the anti test-set
top_n = {}
for uid, iid, _, est, _ in predictions:
if uid not in top_n:
top_n[uid] = []
top_n[uid].append((iid, est))
for uid, user_ratings in top_n.items():
user_ratings.sort(key=lambda x: x[1], reverse=True)
top_n[uid] = user_ratings[:eval_config.top_n_value] # Get top-N recommendations
anti_testset_top_n = top_n
return anti_testset_top_n, testset
def generate_full_top_n(algo, ratings_dataset, eval_config):
"""Generate top-n recommendations for each user with full training set (LOO)"""
full_trainset = ratings_dataset.build_full_trainset() # Build the full training set
algo.fit(full_trainset) # Train the algorithm on the full training set
anti_testset = full_trainset.build_anti_testset() # Build the anti test-set
predictions = algo.test(anti_testset) # Get predictions on the anti test-set
top_n = {}
for uid, iid, _, est, _ in predictions:
if uid not in top_n:
top_n[uid] = []
top_n[uid].append((iid, est))
for uid, user_ratings in top_n.items():
user_ratings.sort(key=lambda x: x[1], reverse=True)
top_n[uid] = user_ratings[:eval_config.top_n_value] # Get top-N recommendations
anti_testset_top_n = top_n
return anti_testset_top_n
def precompute_information(movie_data):
def precomputed_information(movie_data):
""" Returns a dictionary that precomputes relevant information for evaluating in full mode
Dictionary keys:
- precomputed_dict["item_to_rank"] : contains a dictionary mapping movie ids to rankings
- (-- for your project, add other relevant information here -- )
"""
# Initialize an empty dictionary to store item_id to rank mapping
item_to_rank = {}
# Calculate popularity rank for each movie
ratings_count = movie_data.groupby('movieId').size().sort_values(ascending=False)
# Assign ranks to movies based on their popularity
for rank, (movie_id, _) in enumerate(ratings_count.items(), start=1):
item_to_rank[movie_id] = rank
# Create the precomputed dictionary
precomputed_dict = {}
precomputed_dict["item_to_rank"] = item_to_rank
return precomputed_dict
def create_evaluation_report(eval_config, sp_ratings, precomputed_dict, available_metrics):
""" Create a DataFrame evaluating various models on metrics specified in an evaluation config.
"""
evaluation_dict = {}
for model_name, model, arguments in eval_config.models:
print(f'Handling model {model_name}')
algo = model(**arguments)
evaluation_dict[model_name] = {}
# Type 1 : split evaluations
if len(eval_config.split_metrics) > 0:
print('Training split predictions')
predictions = generate_split_predictions(algo, sp_ratings, eval_config)
for metric in eval_config.split_metrics:
print(f'- computing metric {metric}')
assert metric in available_metrics['split']
evaluation_function, parameters = available_metrics["split"][metric]
evaluation_dict[model_name][metric] = evaluation_function(predictions, **parameters)
# Type 2 : loo evaluations
if len(eval_config.loo_metrics) > 0:
print('Training loo predictions')
anti_testset_top_n, testset = generate_loo_top_n(algo, sp_ratings, eval_config)
for metric in eval_config.loo_metrics:
assert metric in available_metrics['loo']
evaluation_function, parameters = available_metrics["loo"][metric]
evaluation_dict[model_name][metric] = evaluation_function(anti_testset_top_n, testset, **parameters)
# Type 3 : full evaluations
if len(eval_config.full_metrics) > 0:
print('Training full predictions')
anti_testset_top_n = generate_full_top_n(algo, sp_ratings, eval_config)
for metric in eval_config.full_metrics:
assert metric in available_metrics['full']
evaluation_function, parameters = available_metrics["full"][metric]
evaluation_dict[model_name][metric] = evaluation_function(
anti_testset_top_n,
**precomputed_dict,
**parameters
)
return pd.DataFrame.from_dict(evaluation_dict).T
```
%% Cell type:markdown id:f7e83d1d tags:
# 2. Evaluation metrics
Implement evaluation metrics for either rating predictions (split metrics) or for top-n recommendations (loo metric, full metric)
%% Cell type:code id:f1849e55 tags:
``` python
def get_hit_rate(anti_testset_top_n, testset):
"""Compute the average hit over the users (loo metric)
A hit (1) happens when the movie in the testset has been picked by the top-n recommender
A fail (0) happens when the movie in the testset has not been picked by the top-n recommender
"""
# -- implement the function get_hit_rate --
hits = 0
total_users = len(testset)
for uid, true_iid, _ in testset:
if uid in anti_testset_top_n and true_iid in {iid for iid, _ in anti_testset_top_n[uid]}:
hits += 1
hit_rate = hits / total_users
return hit_rate
def get_novelty(anti_testset_top_n, item_to_rank):
"""Compute the average novelty of the top-n recommendation over the users (full metric)
The novelty is defined as the average ranking of the movies recommended
"""
# -- implement the function get_novelty --
total_rank_sum = 0
total_recommendations = 0
for uid, recommendations in anti_testset_top_n.items():
for iid, _ in recommendations:
if iid in item_to_rank:
total_rank_sum += item_to_rank[iid]
total_recommendations += 1
if total_recommendations == 0:
return 0 # Avoid division by zero
average_rank_sum = total_rank_sum / total_recommendations
return average_rank_sum
```
%% Cell type:markdown id:1a9855b3 tags:
# 3. Evaluation workflow
Load data, evaluate models and save the experimental outcomes
%% Cell type:code id:704f4d2a tags:
``` python
AVAILABLE_METRICS = {
"split": {
"mae": (accuracy.mae, {'verbose': False}),
"rmse": (accuracy.rmse, {'verbose': False})
# Add new split metrics here if needed
},
"loo": {
"hit_rate": (get_hit_rate, {}),
# Add new loo metrics here if needed
},
"full": {
"novelty": (get_novelty, {}),
# Add new full metrics here if needed
}
}
sp_ratings = load_ratings(surprise_format=True)
precomputed_dict = precompute_information(pd.read_csv("../data/small/evidence/ratings.csv"))
precomputed_dict = precomputed_information(pd.read_csv("../data/tiny/evidence/ratings.csv"))
evaluation_report = create_evaluation_report(EvalConfig, sp_ratings, precomputed_dict, AVAILABLE_METRICS)
export_evaluation_report(evaluation_report)
```
%% Output
Handling model baseline_1
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_2
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_3
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_4
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
The data has been exported to the evaluation report
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[185], line 20
18 precomputed_dict = precompute_information(pd.read_csv("../data/small/evidence/ratings.csv"))
19 evaluation_report = create_evaluation_report(EvalConfig, sp_ratings, precomputed_dict, AVAILABLE_METRICS)
---> 20 export_evaluation_report(evaluation_report)
File ~/vscodeworkspace/recomsys/Analytics_UI/loaders.py:49, in export_evaluation_report(report)
47 report_name = f"evaluation_report_{pd.Timestamp.now().strftime('%Y-%m-%d')}.csv"
48 export_path = os.path.join("../data", "small", "evaluations", report_name)
---> 49 df_items.to_csv(export_path, index=False)
50 print("The data has been exported to the evaluation report")
51 return df_items
AttributeError: 'dict' object has no attribute 'to_csv'
mae rmse hit_rate novelty
baseline_1 1.544940 1.776982 0.112150 99.405607
baseline_2 1.491063 1.844761 0.009346 429.942991
baseline_3 0.868139 1.066303 0.074766 99.405607
baseline_4 0.727803 0.927636 0.158879 57.328037
......
2_Evaluator_Block/loaders.py
+ 4
− 8
Voir le fichier @ e78fa579
......@@ -24,7 +24,7 @@ def load_ratings(surprise_format=False):
return surprise_data
else:
return df_ratings
print(load_ratings())
def load_items():
"""Loads items data.
......@@ -44,12 +44,8 @@ def export_evaluation_report(report):
Returns:
DataFrame: Merged ratings and items data.
"""
df_ratings = load_ratings()
df_items = load_items()
df_merge = pd.merge(df_ratings, df_items, on='movieId')
report_name = f"evaluation_report_{pd.Timestamp.now().strftime('%Y-%m-%d')}.csv"
export_path = os.path.join("../data", "small", "evaluations", report_name)
df_merge.to_csv(export_path, index=False)
export_path = os.path.join("../data", "tiny", "evaluations", report_name)
report.to_csv(export_path, index=False)
print("The data has been exported to the evaluation report")
return df_merge
\ No newline at end of file
return report
\ No newline at end of file
0% ou .
You are about to add 0 people to the discussion. Proceed with caution.
Terminez d'abord l'édition de ce message.
Veuillez vous inscrire ou vous pour commenter