Skip to content
Extraits de code Groupes Projets
Valider 4fa668fe rédigé par Audrey Ghilain's avatar Audrey Ghilain
Parcourir les fichiers

aud

parent 8583e402
Aucune branche associée trouvée
Aucune étiquette associée trouvée
Aucune requête de fusion associée trouvée
Masquer les modifications d'espaces
En ligne Côte à côte
Affichage de
avec 8 ajouts et 40 suppressions
evaluator.ipynb
+ 8
− 40
Voir le fichier @ 4fa668fe
%% Cell type:markdown id:a665885b tags:
# Evaluator Module
The Evaluator module creates evaluation reports.
Reports contain evaluation metrics depending on models specified in the evaluation config.
%% Cell type:code id:6aaf9140 tags:
``` python
# reloads modules automatically before entering the execution of code
%load_ext autoreload
%autoreload 2
# third parties imports
import numpy as np
import pandas as pd
# -- add new imports here --
# local imports
from configs import EvalConfig
from constants import Constant as C
from loaders import export_evaluation_report
from loaders import load_ratings
# -- add new imports here --
from surprise.model_selection import train_test_split
from surprise import accuracy
from surprise.model_selection import LeaveOneOut
from collections import Counter
```
%% Output
The autoreload extension is already loaded. To reload it, use:
%reload_ext autoreload
%% Cell type:markdown id:d47c24a4 tags:
# 1. Model validation functions
Validation functions are a way to perform crossvalidation on recommender system models.
%% Cell type:code id:d6d82188 tags:
``` python
def generate_split_predictions(algo, ratings_dataset, eval_config):
"""Generate predictions on a random test set specified in eval_config"""
# -- implement the function generate_split_predictions --
# Spliting the data into train and test sets
trainset, testset = train_test_split(ratings_dataset, test_size=eval_config.test_size)
# Training the algorithm on the train data set
algo.fit(trainset)
# Predict ratings for the testset
predictions = algo.test(testset)
return predictions
def generate_loo_top_n(algo, ratings_dataset, eval_config):
"""Generate top-n recommendations for each user on a random Leave-one-out split (LOO)"""
# -- implement the function generate_loo_top_n --
# Create a LeaveOneOut split
loo = LeaveOneOut(n_splits=1)
for trainset, testset in loo.split(ratings_dataset):
algo.fit(trainset) # Train the algorithm on the training set
anti_testset = trainset.build_anti_testset() # Build the anti test-set
predictions = algo.test(anti_testset) # Get predictions on the anti test-set
top_n = {}
for uid, iid, _, est, _ in predictions:
if uid not in top_n:
top_n[uid] = []
top_n[uid].append((iid, est))
for uid, user_ratings in top_n.items():
user_ratings.sort(key=lambda x: x[1], reverse=True)
top_n[uid] = user_ratings[:eval_config.top_n_value] # Get top-N recommendations
anti_testset_top_n = top_n
return anti_testset_top_n, testset
def generate_full_top_n(algo, ratings_dataset, eval_config):
"""Generate top-n recommendations for each user with full training set (LOO)"""
full_trainset = ratings_dataset.build_full_trainset() # Build the full training set
algo.fit(full_trainset) # Train the algorithm on the full training set
anti_testset = full_trainset.build_anti_testset() # Build the anti test-set
predictions = algo.test(anti_testset) # Get predictions on the anti test-set
top_n = {}
for uid, iid, _, est, _ in predictions:
if uid not in top_n:
top_n[uid] = []
top_n[uid].append((iid, est))
for uid, user_ratings in top_n.items():
user_ratings.sort(key=lambda x: x[1], reverse=True)
top_n[uid] = user_ratings[:eval_config.top_n_value] # Get top-N recommendations
anti_testset_top_n = top_n
return anti_testset_top_n
def precomputed_information(movie_data):
""" Returns a dictionary that precomputes relevant information for evaluating in full mode
Dictionary keys:
- precomputed_dict["item_to_rank"] : contains a dictionary mapping movie ids to rankings
- (-- for your project, add other relevant information here -- )
"""
# Initialize an empty dictionary to store item_id to rank mapping
item_to_rank = {}
# Calculate popularity rank for each movie
ratings_count = movie_data.groupby('movieId').size().sort_values(ascending=False)
# Assign ranks to movies based on their popularity
for rank, (movie_id, _) in enumerate(ratings_count.items(), start=1):
item_to_rank[movie_id] = rank
# Create the precomputed dictionary
precomputed_dict = {}
precomputed_dict["item_to_rank"] = item_to_rank
return precomputed_dict
def create_evaluation_report(eval_config, sp_ratings, precomputed_dict, available_metrics):
""" Create a DataFrame evaluating various models on metrics specified in an evaluation config.
"""
evaluation_dict = {}
for model_name, model, arguments in eval_config.models:
print(f'Handling model {model_name}')
algo = model(**arguments)
evaluation_dict[model_name] = {}
# Type 1 : split evaluations
if len(eval_config.split_metrics) > 0:
print('Training split predictions')
predictions = generate_split_predictions(algo, sp_ratings, eval_config)
for metric in eval_config.split_metrics:
print(f'- computing metric {metric}')
assert metric in available_metrics['split']
evaluation_function, parameters = available_metrics["split"][metric]
evaluation_dict[model_name][metric] = evaluation_function(predictions, **parameters)
# Type 2 : loo evaluations
if len(eval_config.loo_metrics) > 0:
print('Training loo predictions')
anti_testset_top_n, testset = generate_loo_top_n(algo, sp_ratings, eval_config)
for metric in eval_config.loo_metrics:
assert metric in available_metrics['loo']
evaluation_function, parameters = available_metrics["loo"][metric]
evaluation_dict[model_name][metric] = evaluation_function(anti_testset_top_n, testset, **parameters)
# Type 3 : full evaluations
if len(eval_config.full_metrics) > 0:
print('Training full predictions')
anti_testset_top_n = generate_full_top_n(algo, sp_ratings, eval_config)
for metric in eval_config.full_metrics:
assert metric in available_metrics['full']
evaluation_function, parameters = available_metrics["full"][metric]
evaluation_dict[model_name][metric] = evaluation_function(
anti_testset_top_n,
**precomputed_dict,
**parameters
)
return pd.DataFrame.from_dict(evaluation_dict).T
```
%% Cell type:markdown id:f7e83d1d tags:
# 2. Evaluation metrics
Implement evaluation metrics for either rating predictions (split metrics) or for top-n recommendations (loo metric, full metric)
%% Cell type:code id:f1849e55 tags:
``` python
def get_hit_rate(anti_testset_top_n, testset):
"""Compute the average hit over the users (loo metric)
A hit (1) happens when the movie in the testset has been picked by the top-n recommender
A fail (0) happens when the movie in the testset has not been picked by the top-n recommender
"""
# -- implement the function get_hit_rate --
hits = 0
total_users = len(testset)
for uid, true_iid, _ in testset:
if uid in anti_testset_top_n and true_iid in {iid for iid, _ in anti_testset_top_n[uid]}:
hits += 1
hit_rate = hits / total_users
return hit_rate
def get_novelty(anti_testset_top_n, item_to_rank):
"""Compute the average novelty of the top-n recommendation over the users (full metric)
The novelty is defined as the average ranking of the movies recommended
"""
# -- implement the function get_novelty --
total_rank_sum = 0
total_recommendations = 0
for uid, recommendations in anti_testset_top_n.items():
for iid, _ in recommendations:
if iid in item_to_rank:
total_rank_sum += item_to_rank[iid]
total_recommendations += 1
if total_recommendations == 0:
return 0 # Avoid division by zero
average_rank_sum = total_rank_sum / total_recommendations
return average_rank_sum
```
%% Cell type:markdown id:1a9855b3 tags:
# 3. Evaluation workflow
Load data, evaluate models and save the experimental outcomes
%% Cell type:code id:704f4d2a tags:
``` python
AVAILABLE_METRICS = {
"split": {
"mae": (accuracy.mae, {'verbose': False}),
"rmse": (accuracy.rmse, {'verbose': False})
# Add new split metrics here if needed
},
"loo": {
"hit_rate": (get_hit_rate, {}),
# Add new loo metrics here if needed
},
"full": {
"novelty": (get_novelty, {}),
# Add new full metrics here if needed
}
}
sp_ratings = load_ratings(surprise_format=True)
precomputed_dict = precomputed_information(pd.read_csv("data/tiny/evidence/ratings.csv"))
evaluation_report = create_evaluation_report(EvalConfig, sp_ratings, precomputed_dict, AVAILABLE_METRICS)
export_evaluation_report(evaluation_report)
```
%% Output
Handling model baseline_1
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_2
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_3
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model baseline_4
Training split predictions
- computing metric mae
- computing metric rmse
Training loo predictions
Training full predictions
Handling model ContentBased
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[12], line 19
NameError Traceback (most recent call last)
Cell In[6], line 3
1 AVAILABLE_METRICS = {
2 "split": {
----> 3 "mae": (accuracy.mae, {'verbose': False}),
4 "rmse": (accuracy.rmse, {'verbose': False})
5 # Add new split metrics here if needed
6 },
7 "loo": {
8 "hit_rate": (get_hit_rate, {}),
9 # Add new loo metrics here if needed
10 },
11 "full": {
12 "novelty": (get_novelty, {}),
13 # Add new full metrics here if needed
14 }
15 }
17 sp_ratings = load_ratings(surprise_format=True)
18 precomputed_dict = precomputed_information(pd.read_csv("data/tiny/evidence/ratings.csv"))
---> 19 evaluation_report = create_evaluation_report(EvalConfig, sp_ratings, precomputed_dict, AVAILABLE_METRICS)
20 export_evaluation_report(evaluation_report)
Cell In[10], line 81, in create_evaluation_report(eval_config, sp_ratings, precomputed_dict, available_metrics)
79 for model_name, model, arguments in eval_config.models:
80 print(f'Handling model {model_name}')
---> 81 algo = model(**arguments)
82 evaluation_dict[model_name] = {}
84 # Type 1 : split evaluations
TypeError: ContentBased.__init__() missing 2 required positional arguments: 'features_method' and 'regressor_method'
NameError: name 'accuracy' is not defined
......
0% ou .
You are about to add 0 people to the discussion. Proceed with caution.
Terminez d'abord l'édition de ce message.
Veuillez vous inscrire ou vous pour commenter