Observe differences in the behavior of recommendation models using RecBole

pip install recbole
python run_recbole.py --model=<your favorite model> --dataset_name ml-100k

from recbole.quick_start import run_recbole
run_recbole(model={model_name}, dataset="movielens-1m")

# general
gpu_id: 0
use_gpu: True
seed: 2020
state: INFO
reproducibility: True
data_path: 'dataset/'
checkpoint_dir: 'saved/movielens-1m'
show_progress: True
save_dataset: False
save_dataloaders: False

# Atomic File Format
field_separator: "\t"
seq_separator: "@"

# Common Features
USER_ID_FIELD: user_id
ITEM_ID_FIELD: item_id
RATING_FIELD: rating
TIME_FIELD: timestamp
seq_len: ~
# Label for Point-wise DataLoader
LABEL_FIELD: label
# NegSample Prefix for Pair-wise DataLoader
NEG_PREFIX: neg_
# Sequential Model Needed
ITEM_LIST_LENGTH_FIELD: item_length
LIST_SUFFIX: _list
MAX_ITEM_LIST_LENGTH: 50
POSITION_FIELD: position_id
# Knowledge-based Model Needed
HEAD_ENTITY_ID_FIELD: head_id
TAIL_ENTITY_ID_FIELD: tail_id
RELATION_ID_FIELD: relation_id
ENTITY_ID_FIELD: entity_id

# Selectively Loading
load_col:
    inter: [user_id, item_id, timestamp, rating]
    user: [user_id, age, gender, occupation, zip_code]
    item: [item_id, movie_title, release_year, genre]
unused_col:
    inter: [timestamp, rating]

# Filtering
rm_dup_inter: ~
val_interval: ~
filter_inter_by_user_or_item: True
user_inter_num_interval: "[1,inf]"
item_inter_num_interval: "[1,inf]"

# Preprocessing
alias_of_user_id: ~
alias_of_item_id: ~
alias_of_entity_id: ~
alias_of_relation_id: ~
preload_weight: ~
normalize_field: ~
normalize_all: True

# Training and evaluation config
epochs: 50
stopping_step: 10
train_batch_size: 4096
eval_batch_size: 4096
neg_sampling:
    uniform: 1
eval_args:
    group_by: user
    order: TO
    split: {'RS': [0.8,0.1,0.1]}
    mode: full
metrics: ['Recall', 'MRR', 'NDCG', 'Hit', 'Precision']
topk: 10
valid_metric: MRR@10
metric_decimal_place: 4

run_recbole(model={model_name}, dataset="movielens-1m", config_file_list=["config/movielens-1m.yml"])

The number of users: 1084
Average actions of users: 84.04801477377654
The number of items: 38334
Average actions of items: 2.374559778780685
The number of inters: 91024
The sparsity of the dataset: 99.78095038424168%

The number of users: 1084
Average actions of users: 84.04801477377654
The number of items: 38334
Average actions of items: 2.374559778780685
The number of inters: 91024
The sparsity of the dataset: 99.78095038424168%

# Selectively Loading
load_col:
    inter: [user_id, venue_id, timestamp]
unused_col:
    inter: [timestamp]

import json
import click
import numpy as np
import torch
from recbole.config import Config
from recbole.data import create_dataset, data_preparation
from recbole.quick_start.quick_start import load_data_and_model
from recbole.utils import get_model
from recbole.utils.case_study import full_sort_topk
import pandas as pd
from src.custom_models.Item2Vec import Item2Vec
from src.metrics import calculate_indicators
from tqdm.auto import tqdm

@click.command()
@click.option(
    "--model_file",
    required=True,
    type=str,
    help="example. saved/ckpd_recipe/Item2Vec-Nov-06-2021_02-41-35.pth",
)
@click.option(
    "--output_file",
    required=True,
    type=str,
    help="example. pop.json",
)
@click.option(
    "--is_item2vec",
    type=bool,
    is_flag=True
)
def main(model_file, output_file, is_item2vec):
    print("=====")
    print(model_file)
    print("=====")

    # for get Item title
    # e.g. foursquare
    _df = pd.read_csv("dataset/foursquare-nyc-merged/foursquare-nyc-merged.item", sep="\t")
    internal_id_to_title = _df["venue_category_name:token"].to_dict()

    # custom model(Item2Vec)
    if is_item2vec:
        checkpoint = torch.load(model_file)
        config = checkpoint["config"]

        config.seq_separator = "@"
        dataset = create_dataset(config)
        train_data, valid_data, test_data = data_preparation(config, dataset)

        model = Item2Vec(config, train_data.dataset).to(config["device"])
        model.load_state_dict(checkpoint["state_dict"])
        model.load_other_parameter(checkpoint.get("other_parameter"))

    # when not custom model
    else:
        config, model, dataset, train_data, valid_data, test_data = load_data_and_model(
            model_file=model_file
        )

    ground_list = []
    uid_list = []
    for batch_idx, batched_data in enumerate(test_data):
        interaction, row_idx, positive_u, positive_i = batched_data
        ground_list.append([int(v) for v in positive_i.numpy().tolist()])
        uid_list.append(interaction.user_id.numpy()[0])

    ranked_list = []
    for uid in tqdm(uid_list):
        topk_score, topk_iid_list = full_sort_topk(
            [uid], model, test_data, k=10, device="cuda"
        )
        ranked_list += topk_iid_list.cpu()

    all_metrics_results = {}
    for uid, g_list, r_list in zip(uid_list, ground_list, ranked_list):
        external_uid = dataset.id2token(dataset.uid_field, uid)
        all_metrics_results[external_uid] = {
            "ground_list_id": [v for v in dataset.id2token(dataset.iid_field, g_list)],
            "predict_list_id": [v for v in dataset.id2token(dataset.iid_field, r_list)],
            "ground_list": [internal_id_to_title[v-1] for v in g_list],
            "predict_list": [internal_id_to_title[v-1] for v in r_list.numpy()],
        }

    text = json.dumps(all_metrics_results, sort_keys=True, ensure_ascii=False, indent=2)
    with open(output_file, "w") as fh:
        fh.write(text)


if __name__ == "__main__":
    main()

import glob
import itertools

import numpy as np
import pandas as pd
from tqdm.auto import tqdm


def main():
    filelist = glob.glob("output/foursquare_nyc_case_study/*.json")
    model_results = {}
    for file in tqdm(filelist):
        _model = file.split("/")[-1].split(".")[0]
        try:
            _df = pd.read_json(file).T
            model_results[_model] = _df
        except:
            print(f"{_model} read is failed")
    _models = model_results.keys()
    combis = list(itertools.combinations(_models, 2))
    model_similarities = []

    for c in tqdm(combis):
        model1 = c[0]
        model2 = c[1]
        model1_result = model_results[model1]
        model2_result = model_results[model2]

        model1_predict_list = model1_result["predict_list_id"].values
        model2_predict_list = model2_result["predict_list_id"].values
        sims = []
        for m1_preds, m2_preds in zip(model1_predict_list, model2_predict_list):
            _sim = len(set(m1_preds) & set(m2_preds)) / len(m1_preds)
            sims.append(_sim)
        similarity = np.mean(sims)
        model_similarities.append([model1, model2, similarity])
        result = pd.DataFrame(
            model_similarities, columns=["source_model", "dest_model", "similarity"]
        )
        result.to_csv("foursquare_nyc_survey_with_recbole.csv", index=False)


if __name__ == "__main__":
    main()