optimization.py

"""Optimization of progress measure parameter values for a given scenario."""

import inspect
import optuna as opt
import progress_measures as pm
import evaluation_metrics as em
import config as cf
import models


def evaluate_single_measure(scenario, sequences, measure):
    eval_specs = cf.get_eval_specs(scenario)
    durations = {skill: [measure.steps_to_mastery(seq)
                         for seq in sequences[skill]]
                 for skill in cf.SKILLS}
    metric_values = [metric_fun(durations, eval_specs)
                     for metric_fun in em.METRICS]
    return em.mellow_max(metric_values)


def optimize(scenario, pm_class, n_trials=300, rep=600):
    def objective(trial):
        for name, type_str, low, high in pm_class.optimization_params:
            if type_str == "int":
                trial.suggest_int(name, low, high)
            elif type_str == "float":
                trial.suggest_float(name, low, high)
        if "guess_chance" in inspect.signature(pm_class.__init__).parameters:
            measure = pm_class(guess_chance=scenario.guess_chance, **trial.params)
        else:
            measure = pm_class( **trial.params)
        return evaluate_single_measure(scenario, sequences, measure)

    opt.logging.disable_default_handler()
    limit = cf.get_attempts_limit(scenario)
    sequences = {skill: models.get_answer_sequences(scenario, skill, rep, limit)
                 for skill in cf.SKILLS}
    study = opt.create_study()
    study.optimize(objective, n_trials=n_trials)
    return study.best_params, study.best_value


def optimize_all(scenario):
    for pm_class in [
        pm.ProgressMeasureTC,
        pm.ProgressMeasureNCC,
        pm.ProgressMeasureTOW,
        pm.ProgressMeasureEMA,
        pm.ProgressMeasureVSLinear,
        pm.ProgressMeasureVSHMM
    ]:
        print("Processing", pm_class.__name__)
        params, value = optimize(scenario, pm_class)
        print("\t", "best value of optimization metric:", round(value, 3))
        for param in params:
            print("\t", param, "=", round(params[param], 3))