%
% Generic hyperparameter tuner
%
% Searches hyperparameters by cross-validation of leave-one-out grid search.
%
% This code is part of the supplementary material to the CVPR 2013 paper
% "From N to N+1: Multiclass Transfer Incremental Learning",
% Ilja Kuzborskij, Francesco Orabona and Barbara Caputo
%
% Copyright (C) 2012-2013, Ilja Kuzborskij
%
% This program is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program.  If not, see <http://www.gnu.org/licenses/>.
%
% Contact the author: ilja.kuzborskij [at] gmail.com
%

classdef HyperSearch < handle
    properties
        classifier = [];
        ranges = [];
        cv_folds = 5;
        loo2cv_num_samples = 20;
        samples_x;
        param_combs = [];
    end

    methods
        function obj = HyperSearch(classifier, ranges, loo2cv_num_samples, cv_folds)
        % Constructs HyperSearch
        %
        % classifier                          -- instance of classifier, should have methods train and evaluate
        % ranges                              -- structure with field names as hyperparameter names and array values.
        %                                        search occurs at each element of cartesian product of these arrays.
        % (optional) loo2cv_num_samples       -- minimal number of sample for cross-validation.
        %                                        if number of samples is smaller, leave-one-out is used.
        %                                        default = 20;
        % (optional) cv_folds                 -- number of cross-validation folds
        %
        % Example:
        %
        % hyper_range.c_val = 10.^(-5:5);
        % hyper_range.kernel_type = {'rbf'};  % should be in brackets, otherwise is treated as character array
        % hyperr.gamma = 10.^(-5:5);
        %
        % tuner = HyperSearch(MulticlassRLS, hyperr);
        % hyper = tuner.search(train_x, train_y); % when complete, hyper contains found hyperparameters
        %

            obj.classifier = classifier;
            obj.ranges = ranges;

            if nargin >= 3
                obj.loo2cv_num_samples = loo2cv_num_samples;
            end

            if nargin >= 4
                obj.cv_folds = cv_folds;
            end
        end

        function testval = eval(obj, train_x_i, train_y, test_x_i, test_y)
            data_obj.train_x = cellfun(@(x) x(train_x_i, :), obj.samples_x, 'UniformOutput', 0);
            data_obj.train_y = train_y;
            data_obj.grid_search_x = cellfun(@(x) x(train_x_i, :), obj.samples_x, 'UniformOutput', 0);
            data_obj.grid_search_y = train_y;
            data_obj.test_x = cellfun(@(x) x(test_x_i, :), obj.samples_x, 'UniformOutput', 0);
            data_obj.test_y = test_y;

            testval = [];

            if ~isfield(obj.ranges, 'param_order')
                ranges = fields(obj.ranges);
            else
                ranges = obj.ranges.param_order';
            end

            prev_values = struct;

            for comb=obj.param_combs
                hyper = struct;
                for i=1:numel(ranges)
                    val = comb(i);
                    hyper.(char(ranges{i})) = val{1};
                end

                try
                    obj.classifier.train_model(hyper, data_obj);
                    rs = obj.classifier.evaluate_model(data_obj);

                    rs.hyper = hyper;
                    testval = [testval rs];
                    fprintf('.');

                    for f=fields(prev_values)'
                        f=f{1};
                        if ~isequal(prev_values.(f), hyper.(f)) && ...
                            (isfield(obj.ranges, 'dont_reset_on') && ~isequal(obj.ranges.dont_reset_on, f))
                            fprintf('\nResetting learner since "%s" has changed.\n', f);
                            obj.classifier.clear;
                        end
                    end
                    prev_values = hyper;
                catch err
                    if ~isempty(strfind(err.identifier,'terminated'))
                        if exist('rs', 'var')
                            dummy = rs;
                            dummy.test_accuracy = 0;
                        else
                            dummy = struct('test_accuracy', 0, 'n_accuracy', 0, 'npp_accuracy', 0, ...
                                           'conf_mat', 0, 'roc', 0, 'new_TP', 0, 'new_FP', 0, ...
                                           'new_FN', 0, 'new_TN', 0);
                        end
                        testval = [testval dummy];
                        continue;
                    else
                        rethrow(err);
                    end
                end
            end
            disp(' done.');
            obj.classifier.clear;
        end

        function [max_hyper, max_cvval] = search(obj, samples_x, samples_y)
            if ~isfield(obj.ranges, 'param_order')
                ranges = fields(obj.ranges);
            else
                ranges = obj.ranges.param_order';
            end

            catranges = {};
            for f=ranges'
                val = obj.ranges.(char(f));
                if ~iscell(val)
                    val = arrayfun(@(x) {x}, val);
                end
                catranges{end+1} = val;
            end

            max_cvval = -Inf;
            max_hyper = [];

            obj.samples_x = samples_x;

            param_combs = allcombs(catranges{:})';
            if size(param_combs, 2) == 1
                max_hyper = struct;
                for i=1:numel(ranges)
                    max_hyper.(char(ranges{i})) = param_combs{i};
                end
                max_cvval = -1;
                return;
            end

            obj.param_combs = param_combs;

            % Relies on MATLAB's cvpartition and crossval
            if numel(samples_y) <= obj.loo2cv_num_samples
                part = cvpartition(samples_y, 'leaveout');
            else
                part = cvpartition(samples_y, 'kfold', obj.cv_folds);
                if min(part.TrainSize) ~= max(part.TrainSize)
                    part = cvpartition(samples_y, 'leaveout');
                    warning('Unable to partition with CV, falling back to LOO.');
                end
            end

            fn = @(a,b,c,d) obj.eval(a,b,c,d);
            cvval = crossval(fn, (1:numel(samples_y))', samples_y, 'partition', part, 'options', statset('UseParallel', 'always'));
            [junk, max_i] = max(arrayfun(@(i) mean([cvval(:,i).test_accuracy]), 1:size(cvval, 2)));
            max_hyper = cvval(1, max_i).hyper;
        end
    end
end
