From 3599be05d0b053fc476fa8b68b4b53f5813b737a Mon Sep 17 00:00:00 2001 From: be-marc Date: Tue, 10 Dec 2024 14:04:57 +0100 Subject: [PATCH] ... --- R/CallbackEvaluation.R | 44 ++++++++++++++++++++++++++++++-------- man/CallbackEvaluation.Rd | 7 ++++-- man/callback_evaluation.Rd | 29 +++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/R/CallbackEvaluation.R b/R/CallbackEvaluation.R index 732fa07f5..9f064dd6f 100644 --- a/R/CallbackEvaluation.R +++ b/R/CallbackEvaluation.R @@ -1,7 +1,10 @@ -#' @title Create Evaluation Callback +#' @title Evaluation Callback #' #' @description -#' Callbacks allow to customize the behavior of `resample()` and `benchmark()` in mlr3. +#' Specialized [mlr3misc::Callback] to customize the behavior of [resample()] and [benchmark()] in mlr3. +#' The [callback_evaluation()] function is used to create instances of this class. +#' Predefined callbacks are stored in the [dictionary][mlr3misc::Dictionary] [mlr_callbacks] and can be retrieved with [clbk()]. +#' For more information on callbacks, see the [callback_evaluation()] documentation. #' #' @export CallbackEvaluation= R6Class("CallbackEvaluation", @@ -30,13 +33,14 @@ CallbackEvaluation= R6Class("CallbackEvaluation", ) ) -#' @title Create Workhorse Callback +#' @title Create Evaluation Callback #' #' @description #' Function to create a [CallbackEvaluation]. #' Predefined callbacks are stored in the [dictionary][mlr3misc::Dictionary] [mlr_callbacks] and can be retrieved with [clbk()]. #' #' Evaluation callbacks are called at different stages of the resampling process. +#' Each stage is called once per resampling iteration. #' The stages are prefixed with `on_*`. #' #' ``` @@ -49,21 +53,22 @@ CallbackEvaluation= R6Class("CallbackEvaluation", #' ``` #' #' See also the section on parameters for more information on the stages. -#' A evaluation callback works with [ContextEvaluation]. +#' An evaluation callback works with [ContextEvaluation]. # #' @details #' When implementing a callback, each function must have two arguments named `callback` and `context`. #' A callback can write data to the state (`$state`), e.g. settings that affect the callback itself. #' Evaluation callbacks access [ContextEvaluation]. +#' Data can be stored in the [ResampleResult] and [BenchmarkResult] objects via `context$data_extra`. +#' Alternatively results can be stored in the learner state via `context$learner$state`. #' #' @param id (`character(1)`)\cr -#' Identifier for the new instance. +#' Identifier for the new instance. #' @param label (`character(1)`)\cr -#' Label for the new instance. +#' Label for the new instance. #' @param man (`character(1)`)\cr -#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. -#' The referenced help package can be opened via method `$help()`. -#' +#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. +#' The referenced help package can be opened via method `$help()`. #' @param on_evaluation_begin (`function()`)\cr #' Stage called at the beginning of an evaluation. #' Called in `workhorse()` (internal). @@ -78,6 +83,27 @@ CallbackEvaluation= R6Class("CallbackEvaluation", #' Called in `workhorse()` (internal). #' #' @export +#' @examples +#' callback = callback_evaluation("selected_features", +#' label = "Selected Features", +#' +#' on_evaluation_end = function(callback, context) { +#' pred = as_prediction(context$pdatas$test) +#' selected_features = pred$score( +#' measure = msr("selected_features"), +#' learner = context$learner, +#' task = context$task) +#' context$learner$state$selected_features = selected_features +#' } +#' ) +#' +#' task = tsk("pima") +#' learner = lrn("classif.rpart") +#' resampling = rsmp("cv", folds = 3) +#' +#' rr = resample(task, learner, resampling, callbacks = callback) +#' +#' rr$learners[[1]]$state$selected_features callback_evaluation = function( id, label = NA_character_, diff --git a/man/CallbackEvaluation.Rd b/man/CallbackEvaluation.Rd index 67dfd7124..bd075c53c 100644 --- a/man/CallbackEvaluation.Rd +++ b/man/CallbackEvaluation.Rd @@ -2,9 +2,12 @@ % Please edit documentation in R/CallbackEvaluation.R \name{CallbackEvaluation} \alias{CallbackEvaluation} -\title{Create Evaluation Callback} +\title{Evaluation Callback} \description{ -Callbacks allow to customize the behavior of \code{resample()} and \code{benchmark()} in mlr3. +Specialized \link[mlr3misc:Callback]{mlr3misc::Callback} to customize the behavior of \code{\link[=resample]{resample()}} and \code{\link[=benchmark]{benchmark()}} in mlr3. +The \code{\link[=callback_evaluation]{callback_evaluation()}} function is used to create instances of this class. +Predefined callbacks are stored in the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_callbacks} and can be retrieved with \code{\link[=clbk]{clbk()}}. +For more information on callbacks, see the \code{\link[=callback_evaluation]{callback_evaluation()}} documentation. } \section{Super class}{ \code{\link[mlr3misc:Callback]{mlr3misc::Callback}} -> \code{CallbackEvaluation} diff --git a/man/callback_evaluation.Rd b/man/callback_evaluation.Rd index 4a74900bc..bf76a2944 100644 --- a/man/callback_evaluation.Rd +++ b/man/callback_evaluation.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/CallbackEvaluation.R \name{callback_evaluation} \alias{callback_evaluation} -\title{Create Workhorse Callback} +\title{Create Evaluation Callback} \usage{ callback_evaluation( id, @@ -46,6 +46,7 @@ Function to create a \link{CallbackEvaluation}. Predefined callbacks are stored in the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_callbacks} and can be retrieved with \code{\link[=clbk]{clbk()}}. Evaluation callbacks are called at different stages of the resampling process. +Each stage is called once per resampling iteration. The stages are prefixed with \verb{on_*}. \if{html}{\out{
}}\preformatted{Start Resampling Iteration on Worker @@ -57,10 +58,34 @@ End Resampling Iteration on Worker }\if{html}{\out{
}} See also the section on parameters for more information on the stages. -A evaluation callback works with \link{ContextEvaluation}. +An evaluation callback works with \link{ContextEvaluation}. } \details{ When implementing a callback, each function must have two arguments named \code{callback} and \code{context}. A callback can write data to the state (\verb{$state}), e.g. settings that affect the callback itself. Evaluation callbacks access \link{ContextEvaluation}. +Data can be stored in the \link{ResampleResult} and \link{BenchmarkResult} objects via \code{context$data_extra}. +Alternatively results can be stored in the learner state via \code{context$learner$state}. +} +\examples{ +callback = callback_evaluation("selected_features", + label = "Selected Features", + + on_evaluation_end = function(callback, context) { + pred = as_prediction(context$pdatas$test) + selected_features = pred$score( + measure = msr("selected_features"), + learner = context$learner, + task = context$task) + context$learner$state$selected_features = selected_features + } +) + +task = tsk("pima") +learner = lrn("classif.rpart") +resampling = rsmp("cv", folds = 3) + +rr = resample(task, learner, resampling, callbacks = callback) + +rr$learners[[1]]$state$selected_features }