From 0440ef3713c33b8961e61010c7d2d1b46f60d37a Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:20:13 +0100 Subject: [PATCH 1/8] chore(#398): remove dead svm/search module --- src/svm/search/mod.rs | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 src/svm/search/mod.rs diff --git a/src/svm/search/mod.rs b/src/svm/search/mod.rs deleted file mode 100644 index d61b8326..00000000 --- a/src/svm/search/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! SVC and Grid Search - -/// SVC search parameters -pub mod svc_params; -/// SVC search parameters -pub mod svr_params; From d96a406daca73a406c1510adf6b3375dd7d86c03 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:20:20 +0100 Subject: [PATCH 2/8] chore(#398): remove dead svm/search/svc_params --- src/svm/search/svc_params.rs | 183 ----------------------------------- 1 file changed, 183 deletions(-) delete mode 100644 src/svm/search/svc_params.rs diff --git a/src/svm/search/svc_params.rs b/src/svm/search/svc_params.rs deleted file mode 100644 index 42f686b3..00000000 --- a/src/svm/search/svc_params.rs +++ /dev/null @@ -1,183 +0,0 @@ -// /// SVC grid search parameters -// #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -// #[derive(Debug, Clone)] -// pub struct SVCSearchParameters< -// TX: Number + RealNumber, -// TY: Number + Ord, -// X: Array2, -// Y: Array1, -// K: Kernel, -// > { -// #[cfg_attr(feature = "serde", serde(default))] -// /// Number of epochs. -// pub epoch: Vec, -// #[cfg_attr(feature = "serde", serde(default))] -// /// Regularization parameter. -// pub c: Vec, -// #[cfg_attr(feature = "serde", serde(default))] -// /// Tolerance for stopping epoch. -// pub tol: Vec, -// #[cfg_attr(feature = "serde", serde(default))] -// /// The kernel function. -// pub kernel: Vec, -// #[cfg_attr(feature = "serde", serde(default))] -// /// Unused parameter. -// m: PhantomData<(X, Y, TY)>, -// #[cfg_attr(feature = "serde", serde(default))] -// /// Controls the pseudo random number generation for shuffling the data for probability estimates -// seed: Vec>, -// } - -// /// SVC grid search iterator -// pub struct SVCSearchParametersIterator< -// TX: Number + RealNumber, -// TY: Number + Ord, -// X: Array2, -// Y: Array1, -// K: Kernel, -// > { -// svc_search_parameters: SVCSearchParameters, -// current_epoch: usize, -// current_c: usize, -// current_tol: usize, -// current_kernel: usize, -// current_seed: usize, -// } - -// impl, Y: Array1, K: Kernel> -// IntoIterator for SVCSearchParameters -// { -// type Item = SVCParameters<'a, TX, TY, X, Y>; -// type IntoIter = SVCSearchParametersIterator; - -// fn into_iter(self) -> Self::IntoIter { -// SVCSearchParametersIterator { -// svc_search_parameters: self, -// current_epoch: 0, -// current_c: 0, -// current_tol: 0, -// current_kernel: 0, -// current_seed: 0, -// } -// } -// } - -// impl, Y: Array1, K: Kernel> -// Iterator for SVCSearchParametersIterator -// { -// type Item = SVCParameters; - -// fn next(&mut self) -> Option { -// if self.current_epoch == self.svc_search_parameters.epoch.len() -// && self.current_c == self.svc_search_parameters.c.len() -// && self.current_tol == self.svc_search_parameters.tol.len() -// && self.current_kernel == self.svc_search_parameters.kernel.len() -// && self.current_seed == self.svc_search_parameters.seed.len() -// { -// return None; -// } - -// let next = SVCParameters { -// epoch: self.svc_search_parameters.epoch[self.current_epoch], -// c: self.svc_search_parameters.c[self.current_c], -// tol: self.svc_search_parameters.tol[self.current_tol], -// kernel: self.svc_search_parameters.kernel[self.current_kernel].clone(), -// m: PhantomData, -// seed: self.svc_search_parameters.seed[self.current_seed], -// }; - -// if self.current_epoch + 1 < self.svc_search_parameters.epoch.len() { -// self.current_epoch += 1; -// } else if self.current_c + 1 < self.svc_search_parameters.c.len() { -// self.current_epoch = 0; -// self.current_c += 1; -// } else if self.current_tol + 1 < self.svc_search_parameters.tol.len() { -// self.current_epoch = 0; -// self.current_c = 0; -// self.current_tol += 1; -// } else if self.current_kernel + 1 < self.svc_search_parameters.kernel.len() { -// self.current_epoch = 0; -// self.current_c = 0; -// self.current_tol = 0; -// self.current_kernel += 1; -// } else if self.current_seed + 1 < self.svc_search_parameters.seed.len() { -// self.current_epoch = 0; -// self.current_c = 0; -// self.current_tol = 0; -// self.current_kernel = 0; -// self.current_seed += 1; -// } else { -// self.current_epoch += 1; -// self.current_c += 1; -// self.current_tol += 1; -// self.current_kernel += 1; -// self.current_seed += 1; -// } - -// Some(next) -// } -// } - -// impl, Y: Array1, K: Kernel> Default -// for SVCSearchParameters -// { -// fn default() -> Self { -// let default_params: SVCParameters = SVCParameters::default(); - -// SVCSearchParameters { -// epoch: vec![default_params.epoch], -// c: vec![default_params.c], -// tol: vec![default_params.tol], -// kernel: vec![default_params.kernel], -// m: PhantomData, -// seed: vec![default_params.seed], -// } -// } -// } - -// #[cfg(test)] -// mod tests { -// use num::ToPrimitive; - -// use super::*; -// use crate::linalg::basic::matrix::DenseMatrix; -// use crate::metrics::accuracy; -// #[cfg(feature = "serde")] -// use crate::svm::*; - -// #[test] -// fn search_parameters() { -// let parameters: SVCSearchParameters, LinearKernel> = -// SVCSearchParameters { -// epoch: vec![10, 100], -// kernel: vec![LinearKernel {}], -// ..Default::default() -// }; -// let mut iter = parameters.into_iter(); -// let next = iter.next().unwrap(); -// assert_eq!(next.epoch, 10); -// assert_eq!(next.kernel, LinearKernel {}); -// let next = iter.next().unwrap(); -// assert_eq!(next.epoch, 100); -// assert_eq!(next.kernel, LinearKernel {}); -// assert!(iter.next().is_none()); -// } - -// #[test] -// fn search_parameters() { -// let parameters: SVCSearchParameters, LinearKernel> = -// SVCSearchParameters { -// epoch: vec![10, 100], -// kernel: vec![LinearKernel {}], -// ..Default::default() -// }; -// let mut iter = parameters.into_iter(); -// let next = iter.next().unwrap(); -// assert_eq!(next.epoch, 10); -// assert_eq!(next.kernel, LinearKernel {}); -// let next = iter.next().unwrap(); -// assert_eq!(next.epoch, 100); -// assert_eq!(next.kernel, LinearKernel {}); -// assert!(iter.next().is_none()); -// } -// } From f9b7b06606974fda144508567ee6a67b547a2b34 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:20:27 +0100 Subject: [PATCH 3/8] chore(#398): remove dead svm/search/svr_params --- src/svm/search/svr_params.rs | 293 ----------------------------------- 1 file changed, 293 deletions(-) delete mode 100644 src/svm/search/svr_params.rs diff --git a/src/svm/search/svr_params.rs b/src/svm/search/svr_params.rs deleted file mode 100644 index f455c119..00000000 --- a/src/svm/search/svr_params.rs +++ /dev/null @@ -1,293 +0,0 @@ -//! # SVR Grid Search Parameters -//! -//! This module provides utilities for defining and iterating over grid search parameter spaces -//! for Support Vector Regression (SVR) models in [smartcore](https://github.com/smartcorelib/smartcore). -//! -//! The main struct, [`SVRSearchParameters`], allows users to specify multiple values for each -//! SVR hyperparameter (epsilon, regularization parameter C, tolerance, and kernel function). -//! The provided iterator yields all possible combinations (the Cartesian product) of these parameters, -//! enabling exhaustive grid search for hyperparameter tuning. -//! -//! -//! ## Example -//! ``` -//! use smartcore::svm::Kernels; -//! use smartcore::svm::search::svr_params::SVRSearchParameters; -//! use smartcore::linalg::basic::matrix::DenseMatrix; -//! -//! let params = SVRSearchParameters::> { -//! eps: vec![0.1, 0.2], -//! c: vec![1.0, 10.0], -//! tol: vec![1e-3], -//! kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], -//! m: std::marker::PhantomData, -//! }; -//! -//! // for param_set in params.into_iter() { -//! // Use param_set (of type svr::SVRParameters) to fit and evaluate your SVR model. -//! // } -//! ``` -//! -//! -//! ## Note -//! This module is intended for use with smartcore version 0.4 or later. The API is not compatible with older versions[1]. -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; - -use crate::linalg::basic::arrays::Array2; -use crate::numbers::basenum::Number; -use crate::numbers::floatnum::FloatNumber; -use crate::numbers::realnum::RealNumber; -use crate::svm::{Kernels, svr}; -use std::marker::PhantomData; - -/// ## SVR grid search parameters -/// A struct representing a grid of hyperparameters for SVR grid search in smartcore. -/// -/// Each field is a vector of possible values for the corresponding SVR hyperparameter. -/// The [`IntoIterator`] implementation yields every possible combination of these parameters -/// as an `svr::SVRParameters` struct, suitable for use in model selection routines. -/// -/// # Type Parameters -/// - `T`: Numeric type for parameters (e.g., `f64`) -/// - `M`: Matrix type implementing [`Array2`] -/// -/// # Fields -/// - `eps`: Vector of epsilon values for the epsilon-insensitive loss in SVR. -/// - `c`: Vector of regularization parameters (C) for SVR. -/// - `tol`: Vector of tolerance values for the stopping criterion. -/// - `kernel`: Vector of kernel function variants (see [`Kernels`]). -/// - `m`: Phantom data for the matrix type parameter. -/// -/// # Example -/// ``` -/// use smartcore::svm::Kernels; -/// use smartcore::svm::search::svr_params::SVRSearchParameters; -/// use smartcore::linalg::basic::matrix::DenseMatrix; -/// -/// let params = SVRSearchParameters::> { -/// eps: vec![0.1, 0.2], -/// c: vec![1.0, 10.0], -/// tol: vec![1e-3], -/// kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], -/// m: std::marker::PhantomData, -/// }; -/// ``` -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Debug, Clone)] -pub struct SVRSearchParameters> { - /// Epsilon in the epsilon-SVR model. - pub eps: Vec, - /// Regularization parameter. - pub c: Vec, - /// Tolerance for stopping eps. - pub tol: Vec, - /// The kernel function. - pub kernel: Vec, - /// Unused parameter. - pub m: PhantomData, -} - -/// SVR grid search iterator -pub struct SVRSearchParametersIterator> { - svr_search_parameters: SVRSearchParameters, - current_eps: usize, - current_c: usize, - current_tol: usize, - current_kernel: usize, -} - -impl> IntoIterator - for SVRSearchParameters -{ - type Item = svr::SVRParameters; - type IntoIter = SVRSearchParametersIterator; - - fn into_iter(self) -> Self::IntoIter { - SVRSearchParametersIterator { - svr_search_parameters: self, - current_eps: 0, - current_c: 0, - current_tol: 0, - current_kernel: 0, - } - } -} - -impl> Iterator - for SVRSearchParametersIterator -{ - type Item = svr::SVRParameters; - - fn next(&mut self) -> Option { - if self.current_eps == self.svr_search_parameters.eps.len() - && self.current_c == self.svr_search_parameters.c.len() - && self.current_tol == self.svr_search_parameters.tol.len() - && self.current_kernel == self.svr_search_parameters.kernel.len() - { - return None; - } - - let next = svr::SVRParameters:: { - eps: self.svr_search_parameters.eps[self.current_eps], - c: self.svr_search_parameters.c[self.current_c], - tol: self.svr_search_parameters.tol[self.current_tol], - kernel: Some(self.svr_search_parameters.kernel[self.current_kernel].clone()), - }; - - if self.current_eps + 1 < self.svr_search_parameters.eps.len() { - self.current_eps += 1; - } else if self.current_c + 1 < self.svr_search_parameters.c.len() { - self.current_eps = 0; - self.current_c += 1; - } else if self.current_tol + 1 < self.svr_search_parameters.tol.len() { - self.current_eps = 0; - self.current_c = 0; - self.current_tol += 1; - } else if self.current_kernel + 1 < self.svr_search_parameters.kernel.len() { - self.current_eps = 0; - self.current_c = 0; - self.current_tol = 0; - self.current_kernel += 1; - } else { - self.current_eps += 1; - self.current_c += 1; - self.current_tol += 1; - self.current_kernel += 1; - } - - Some(next) - } -} - -impl> Default for SVRSearchParameters { - fn default() -> Self { - let default_params: svr::SVRParameters = svr::SVRParameters::default(); - - SVRSearchParameters { - eps: vec![default_params.eps], - c: vec![default_params.c], - tol: vec![default_params.tol], - kernel: vec![default_params.kernel.unwrap_or_else(Kernels::linear)], - m: PhantomData, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::linalg::basic::matrix::DenseMatrix; - use crate::svm::Kernels; - - type T = f64; - type M = DenseMatrix; - - #[test] - fn test_default_parameters() { - let params = SVRSearchParameters::::default(); - assert_eq!(params.eps.len(), 1); - assert_eq!(params.c.len(), 1); - assert_eq!(params.tol.len(), 1); - assert_eq!(params.kernel.len(), 1); - // Check that the default kernel is linear - assert_eq!(params.kernel[0], Kernels::linear()); - } - - #[test] - fn test_single_grid_iteration() { - let params = SVRSearchParameters:: { - eps: vec![0.1], - c: vec![1.0], - tol: vec![1e-3], - kernel: vec![Kernels::rbf().with_gamma(0.5)], - m: PhantomData, - }; - let mut iter = params.into_iter(); - let param = iter.next().unwrap(); - assert_eq!(param.eps, 0.1); - assert_eq!(param.c, 1.0); - assert_eq!(param.tol, 1e-3); - assert_eq!(param.kernel, Some(Kernels::rbf().with_gamma(0.5))); - assert!(iter.next().is_none()); - } - - #[test] - fn test_cartesian_grid_iteration() { - let params = SVRSearchParameters:: { - eps: vec![0.1, 0.2], - c: vec![1.0, 2.0], - tol: vec![1e-3], - kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], - m: PhantomData, - }; - let expected_count = - params.eps.len() * params.c.len() * params.tol.len() * params.kernel.len(); - let results: Vec<_> = params.into_iter().collect(); - assert_eq!(results.len(), expected_count); - - // Check that all parameter combinations are present - let mut seen = vec![]; - for p in &results { - seen.push((p.eps, p.c, p.tol, p.kernel.clone().unwrap())); - } - for &eps in &[0.1, 0.2] { - for &c in &[1.0, 2.0] { - for &tol in &[1e-3] { - for kernel in &[Kernels::linear(), Kernels::rbf().with_gamma(0.5)] { - assert!(seen.contains(&(eps, c, tol, kernel.clone()))); - } - } - } - } - } - - #[test] - fn test_empty_grid() { - let params = SVRSearchParameters:: { - eps: vec![], - c: vec![], - tol: vec![], - kernel: vec![], - m: PhantomData, - }; - let mut iter = params.into_iter(); - assert!(iter.next().is_none()); - } - - #[test] - fn test_kernel_enum_variants() { - let lin = Kernels::linear(); - let rbf = Kernels::rbf().with_gamma(0.2); - let poly = Kernels::polynomial() - .with_degree(2.0) - .with_gamma(1.0) - .with_coef0(0.5); - let sig = Kernels::sigmoid().with_gamma(0.3).with_coef0(0.1); - - assert_eq!(lin, Kernels::Linear); - match rbf { - Kernels::RBF { gamma } => assert_eq!(gamma, Some(0.2)), - _ => panic!("Not RBF"), - } - match poly { - Kernels::Polynomial { - degree, - gamma, - coef0, - } => { - assert_eq!(degree, Some(2.0)); - assert_eq!(gamma, Some(1.0)); - assert_eq!(coef0, Some(0.5)); - } - _ => panic!("Not Polynomial"), - } - match sig { - Kernels::Sigmoid { gamma, coef0 } => { - assert_eq!(gamma, Some(0.3)); - assert_eq!(coef0, Some(0.1)); - } - _ => panic!("Not Sigmoid"), - } - } -} From 4b5a6990aace8d6a9cd11ec1962b8fadc6bef218 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:20:35 +0100 Subject: [PATCH 4/8] chore(#398): remove dead model_selection/hyper_tuning module --- src/model_selection/hyper_tuning/mod.rs | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 src/model_selection/hyper_tuning/mod.rs diff --git a/src/model_selection/hyper_tuning/mod.rs b/src/model_selection/hyper_tuning/mod.rs deleted file mode 100644 index dfe0d06b..00000000 --- a/src/model_selection/hyper_tuning/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod grid_search; -pub use grid_search::{GridSearchCV, GridSearchCVParameters}; From 513a1c0266ac2325398ff835e891ff1a95bb0767 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:20:45 +0100 Subject: [PATCH 5/8] chore(#398): remove dead model_selection/hyper_tuning/grid_search --- .../hyper_tuning/grid_search.rs | 239 ------------------ 1 file changed, 239 deletions(-) delete mode 100644 src/model_selection/hyper_tuning/grid_search.rs diff --git a/src/model_selection/hyper_tuning/grid_search.rs b/src/model_selection/hyper_tuning/grid_search.rs deleted file mode 100644 index 74242c60..00000000 --- a/src/model_selection/hyper_tuning/grid_search.rs +++ /dev/null @@ -1,239 +0,0 @@ -// TODO: missing documentation - -use crate::{ - api::{Predictor, SupervisedEstimator}, - error::{Failed, FailedError}, - linalg::basic::arrays::{Array1, Array2}, - numbers::basenum::Number, - numbers::realnum::RealNumber, -}; - -use crate::model_selection::{cross_validate, BaseKFold, CrossValidationResult}; - -/// Parameters for GridSearchCV -#[derive(Debug)] -pub struct GridSearchCVParameters< - T: Number, - M: Array2, - C: Clone, - I: Iterator, - E: Predictor, - F: Fn(&M, &M::RowVector, C) -> Result, - K: BaseKFold, - S: Fn(&M::RowVector, &M::RowVector) -> T, -> { - _phantom: std::marker::PhantomData<(T, M)>, - - parameters_search: I, - estimator: F, - score: S, - cv: K, -} - -impl< - T: RealNumber, - M: Array2, - C: Clone, - I: Iterator, - E: Predictor, - F: Fn(&M, &M::RowVector, C) -> Result, - K: BaseKFold, - S: Fn(&M::RowVector, &M::RowVector) -> T, - > GridSearchCVParameters -{ - /// Create new GridSearchCVParameters - pub fn new(parameters_search: I, estimator: F, score: S, cv: K) -> Self { - GridSearchCVParameters { - _phantom: std::marker::PhantomData, - parameters_search, - estimator, - score, - cv, - } - } -} -/// Exhaustive search over specified parameter values for an estimator. -#[derive(Debug)] -pub struct GridSearchCV, C: Clone, E: Predictor> { - _phantom: std::marker::PhantomData<(T, M)>, - predictor: E, - /// Cross validation results. - pub cross_validation_result: CrossValidationResult, - /// best parameter - pub best_parameter: C, -} - -impl, E: Predictor, C: Clone> - GridSearchCV -{ - /// Search for the best estimator by testing all possible combinations with cross-validation using given metric. - /// * `x` - features, matrix of size _NxM_ where _N_ is number of samples and _M_ is number of attributes. - /// * `y` - target values, should be of size _N_ - /// * `gs_parameters` - GridSearchCVParameters struct - pub fn fit< - I: Iterator, - K: BaseKFold, - F: Fn(&M, &M::RowVector, C) -> Result, - S: Fn(&M::RowVector, &M::RowVector) -> T, - >( - x: &M, - y: &M::RowVector, - gs_parameters: GridSearchCVParameters, - ) -> Result { - let mut best_result: Option> = None; - let mut best_parameters = None; - let parameters_search = gs_parameters.parameters_search; - let estimator = gs_parameters.estimator; - let cv = gs_parameters.cv; - let score = gs_parameters.score; - - for parameters in parameters_search { - let result = cross_validate(&estimator, x, y, ¶meters, &cv, &score)?; - if best_result.is_none() - || result.mean_test_score() > best_result.as_ref().unwrap().mean_test_score() - { - best_parameters = Some(parameters); - best_result = Some(result); - } - } - - if let (Some(best_parameter), Some(cross_validation_result)) = - (best_parameters, best_result) - { - let predictor = estimator(x, y, best_parameter.clone())?; - Ok(Self { - _phantom: gs_parameters._phantom, - predictor, - cross_validation_result, - best_parameter, - }) - } else { - Err(Failed::because( - FailedError::FindFailed, - "there were no parameter sets found", - )) - } - } - - /// Return grid search cross validation results - pub fn cv_results(&self) -> &CrossValidationResult { - &self.cross_validation_result - } - - /// Return best parameters found - pub fn best_parameters(&self) -> &C { - &self.best_parameter - } - - /// Call predict on the estimator with the best found parameters - pub fn predict(&self, x: &M) -> Result { - self.predictor.predict(x) - } -} - -impl< - T: RealNumber, - M: Array2, - C: Clone, - I: Iterator, - E: Predictor, - F: Fn(&M, &M::RowVector, C) -> Result, - K: BaseKFold, - S: Fn(&M::RowVector, &M::RowVector) -> T, - > SupervisedEstimator> - for GridSearchCV -{ - fn fit( - x: &M, - y: &M::RowVector, - parameters: GridSearchCVParameters, - ) -> Result { - GridSearchCV::fit(x, y, parameters) - } -} - -impl, C: Clone, E: Predictor> - Predictor for GridSearchCV -{ - fn predict(&self, x: &M) -> Result { - self.predict(x) - } -} - -#[cfg(test)] -mod tests { - - use crate::{ - linalg::naive::dense_matrix::DenseMatrix, - linear::logistic_regression::{LogisticRegression, LogisticRegressionSearchParameters}, - metrics::accuracy, - model_selection::{ - hyper_tuning::grid_search::{self, GridSearchCVParameters}, - KFold, - }, - }; - use grid_search::GridSearchCV; - - #[test] - fn test_grid_search() { - let x = DenseMatrix::from_2d_array(&[ - &[5.1, 3.5, 1.4, 0.2], - &[4.9, 3.0, 1.4, 0.2], - &[4.7, 3.2, 1.3, 0.2], - &[4.6, 3.1, 1.5, 0.2], - &[5.0, 3.6, 1.4, 0.2], - &[5.4, 3.9, 1.7, 0.4], - &[4.6, 3.4, 1.4, 0.3], - &[5.0, 3.4, 1.5, 0.2], - &[4.4, 2.9, 1.4, 0.2], - &[4.9, 3.1, 1.5, 0.1], - &[7.0, 3.2, 4.7, 1.4], - &[6.4, 3.2, 4.5, 1.5], - &[6.9, 3.1, 4.9, 1.5], - &[5.5, 2.3, 4.0, 1.3], - &[6.5, 2.8, 4.6, 1.5], - &[5.7, 2.8, 4.5, 1.3], - &[6.3, 3.3, 4.7, 1.6], - &[4.9, 2.4, 3.3, 1.0], - &[6.6, 2.9, 4.6, 1.3], - &[5.2, 2.7, 3.9, 1.4], - ]); - let y = vec![ - 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - ]; - - let cv = KFold { - n_splits: 5, - ..KFold::default() - }; - - let parameters = LogisticRegressionSearchParameters { - alpha: vec![0., 1.], - ..Default::default() - }; - - let grid_search = GridSearchCV::fit( - &x, - &y, - GridSearchCVParameters { - estimator: LogisticRegression::fit, - score: accuracy, - cv, - parameters_search: parameters.into_iter(), - _phantom: Default::default(), - }, - ) - .unwrap(); - let best_parameters = grid_search.best_parameters(); - - assert!([1.].contains(&best_parameters.alpha)); - - let cv_results = grid_search.cv_results(); - - assert_eq!(cv_results.mean_test_score(), 0.9); - - let x = DenseMatrix::from_2d_array(&[&[5., 3., 1., 0.]]); - let result = grid_search.predict(&x).unwrap(); - assert_eq!(result, vec![0.]); - } -} From 68e51cf88d540448b5ff09249937acfd43fade8d Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:23:39 +0100 Subject: [PATCH 6/8] chore(#398): remove dead mod declarations from svm and model_selection --- src/model_selection/mod.rs | 5 +---- src/svm/mod.rs | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/model_selection/mod.rs b/src/model_selection/mod.rs index fe74fce4..bdd48d76 100644 --- a/src/model_selection/mod.rs +++ b/src/model_selection/mod.rs @@ -2,7 +2,7 @@ //! //! In statistics and machine learning we usually split our data into two sets: one for training and the other one for testing. //! We fit our model to the training data, in order to make predictions on the test data. We do that to avoid overfitting or underfitting model to our data. -//! Overfitting is bad because the model we trained fits trained data too well and can’t make any inferences on new data. +//! Overfitting is bad because the model we trained fits trained data too well and can't make any inferences on new data. //! Underfitted is bad because the model is undetrained and does not fit the training data well. //! Splitting data into multiple subsets helps us to find the right combination of hyperparameters, estimate model performance and choose the right model for //! the data. @@ -116,11 +116,8 @@ use crate::numbers::basenum::Number; use crate::numbers::realnum::RealNumber; use crate::rand_custom::get_rng_impl; -// TODO: fix this module -// pub(crate) mod hyper_tuning; pub(crate) mod kfold; -// pub use hyper_tuning::{GridSearchCV, GridSearchCVParameters}; pub use kfold::{KFold, KFoldIter}; /// An interface for the K-Folds cross-validator diff --git a/src/svm/mod.rs b/src/svm/mod.rs index 20489fd5..eca1f4c5 100644 --- a/src/svm/mod.rs +++ b/src/svm/mod.rs @@ -22,11 +22,8 @@ //! //! //! -/// search parameters pub mod svc; pub mod svr; -// search parameters space -pub mod search; use core::fmt::Debug; @@ -390,7 +387,6 @@ mod tests { .with_gamma(0.5) .with_degree(3.0) .with_coef0(1.0) - //.with_params(3.0, 0.5, 1.0) .apply(&v1, &v2) .unwrap() .abs(); From 3e300448ca2e496af435294ee03152ffa5f273d9 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Mon, 10 Aug 2026 19:26:55 +0100 Subject: [PATCH 7/8] fix(#398): remove unused Array1/ArrayView1 imports from svm/mod.rs --- src/svm/mod.rs | 88 -------------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/src/svm/mod.rs b/src/svm/mod.rs index eca1f4c5..647efd8f 100644 --- a/src/svm/mod.rs +++ b/src/svm/mod.rs @@ -30,12 +30,10 @@ use core::fmt::Debug; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -// Only import typetag if not compiling for wasm32 and serde is enabled #[cfg(all(feature = "serde", not(target_arch = "wasm32")))] use typetag; use crate::error::{Failed, FailedError}; -use crate::linalg::basic::arrays::{Array1, ArrayView1}; /// Defines a kernel function. /// This is a object-safe trait. @@ -126,32 +124,16 @@ pub enum Kernels { impl Kernels { /// Create a linear kernel. - /// - /// The linear kernel computes the dot product between two vectors: - /// K(x, y) = pub fn linear() -> Self { Kernels::Linear } /// Create an RBF kernel with unspecified gamma. - /// - /// The RBF kernel is defined as: - /// K(x, y) = exp(-gamma * ||x-y||²) - /// - /// You should specify gamma using `with_gamma()` before using this kernel. pub fn rbf() -> Self { Kernels::RBF { gamma: None } } /// Create a polynomial kernel with default parameters. - /// - /// The polynomial kernel is defined as: - /// K(x, y) = (gamma * + coef0)^degree - /// - /// Default values: - /// - gamma: None (must be specified) - /// - degree: None (must be specified) - /// - coef0: 1.0 pub fn polynomial() -> Self { Kernels::Polynomial { gamma: None, @@ -161,14 +143,6 @@ impl Kernels { } /// Create a sigmoid kernel with default parameters. - /// - /// The sigmoid kernel is defined as: - /// K(x, y) = tanh(gamma * + coef0) - /// - /// Default values: - /// - gamma: None (must be specified) - /// - coef0: 1.0 - /// pub fn sigmoid() -> Self { Kernels::Sigmoid { gamma: None, @@ -177,12 +151,6 @@ impl Kernels { } /// Set the `gamma` parameter for RBF, polynomial, or sigmoid kernels. - /// - /// The gamma parameter has different interpretations depending on the kernel: - /// - For RBF: Controls the width of the Gaussian. Larger values mean tighter fit. - /// - For Polynomial: Scaling factor for the dot product. - /// - For Sigmoid: Scaling factor for the dot product. - /// pub fn with_gamma(self, gamma: f64) -> Self { match self { Kernels::RBF { .. } => Kernels::RBF { gamma: Some(gamma) }, @@ -200,10 +168,6 @@ impl Kernels { } /// Set the `degree` parameter for the polynomial kernel. - /// - /// The degree parameter controls the flexibility of the decision boundary. - /// Higher degrees create more complex boundaries but may lead to overfitting. - /// pub fn with_degree(self, degree: f64) -> Self { match self { Kernels::Polynomial { gamma, coef0, .. } => Kernels::Polynomial { @@ -216,11 +180,6 @@ impl Kernels { } /// Set the `coef0` parameter for polynomial or sigmoid kernels. - /// - /// The coef0 parameter is the independent term in the kernel function: - /// - For Polynomial: Controls the influence of higher-degree vs. lower-degree terms. - /// - For Sigmoid: Acts as a threshold/bias term. - /// pub fn with_coef0(self, coef0: f64) -> Self { match self { Kernels::Polynomial { degree, gamma, .. } => Kernels::Polynomial { @@ -237,53 +196,6 @@ impl Kernels { } } -/// Implementation of the [`Kernel`] trait for the [`Kernels`] enum in smartcore. -/// -/// This method computes the value of the kernel function between two feature vectors `x_i` and `x_j`, -/// according to the variant and parameters of the [`Kernels`] enum. This enables flexible and type-safe -/// selection of kernel functions for SVM and SVR models in smartcore. -/// -/// # Supported Kernels -/// -/// - [`Kernels::Linear`]: Computes the standard dot product between `x_i` and `x_j`. -/// - [`Kernels::RBF`]: Computes the Radial Basis Function (Gaussian) kernel. Requires `gamma`. -/// - [`Kernels::Polynomial`]: Computes the polynomial kernel. Requires `degree`, `gamma`, and `coef0`. -/// - [`Kernels::Sigmoid`]: Computes the sigmoid kernel. Requires `gamma` and `coef0`. -/// -/// # Parameters -/// -/// - `x_i`: First input vector (feature vector). -/// - `x_j`: Second input vector (feature vector). -/// -/// # Returns -/// -/// - `Ok(f64)`: The computed kernel value. -/// - `Err(Failed)`: If any required kernel parameter is missing. -/// -/// # Errors -/// -/// Returns `Err(Failed)` if a required parameter (such as `gamma`, `degree`, or `coef0`) -/// is `None` for the selected kernel variant. -/// -/// # Example -/// -/// ``` -/// use smartcore::svm::Kernels; -/// use smartcore::svm::Kernel; -/// -/// let x = vec![1.0, 2.0, 3.0]; -/// let y = vec![4.0, 5.0, 6.0]; -/// let kernel = Kernels::rbf().with_gamma(0.5); -/// let value = kernel.apply(&x, &y).unwrap(); -/// ``` -/// -/// # Notes -/// -/// - This implementation follows smartcore's philosophy: pure Rust, no macros, no unsafe code, -/// and an accessible, pythonic API surface for both ML practitioners and Rust beginners. -/// - All kernel parameters must be set before calling `apply`; missing parameters will result in an error. -/// -/// See the [`Kernels`] enum documentation for more details on each kernel type and its parameters. #[cfg_attr(all(feature = "serde", not(target_arch = "wasm32")), typetag::serde)] impl Kernel for Kernels { fn apply(&self, x_i: &Vec, x_j: &Vec) -> Result { From 0828fd93d244996179925ed2f5e43234e7a547bb Mon Sep 17 00:00:00 2001 From: "Lorenzo (Mec-iS)" Date: Mon, 10 Aug 2026 19:34:56 +0100 Subject: [PATCH 8/8] revert --- .../hyper_tuning/grid_search.rs | 239 ++++++++++++++ src/model_selection/hyper_tuning/mod.rs | 2 + src/model_selection/mod.rs | 5 +- src/svm/mod.rs | 4 + src/svm/search/mod.rs | 6 + src/svm/search/svc_params.rs | 183 +++++++++++ src/svm/search/svr_params.rs | 293 ++++++++++++++++++ 7 files changed, 731 insertions(+), 1 deletion(-) create mode 100644 src/model_selection/hyper_tuning/grid_search.rs create mode 100644 src/model_selection/hyper_tuning/mod.rs create mode 100644 src/svm/search/mod.rs create mode 100644 src/svm/search/svc_params.rs create mode 100644 src/svm/search/svr_params.rs diff --git a/src/model_selection/hyper_tuning/grid_search.rs b/src/model_selection/hyper_tuning/grid_search.rs new file mode 100644 index 00000000..74242c60 --- /dev/null +++ b/src/model_selection/hyper_tuning/grid_search.rs @@ -0,0 +1,239 @@ +// TODO: missing documentation + +use crate::{ + api::{Predictor, SupervisedEstimator}, + error::{Failed, FailedError}, + linalg::basic::arrays::{Array1, Array2}, + numbers::basenum::Number, + numbers::realnum::RealNumber, +}; + +use crate::model_selection::{cross_validate, BaseKFold, CrossValidationResult}; + +/// Parameters for GridSearchCV +#[derive(Debug)] +pub struct GridSearchCVParameters< + T: Number, + M: Array2, + C: Clone, + I: Iterator, + E: Predictor, + F: Fn(&M, &M::RowVector, C) -> Result, + K: BaseKFold, + S: Fn(&M::RowVector, &M::RowVector) -> T, +> { + _phantom: std::marker::PhantomData<(T, M)>, + + parameters_search: I, + estimator: F, + score: S, + cv: K, +} + +impl< + T: RealNumber, + M: Array2, + C: Clone, + I: Iterator, + E: Predictor, + F: Fn(&M, &M::RowVector, C) -> Result, + K: BaseKFold, + S: Fn(&M::RowVector, &M::RowVector) -> T, + > GridSearchCVParameters +{ + /// Create new GridSearchCVParameters + pub fn new(parameters_search: I, estimator: F, score: S, cv: K) -> Self { + GridSearchCVParameters { + _phantom: std::marker::PhantomData, + parameters_search, + estimator, + score, + cv, + } + } +} +/// Exhaustive search over specified parameter values for an estimator. +#[derive(Debug)] +pub struct GridSearchCV, C: Clone, E: Predictor> { + _phantom: std::marker::PhantomData<(T, M)>, + predictor: E, + /// Cross validation results. + pub cross_validation_result: CrossValidationResult, + /// best parameter + pub best_parameter: C, +} + +impl, E: Predictor, C: Clone> + GridSearchCV +{ + /// Search for the best estimator by testing all possible combinations with cross-validation using given metric. + /// * `x` - features, matrix of size _NxM_ where _N_ is number of samples and _M_ is number of attributes. + /// * `y` - target values, should be of size _N_ + /// * `gs_parameters` - GridSearchCVParameters struct + pub fn fit< + I: Iterator, + K: BaseKFold, + F: Fn(&M, &M::RowVector, C) -> Result, + S: Fn(&M::RowVector, &M::RowVector) -> T, + >( + x: &M, + y: &M::RowVector, + gs_parameters: GridSearchCVParameters, + ) -> Result { + let mut best_result: Option> = None; + let mut best_parameters = None; + let parameters_search = gs_parameters.parameters_search; + let estimator = gs_parameters.estimator; + let cv = gs_parameters.cv; + let score = gs_parameters.score; + + for parameters in parameters_search { + let result = cross_validate(&estimator, x, y, ¶meters, &cv, &score)?; + if best_result.is_none() + || result.mean_test_score() > best_result.as_ref().unwrap().mean_test_score() + { + best_parameters = Some(parameters); + best_result = Some(result); + } + } + + if let (Some(best_parameter), Some(cross_validation_result)) = + (best_parameters, best_result) + { + let predictor = estimator(x, y, best_parameter.clone())?; + Ok(Self { + _phantom: gs_parameters._phantom, + predictor, + cross_validation_result, + best_parameter, + }) + } else { + Err(Failed::because( + FailedError::FindFailed, + "there were no parameter sets found", + )) + } + } + + /// Return grid search cross validation results + pub fn cv_results(&self) -> &CrossValidationResult { + &self.cross_validation_result + } + + /// Return best parameters found + pub fn best_parameters(&self) -> &C { + &self.best_parameter + } + + /// Call predict on the estimator with the best found parameters + pub fn predict(&self, x: &M) -> Result { + self.predictor.predict(x) + } +} + +impl< + T: RealNumber, + M: Array2, + C: Clone, + I: Iterator, + E: Predictor, + F: Fn(&M, &M::RowVector, C) -> Result, + K: BaseKFold, + S: Fn(&M::RowVector, &M::RowVector) -> T, + > SupervisedEstimator> + for GridSearchCV +{ + fn fit( + x: &M, + y: &M::RowVector, + parameters: GridSearchCVParameters, + ) -> Result { + GridSearchCV::fit(x, y, parameters) + } +} + +impl, C: Clone, E: Predictor> + Predictor for GridSearchCV +{ + fn predict(&self, x: &M) -> Result { + self.predict(x) + } +} + +#[cfg(test)] +mod tests { + + use crate::{ + linalg::naive::dense_matrix::DenseMatrix, + linear::logistic_regression::{LogisticRegression, LogisticRegressionSearchParameters}, + metrics::accuracy, + model_selection::{ + hyper_tuning::grid_search::{self, GridSearchCVParameters}, + KFold, + }, + }; + use grid_search::GridSearchCV; + + #[test] + fn test_grid_search() { + let x = DenseMatrix::from_2d_array(&[ + &[5.1, 3.5, 1.4, 0.2], + &[4.9, 3.0, 1.4, 0.2], + &[4.7, 3.2, 1.3, 0.2], + &[4.6, 3.1, 1.5, 0.2], + &[5.0, 3.6, 1.4, 0.2], + &[5.4, 3.9, 1.7, 0.4], + &[4.6, 3.4, 1.4, 0.3], + &[5.0, 3.4, 1.5, 0.2], + &[4.4, 2.9, 1.4, 0.2], + &[4.9, 3.1, 1.5, 0.1], + &[7.0, 3.2, 4.7, 1.4], + &[6.4, 3.2, 4.5, 1.5], + &[6.9, 3.1, 4.9, 1.5], + &[5.5, 2.3, 4.0, 1.3], + &[6.5, 2.8, 4.6, 1.5], + &[5.7, 2.8, 4.5, 1.3], + &[6.3, 3.3, 4.7, 1.6], + &[4.9, 2.4, 3.3, 1.0], + &[6.6, 2.9, 4.6, 1.3], + &[5.2, 2.7, 3.9, 1.4], + ]); + let y = vec![ + 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ]; + + let cv = KFold { + n_splits: 5, + ..KFold::default() + }; + + let parameters = LogisticRegressionSearchParameters { + alpha: vec![0., 1.], + ..Default::default() + }; + + let grid_search = GridSearchCV::fit( + &x, + &y, + GridSearchCVParameters { + estimator: LogisticRegression::fit, + score: accuracy, + cv, + parameters_search: parameters.into_iter(), + _phantom: Default::default(), + }, + ) + .unwrap(); + let best_parameters = grid_search.best_parameters(); + + assert!([1.].contains(&best_parameters.alpha)); + + let cv_results = grid_search.cv_results(); + + assert_eq!(cv_results.mean_test_score(), 0.9); + + let x = DenseMatrix::from_2d_array(&[&[5., 3., 1., 0.]]); + let result = grid_search.predict(&x).unwrap(); + assert_eq!(result, vec![0.]); + } +} diff --git a/src/model_selection/hyper_tuning/mod.rs b/src/model_selection/hyper_tuning/mod.rs new file mode 100644 index 00000000..dfe0d06b --- /dev/null +++ b/src/model_selection/hyper_tuning/mod.rs @@ -0,0 +1,2 @@ +mod grid_search; +pub use grid_search::{GridSearchCV, GridSearchCVParameters}; diff --git a/src/model_selection/mod.rs b/src/model_selection/mod.rs index bdd48d76..fe74fce4 100644 --- a/src/model_selection/mod.rs +++ b/src/model_selection/mod.rs @@ -2,7 +2,7 @@ //! //! In statistics and machine learning we usually split our data into two sets: one for training and the other one for testing. //! We fit our model to the training data, in order to make predictions on the test data. We do that to avoid overfitting or underfitting model to our data. -//! Overfitting is bad because the model we trained fits trained data too well and can't make any inferences on new data. +//! Overfitting is bad because the model we trained fits trained data too well and can’t make any inferences on new data. //! Underfitted is bad because the model is undetrained and does not fit the training data well. //! Splitting data into multiple subsets helps us to find the right combination of hyperparameters, estimate model performance and choose the right model for //! the data. @@ -116,8 +116,11 @@ use crate::numbers::basenum::Number; use crate::numbers::realnum::RealNumber; use crate::rand_custom::get_rng_impl; +// TODO: fix this module +// pub(crate) mod hyper_tuning; pub(crate) mod kfold; +// pub use hyper_tuning::{GridSearchCV, GridSearchCVParameters}; pub use kfold::{KFold, KFoldIter}; /// An interface for the K-Folds cross-validator diff --git a/src/svm/mod.rs b/src/svm/mod.rs index eca1f4c5..20489fd5 100644 --- a/src/svm/mod.rs +++ b/src/svm/mod.rs @@ -22,8 +22,11 @@ //! //! //! +/// search parameters pub mod svc; pub mod svr; +// search parameters space +pub mod search; use core::fmt::Debug; @@ -387,6 +390,7 @@ mod tests { .with_gamma(0.5) .with_degree(3.0) .with_coef0(1.0) + //.with_params(3.0, 0.5, 1.0) .apply(&v1, &v2) .unwrap() .abs(); diff --git a/src/svm/search/mod.rs b/src/svm/search/mod.rs new file mode 100644 index 00000000..d61b8326 --- /dev/null +++ b/src/svm/search/mod.rs @@ -0,0 +1,6 @@ +//! SVC and Grid Search + +/// SVC search parameters +pub mod svc_params; +/// SVC search parameters +pub mod svr_params; diff --git a/src/svm/search/svc_params.rs b/src/svm/search/svc_params.rs new file mode 100644 index 00000000..42f686b3 --- /dev/null +++ b/src/svm/search/svc_params.rs @@ -0,0 +1,183 @@ +// /// SVC grid search parameters +// #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +// #[derive(Debug, Clone)] +// pub struct SVCSearchParameters< +// TX: Number + RealNumber, +// TY: Number + Ord, +// X: Array2, +// Y: Array1, +// K: Kernel, +// > { +// #[cfg_attr(feature = "serde", serde(default))] +// /// Number of epochs. +// pub epoch: Vec, +// #[cfg_attr(feature = "serde", serde(default))] +// /// Regularization parameter. +// pub c: Vec, +// #[cfg_attr(feature = "serde", serde(default))] +// /// Tolerance for stopping epoch. +// pub tol: Vec, +// #[cfg_attr(feature = "serde", serde(default))] +// /// The kernel function. +// pub kernel: Vec, +// #[cfg_attr(feature = "serde", serde(default))] +// /// Unused parameter. +// m: PhantomData<(X, Y, TY)>, +// #[cfg_attr(feature = "serde", serde(default))] +// /// Controls the pseudo random number generation for shuffling the data for probability estimates +// seed: Vec>, +// } + +// /// SVC grid search iterator +// pub struct SVCSearchParametersIterator< +// TX: Number + RealNumber, +// TY: Number + Ord, +// X: Array2, +// Y: Array1, +// K: Kernel, +// > { +// svc_search_parameters: SVCSearchParameters, +// current_epoch: usize, +// current_c: usize, +// current_tol: usize, +// current_kernel: usize, +// current_seed: usize, +// } + +// impl, Y: Array1, K: Kernel> +// IntoIterator for SVCSearchParameters +// { +// type Item = SVCParameters<'a, TX, TY, X, Y>; +// type IntoIter = SVCSearchParametersIterator; + +// fn into_iter(self) -> Self::IntoIter { +// SVCSearchParametersIterator { +// svc_search_parameters: self, +// current_epoch: 0, +// current_c: 0, +// current_tol: 0, +// current_kernel: 0, +// current_seed: 0, +// } +// } +// } + +// impl, Y: Array1, K: Kernel> +// Iterator for SVCSearchParametersIterator +// { +// type Item = SVCParameters; + +// fn next(&mut self) -> Option { +// if self.current_epoch == self.svc_search_parameters.epoch.len() +// && self.current_c == self.svc_search_parameters.c.len() +// && self.current_tol == self.svc_search_parameters.tol.len() +// && self.current_kernel == self.svc_search_parameters.kernel.len() +// && self.current_seed == self.svc_search_parameters.seed.len() +// { +// return None; +// } + +// let next = SVCParameters { +// epoch: self.svc_search_parameters.epoch[self.current_epoch], +// c: self.svc_search_parameters.c[self.current_c], +// tol: self.svc_search_parameters.tol[self.current_tol], +// kernel: self.svc_search_parameters.kernel[self.current_kernel].clone(), +// m: PhantomData, +// seed: self.svc_search_parameters.seed[self.current_seed], +// }; + +// if self.current_epoch + 1 < self.svc_search_parameters.epoch.len() { +// self.current_epoch += 1; +// } else if self.current_c + 1 < self.svc_search_parameters.c.len() { +// self.current_epoch = 0; +// self.current_c += 1; +// } else if self.current_tol + 1 < self.svc_search_parameters.tol.len() { +// self.current_epoch = 0; +// self.current_c = 0; +// self.current_tol += 1; +// } else if self.current_kernel + 1 < self.svc_search_parameters.kernel.len() { +// self.current_epoch = 0; +// self.current_c = 0; +// self.current_tol = 0; +// self.current_kernel += 1; +// } else if self.current_seed + 1 < self.svc_search_parameters.seed.len() { +// self.current_epoch = 0; +// self.current_c = 0; +// self.current_tol = 0; +// self.current_kernel = 0; +// self.current_seed += 1; +// } else { +// self.current_epoch += 1; +// self.current_c += 1; +// self.current_tol += 1; +// self.current_kernel += 1; +// self.current_seed += 1; +// } + +// Some(next) +// } +// } + +// impl, Y: Array1, K: Kernel> Default +// for SVCSearchParameters +// { +// fn default() -> Self { +// let default_params: SVCParameters = SVCParameters::default(); + +// SVCSearchParameters { +// epoch: vec![default_params.epoch], +// c: vec![default_params.c], +// tol: vec![default_params.tol], +// kernel: vec![default_params.kernel], +// m: PhantomData, +// seed: vec![default_params.seed], +// } +// } +// } + +// #[cfg(test)] +// mod tests { +// use num::ToPrimitive; + +// use super::*; +// use crate::linalg::basic::matrix::DenseMatrix; +// use crate::metrics::accuracy; +// #[cfg(feature = "serde")] +// use crate::svm::*; + +// #[test] +// fn search_parameters() { +// let parameters: SVCSearchParameters, LinearKernel> = +// SVCSearchParameters { +// epoch: vec![10, 100], +// kernel: vec![LinearKernel {}], +// ..Default::default() +// }; +// let mut iter = parameters.into_iter(); +// let next = iter.next().unwrap(); +// assert_eq!(next.epoch, 10); +// assert_eq!(next.kernel, LinearKernel {}); +// let next = iter.next().unwrap(); +// assert_eq!(next.epoch, 100); +// assert_eq!(next.kernel, LinearKernel {}); +// assert!(iter.next().is_none()); +// } + +// #[test] +// fn search_parameters() { +// let parameters: SVCSearchParameters, LinearKernel> = +// SVCSearchParameters { +// epoch: vec![10, 100], +// kernel: vec![LinearKernel {}], +// ..Default::default() +// }; +// let mut iter = parameters.into_iter(); +// let next = iter.next().unwrap(); +// assert_eq!(next.epoch, 10); +// assert_eq!(next.kernel, LinearKernel {}); +// let next = iter.next().unwrap(); +// assert_eq!(next.epoch, 100); +// assert_eq!(next.kernel, LinearKernel {}); +// assert!(iter.next().is_none()); +// } +// } diff --git a/src/svm/search/svr_params.rs b/src/svm/search/svr_params.rs new file mode 100644 index 00000000..f455c119 --- /dev/null +++ b/src/svm/search/svr_params.rs @@ -0,0 +1,293 @@ +//! # SVR Grid Search Parameters +//! +//! This module provides utilities for defining and iterating over grid search parameter spaces +//! for Support Vector Regression (SVR) models in [smartcore](https://github.com/smartcorelib/smartcore). +//! +//! The main struct, [`SVRSearchParameters`], allows users to specify multiple values for each +//! SVR hyperparameter (epsilon, regularization parameter C, tolerance, and kernel function). +//! The provided iterator yields all possible combinations (the Cartesian product) of these parameters, +//! enabling exhaustive grid search for hyperparameter tuning. +//! +//! +//! ## Example +//! ``` +//! use smartcore::svm::Kernels; +//! use smartcore::svm::search::svr_params::SVRSearchParameters; +//! use smartcore::linalg::basic::matrix::DenseMatrix; +//! +//! let params = SVRSearchParameters::> { +//! eps: vec![0.1, 0.2], +//! c: vec![1.0, 10.0], +//! tol: vec![1e-3], +//! kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], +//! m: std::marker::PhantomData, +//! }; +//! +//! // for param_set in params.into_iter() { +//! // Use param_set (of type svr::SVRParameters) to fit and evaluate your SVR model. +//! // } +//! ``` +//! +//! +//! ## Note +//! This module is intended for use with smartcore version 0.4 or later. The API is not compatible with older versions[1]. +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +use crate::linalg::basic::arrays::Array2; +use crate::numbers::basenum::Number; +use crate::numbers::floatnum::FloatNumber; +use crate::numbers::realnum::RealNumber; +use crate::svm::{Kernels, svr}; +use std::marker::PhantomData; + +/// ## SVR grid search parameters +/// A struct representing a grid of hyperparameters for SVR grid search in smartcore. +/// +/// Each field is a vector of possible values for the corresponding SVR hyperparameter. +/// The [`IntoIterator`] implementation yields every possible combination of these parameters +/// as an `svr::SVRParameters` struct, suitable for use in model selection routines. +/// +/// # Type Parameters +/// - `T`: Numeric type for parameters (e.g., `f64`) +/// - `M`: Matrix type implementing [`Array2`] +/// +/// # Fields +/// - `eps`: Vector of epsilon values for the epsilon-insensitive loss in SVR. +/// - `c`: Vector of regularization parameters (C) for SVR. +/// - `tol`: Vector of tolerance values for the stopping criterion. +/// - `kernel`: Vector of kernel function variants (see [`Kernels`]). +/// - `m`: Phantom data for the matrix type parameter. +/// +/// # Example +/// ``` +/// use smartcore::svm::Kernels; +/// use smartcore::svm::search::svr_params::SVRSearchParameters; +/// use smartcore::linalg::basic::matrix::DenseMatrix; +/// +/// let params = SVRSearchParameters::> { +/// eps: vec![0.1, 0.2], +/// c: vec![1.0, 10.0], +/// tol: vec![1e-3], +/// kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], +/// m: std::marker::PhantomData, +/// }; +/// ``` +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone)] +pub struct SVRSearchParameters> { + /// Epsilon in the epsilon-SVR model. + pub eps: Vec, + /// Regularization parameter. + pub c: Vec, + /// Tolerance for stopping eps. + pub tol: Vec, + /// The kernel function. + pub kernel: Vec, + /// Unused parameter. + pub m: PhantomData, +} + +/// SVR grid search iterator +pub struct SVRSearchParametersIterator> { + svr_search_parameters: SVRSearchParameters, + current_eps: usize, + current_c: usize, + current_tol: usize, + current_kernel: usize, +} + +impl> IntoIterator + for SVRSearchParameters +{ + type Item = svr::SVRParameters; + type IntoIter = SVRSearchParametersIterator; + + fn into_iter(self) -> Self::IntoIter { + SVRSearchParametersIterator { + svr_search_parameters: self, + current_eps: 0, + current_c: 0, + current_tol: 0, + current_kernel: 0, + } + } +} + +impl> Iterator + for SVRSearchParametersIterator +{ + type Item = svr::SVRParameters; + + fn next(&mut self) -> Option { + if self.current_eps == self.svr_search_parameters.eps.len() + && self.current_c == self.svr_search_parameters.c.len() + && self.current_tol == self.svr_search_parameters.tol.len() + && self.current_kernel == self.svr_search_parameters.kernel.len() + { + return None; + } + + let next = svr::SVRParameters:: { + eps: self.svr_search_parameters.eps[self.current_eps], + c: self.svr_search_parameters.c[self.current_c], + tol: self.svr_search_parameters.tol[self.current_tol], + kernel: Some(self.svr_search_parameters.kernel[self.current_kernel].clone()), + }; + + if self.current_eps + 1 < self.svr_search_parameters.eps.len() { + self.current_eps += 1; + } else if self.current_c + 1 < self.svr_search_parameters.c.len() { + self.current_eps = 0; + self.current_c += 1; + } else if self.current_tol + 1 < self.svr_search_parameters.tol.len() { + self.current_eps = 0; + self.current_c = 0; + self.current_tol += 1; + } else if self.current_kernel + 1 < self.svr_search_parameters.kernel.len() { + self.current_eps = 0; + self.current_c = 0; + self.current_tol = 0; + self.current_kernel += 1; + } else { + self.current_eps += 1; + self.current_c += 1; + self.current_tol += 1; + self.current_kernel += 1; + } + + Some(next) + } +} + +impl> Default for SVRSearchParameters { + fn default() -> Self { + let default_params: svr::SVRParameters = svr::SVRParameters::default(); + + SVRSearchParameters { + eps: vec![default_params.eps], + c: vec![default_params.c], + tol: vec![default_params.tol], + kernel: vec![default_params.kernel.unwrap_or_else(Kernels::linear)], + m: PhantomData, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::linalg::basic::matrix::DenseMatrix; + use crate::svm::Kernels; + + type T = f64; + type M = DenseMatrix; + + #[test] + fn test_default_parameters() { + let params = SVRSearchParameters::::default(); + assert_eq!(params.eps.len(), 1); + assert_eq!(params.c.len(), 1); + assert_eq!(params.tol.len(), 1); + assert_eq!(params.kernel.len(), 1); + // Check that the default kernel is linear + assert_eq!(params.kernel[0], Kernels::linear()); + } + + #[test] + fn test_single_grid_iteration() { + let params = SVRSearchParameters:: { + eps: vec![0.1], + c: vec![1.0], + tol: vec![1e-3], + kernel: vec![Kernels::rbf().with_gamma(0.5)], + m: PhantomData, + }; + let mut iter = params.into_iter(); + let param = iter.next().unwrap(); + assert_eq!(param.eps, 0.1); + assert_eq!(param.c, 1.0); + assert_eq!(param.tol, 1e-3); + assert_eq!(param.kernel, Some(Kernels::rbf().with_gamma(0.5))); + assert!(iter.next().is_none()); + } + + #[test] + fn test_cartesian_grid_iteration() { + let params = SVRSearchParameters:: { + eps: vec![0.1, 0.2], + c: vec![1.0, 2.0], + tol: vec![1e-3], + kernel: vec![Kernels::linear(), Kernels::rbf().with_gamma(0.5)], + m: PhantomData, + }; + let expected_count = + params.eps.len() * params.c.len() * params.tol.len() * params.kernel.len(); + let results: Vec<_> = params.into_iter().collect(); + assert_eq!(results.len(), expected_count); + + // Check that all parameter combinations are present + let mut seen = vec![]; + for p in &results { + seen.push((p.eps, p.c, p.tol, p.kernel.clone().unwrap())); + } + for &eps in &[0.1, 0.2] { + for &c in &[1.0, 2.0] { + for &tol in &[1e-3] { + for kernel in &[Kernels::linear(), Kernels::rbf().with_gamma(0.5)] { + assert!(seen.contains(&(eps, c, tol, kernel.clone()))); + } + } + } + } + } + + #[test] + fn test_empty_grid() { + let params = SVRSearchParameters:: { + eps: vec![], + c: vec![], + tol: vec![], + kernel: vec![], + m: PhantomData, + }; + let mut iter = params.into_iter(); + assert!(iter.next().is_none()); + } + + #[test] + fn test_kernel_enum_variants() { + let lin = Kernels::linear(); + let rbf = Kernels::rbf().with_gamma(0.2); + let poly = Kernels::polynomial() + .with_degree(2.0) + .with_gamma(1.0) + .with_coef0(0.5); + let sig = Kernels::sigmoid().with_gamma(0.3).with_coef0(0.1); + + assert_eq!(lin, Kernels::Linear); + match rbf { + Kernels::RBF { gamma } => assert_eq!(gamma, Some(0.2)), + _ => panic!("Not RBF"), + } + match poly { + Kernels::Polynomial { + degree, + gamma, + coef0, + } => { + assert_eq!(degree, Some(2.0)); + assert_eq!(gamma, Some(1.0)); + assert_eq!(coef0, Some(0.5)); + } + _ => panic!("Not Polynomial"), + } + match sig { + Kernels::Sigmoid { gamma, coef0 } => { + assert_eq!(gamma, Some(0.3)); + assert_eq!(coef0, Some(0.1)); + } + _ => panic!("Not Sigmoid"), + } + } +}