Iterative work on stabilizing XCS

XCS_script.py

@@ -0,0 +1 @@
+

lcs/agents/Agent.py

@@ -3,10 +3,7 @@
 from timeit import default_timer as timer
 from typing import Callable, List, Tuple
 
-import dill
-import mlflow
 import numpy as np
-import tempfile
 
 from lcs.metrics import basic_metrics
 
@@ -119,8 +116,6 @@ def _evaluate(self,
         tuple
             population of classifiers and metrics
         """
-        using_mlflow = hasattr(self.get_cfg(), 'use_mlflow') and self.get_cfg().use_mlflow
-
         current_trial = 0
         steps = 0
 
@@ -143,26 +138,6 @@ def _evaluate(self,
 
                 metrics.append(m)
 
-                if using_mlflow:
-                    mlflow.log_metrics(m, current_trial)
-
-            # checkpoint model and metrics
-            if self.get_cfg().model_checkpoint_freq:
-                if current_trial % self.get_cfg().model_checkpoint_freq == 0:
-                    prefix = f"-trial-{current_trial}"
-                    with tempfile.TemporaryDirectory(prefix) as td:
-                        logger.debug(f"checkpointing model to {td}")
-                        pop_path = f"{td}/population.dill"
-                        metrics_path = f"{td}/metrics.dill"
-
-                        dill.dump(self.get_population(),
-                                  open(pop_path, mode='wb'))
-
-                        dill.dump(metrics, open(metrics_path, mode='wb'))
-
-                        if using_mlflow:
-                            mlflow.log_artifacts(td, f"{current_trial}/")
-
             # Print last metric
             if current_trial % np.round(n_trials / 10) == 0:
                 logger.info(metrics[-1])

lcs/agents/acs/Configuration.py

@@ -12,7 +12,6 @@ def __init__(self,
                  user_metrics_collector_fcn: Callable = None,
                  fitness_fcn=None,
                  metrics_trial_frequency: int = 5,
-                 model_checkpoint_frequency: int = None,
                  do_subsumption: bool = True,
                  beta: float = 0.05,
                  # gamma: float = 0.95,
@@ -21,8 +20,7 @@ def __init__(self,
                  epsilon: float = 0.5,
                  u_max: int = 100000,
                  theta_exp: int = 20,
-                 theta_as: int = 20,
-                 use_mlflow: bool = False) -> None:
+                 theta_as: int = 20) -> None:
         """
         Creates the configuration object used during training the ACS2 agent.
 
@@ -51,7 +49,6 @@ def __init__(self,
         self.classifier_wildcard = classifier_wildcard
         self.environment_adapter = environment_adapter
         self.metrics_trial_frequency = metrics_trial_frequency
-        self.model_checkpoint_freq = model_checkpoint_frequency
         self.user_metrics_collector_fcn = user_metrics_collector_fcn
         self.fitness_fcn = fitness_fcn
         self.do_subsumption = do_subsumption
@@ -62,7 +59,6 @@ def __init__(self,
         self.epsilon = epsilon
         self.u_max = u_max
         self.theta_as = theta_as
-        self.use_mlflow = use_mlflow
 
     def __str__(self) -> str:
         return str(vars(self))

lcs/agents/acs2/Configuration.py

@@ -14,7 +14,6 @@ def __init__(self,
                  user_metrics_collector_fcn: Callable = None,
                  fitness_fcn=None,
                  metrics_trial_frequency: int = 5,
-                 model_checkpoint_frequency: int = None,
                  do_pee: bool = False,
                  do_ga: bool = False,
                  do_subsumption: bool = True,
@@ -33,8 +32,7 @@ def __init__(self,
                  theta_ga: int = 100,
                  theta_as: int = 20,
                  mu: float = 0.3,
-                 chi: float = 0.8,
-                 use_mlflow: bool = False):
+                 chi: float = 0.8):
 
         super(Configuration, self).__init__(
             classifier_length,
@@ -44,16 +42,14 @@ def __init__(self,
             user_metrics_collector_fcn,
             fitness_fcn,
             metrics_trial_frequency,
-            model_checkpoint_frequency,
             do_subsumption,
             beta,
             theta_i,
             theta_r,
             epsilon,
             u_max,
             theta_exp,
-            theta_as,
-            use_mlflow)
+            theta_as)
 
         self.gamma = gamma
         self.do_pee = do_pee

lcs/agents/xcs/Classifier.py

@@ -12,10 +12,13 @@ def __init__(self,
                  condition: Union[Condition, str, None] = None,
                  action: Optional[int] = None,
                  time_stamp: int = None) -> None:
+
         if cfg is None:
             raise TypeError("Configuration should be passed to Classifier")
+
         if type(condition) != Condition:
-            condition = str(condition)
+            condition = Condition(condition)
+
         self.cfg = cfg                  # cfg
         self.condition = condition      # current situation
         self.action = action            # A - int action
@@ -42,9 +45,7 @@ def does_subsume(self, other):
 
     @property
     def could_subsume(self):
-        if self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error:
-                return True
-        return False
+        return self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error
 
     def is_more_general(self, other):
         if self.wildcard_number <= other.wildcard_number:
@@ -60,9 +61,10 @@ def __len__(self):
 
     def __str__(self):
         return f"Cond:{self.condition} - Act:{self.action} - Num:{self.numerosity} " + \
-            f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]"
+            f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, Error:{self.error}]"
 
-    def __eq__(self, other):
-        if other.action == self.action and other.condition == self.condition:
-                return True
-        return False
+    def __eq__(self, o):
+        return o.condition == self.condition and o.action == self.action
+
+    def __hash__(self):
+        return hash((str(self.condition), self.action))

lcs/agents/xcs/ClassifiersList.py

@@ -1,10 +1,10 @@
-import numpy as np
-import random
 import logging
+import random
+
+import numpy as np
 
 from lcs import TypedList, Perception
 from lcs.agents.xcs import Classifier, Condition, Configuration
-
 logger = logging.getLogger(__name__)
 
 
@@ -18,11 +18,12 @@ def __init__(self,
         super().__init__(*args, oktypes=oktypes)
 
     def insert_in_population(self, cl: Classifier):
-        for c in self:
-            if c == cl:
-                c.numerosity += 1
-                return 
-        self.append(cl)
+        existing_classifiers = [c for c in self if c == cl]
+        if len(existing_classifiers) > 0:
+            assert len(existing_classifiers) == 1, 'duplicates found, while inserting'
+            existing_classifiers[0].numerosity += 1
+        else:
+            self.append(cl)
 
     def generate_covering_classifier(self, situation, action, time_stamp):
         # both Perception and string has __getitem__
@@ -33,22 +34,23 @@ def generate_covering_classifier(self, situation, action, time_stamp):
                 generalized.append(self.cfg.classifier_wildcard)
             else:
                 generalized.append(situation[i])
-        cl = Classifier(cfg=self.cfg,
-                        condition=Condition(generalized),
-                        action=action,
-                        time_stamp=time_stamp)
-        return cl
+
+        return Classifier(condition=Condition(generalized),
+                          action=action,
+                          time_stamp=time_stamp,
+                          cfg=self.cfg)
 
     def _generate_covering_and_insert(self, situation, action, time_stamp):
         cl = self.generate_covering_classifier(situation, action, time_stamp)
         self.insert_in_population(cl)
         self.delete_from_population()
         return cl
 
-    # Roulette-Wheel Deletion
-    # TODO: use strategies
     def delete_from_population(self):
-        if self.numerosity > self.cfg.max_population:
+        # TODO: change while to if
+        # During woods the number of rules grew over max number
+        # To remedy this I added while instead of if. Correct issue should be changed.
+        while self.numerosity > self.cfg.max_population:
             average_fitness = sum(cl.fitness for cl in self) / self.numerosity
             deletion_votes = []
             for cl in self:
@@ -59,7 +61,7 @@ def delete_from_population(self):
     def _deletion_vote(self, cl, average_fitness):
         vote = cl.action_set_size * cl.numerosity
         if cl.experience > self.cfg.deletion_threshold and \
-                cl.fitness / cl.numerosity < \
+            cl.fitness / cl.numerosity < \
                 self.cfg.delta * average_fitness:
             vote *= average_fitness / (cl.fitness / cl.numerosity)
         return vote
@@ -70,16 +72,18 @@ def _remove_based_on_votes(self, deletion_votes, selector):
             if selector <= 0:
                 if cl.numerosity > 1:
                     cl.numerosity -= 1
+                    return cl
                 else:
                     self.safe_remove(cl)
-                return None
+                    return cl
 
     def generate_match_set(self, situation: Perception, time_stamp):
         matching_ls = [cl for cl in self if cl.does_match(situation)]
-        while len(matching_ls) < self.cfg.number_of_actions:
-            action = self._find_not_present_action(matching_ls)
+        action = self._find_not_present_action(matching_ls)
+        while action is not None:
             cl = self._generate_covering_and_insert(situation, action, time_stamp)
             matching_ls.append(cl)
+            action = self._find_not_present_action(matching_ls)
         return ClassifiersList(self.cfg, *matching_ls)
 
     def _find_not_present_action(self, matching_set):
@@ -116,14 +120,18 @@ def update_set(self, p):
         for cl in self:
             cl.experience += 1
             # update prediction, prediction error, action set size estimate
-            if cl.experience < 1/self.cfg.learning_rate:
-                cl.prediction += (p - cl.prediction) / cl.experience
-                cl.error += (abs(p - cl.prediction) - cl.error) / cl.experience
-                cl.action_set_size +=\
+            if cl.experience < 1 / self.cfg.learning_rate:
+                cl.prediction += \
+                    (p - cl.prediction) / cl.experience
+                cl.error += \
+                    (abs(p - cl.prediction) - cl.error) / cl.experience
+                cl.action_set_size += \
                     (action_set_numerosity - cl.action_set_size) / cl.experience
             else:
-                cl.prediction += self.cfg.learning_rate * (p - cl.prediction)
-                cl.error += self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error)
+                cl.prediction +=\
+                    self.cfg.learning_rate * (p - cl.prediction)
+                cl.error += \
+                    self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error)
                 cl.action_set_size += \
                     self.cfg.learning_rate * (action_set_numerosity - cl.action_set_size)
         self._update_fitness()
@@ -135,14 +143,13 @@ def _update_fitness(self):
             if cl.error < self.cfg.epsilon_0:
                 tmp_acc = 1
             else:
-                tmp_acc = (self.cfg.alpha *
-                           (cl.error * self.cfg.epsilon_0) **
-                           -self.cfg.v
-                           )
+                tmp_acc = (pow(self.cfg.alpha * (cl.error * self.cfg.epsilon_0), - self.cfg.v))
             accuracy_vector_k.append(tmp_acc)
-            accuracy_sum += tmp_acc + cl.numerosity
+            accuracy_sum += tmp_acc * cl.numerosity
         for cl, k in zip(self, accuracy_vector_k):
             cl.fitness += (
                 self.cfg.learning_rate *
                 (k * cl.numerosity / accuracy_sum - cl.fitness)
             )
+
+

lcs/agents/xcs/Condition.py

@@ -1,5 +1,4 @@
 from __future__ import annotations
-
 from .. import ImmutableSequence
 
 

lcs/agents/xcs/Configuration.py

@@ -29,7 +29,8 @@ def __init__(self,
                  do_ga_subsumption: bool = False,
                  do_action_set_subsumption: bool = False,
                  metrics_trial_frequency: int = 5,
-                 user_metrics_collector_fcn: Callable = None
+                 user_metrics_collector_fcn: Callable = None,
+                 multistep_enfiroment: bool = True
                  ) -> None:
         """
         :param classifier_wildcard: Wildcard symbol
@@ -76,9 +77,9 @@ def __init__(self,
         self.number_of_actions = number_of_actions
         self.do_GA_subsumption = do_ga_subsumption
         self.do_action_set_subsumption = do_action_set_subsumption
-
         self.metrics_trial_frequency = metrics_trial_frequency
         self.user_metrics_collector_fcn = user_metrics_collector_fcn
+        self.multistep_enfiroment = multistep_enfiroment
 
     def __str__(self) -> str:
         return str(vars(self))