From 285a79acebf6da1ca23b83a556ed73176f78e084 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 6 May 2021 13:39:34 +0200 Subject: [PATCH 01/33] Refractor Genetic Algorithm --- lcs/agents/xcs/GeneticAlgorithm.py | 92 ++++++++++----------- tests/lcs/agents/xcs/test_Classifier.py | 7 +- tests/lcs/agents/xcs/test_ClassifierList.py | 17 +++- tests/lcs/agents/xcs/test_Condition.py | 3 +- tests/lcs/agents/xcs/test_Configuration.py | 2 - tests/lcs/agents/xcs/test_XCS.py | 4 +- 6 files changed, 73 insertions(+), 52 deletions(-) diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index ead1bf2b..e80659ce 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -10,29 +10,29 @@ def run_ga(population: ClassifiersList, situation, time_stamp, cfg: Configuration): + if action_set is None: return - # temp_numerosity = sum(cl.numerosity for cl in action_set) - # if temp_numerosity == 0: - # return + assert isinstance(population, ClassifiersList) + assert isinstance(action_set, ClassifiersList) + assert isinstance(cfg, Configuration) if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) - # / temp_numerosity) > cfg.ga_threshold: / (sum(cl.numerosity for cl in action_set) or 1)) > cfg.ga_threshold: for cl in action_set: cl.time_stamp = time_stamp - # select children + parent1 = _select_offspring(action_set) parent2 = _select_offspring(action_set) - child1, child2 = _make_children(parent1, parent2) - # apply crossover + child1, child2 = _make_children(parent1, parent2, cfg, time_stamp) + if np.random.rand() < cfg.chi: _apply_crossover(child1, child2, parent1, parent2) - # apply mutation on both children + _apply_mutation(child1, cfg, situation) _apply_mutation(child2, cfg, situation) - # apply subsumption or just insert into population + _perform_insertion_or_subsumption(cfg, population, child1, child2, parent1, parent2) @@ -41,42 +41,41 @@ def run_ga(population: ClassifiersList, def _perform_insertion_or_subsumption(cfg: Configuration, population: ClassifiersList, child1: Classifier, child2: Classifier, parent1: Classifier, parent2: Classifier): - if child1 is None or child2 is None: - return + + assert isinstance(child1, Classifier) + assert isinstance(child2, Classifier) + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) + if cfg.do_GA_subsumption: - if parent1.does_subsume(child1): - parent1.numerosity += 1 - elif parent2.does_subsume(child1): - parent2.numerosity += 1 - else: - population.insert_in_population(child1) - population.delete_from_population() - - if parent1.does_subsume(child2): - parent1.numerosity += 1 - elif parent2.does_subsume(child2): - parent2.numerosity += 1 - else: - population.insert_in_population(child2) - population.delete_from_population() + for child in child1, child2: + if parent1.does_subsume(child): + parent1.numerosity += 1 + elif parent2.does_subsume(child): + parent2.numerosity += 1 + else: + population.insert_in_population(child) + population.delete_from_population() else: - population.insert_in_population(child1) - population.delete_from_population() - population.insert_in_population(child2) - population.delete_from_population() - - -def _make_children(parent1, parent2): - child1 = copy(parent1) - child2 = copy(parent2) - child1.numerosity = 1 - child2.numerosity = 1 - child1.experience = 0 - child2.experience = 0 + for child in child1, child2: + population.insert_in_population(child) + population.delete_from_population() + + +def _make_children(parent1, parent2, cfg, time_stamp): + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) + + child1 = Classifier(cfg, parent1.condition, parent1.action, time_stamp) + child2 = Classifier(cfg, parent2.condition, parent2.action, time_stamp) + return child1, child2 def _select_offspring(action_set: ClassifiersList) -> Classifier: + + assert isinstance(action_set, ClassifiersList) + fitness_sum = 0 for cl in action_set: fitness_sum += cl.fitness @@ -91,16 +90,17 @@ def _select_offspring(action_set: ClassifiersList) -> Classifier: def _apply_crossover(child1: Classifier, child2: Classifier, parent1: Classifier, parent2: Classifier): - _apply_crossover_in_area(child1, child2, + + _apply_crossover_in_area(child1, + child2, np.random.rand() * len(child1.condition), np.random.rand() * len(child1.condition) ) - child1.prediction = (parent1.prediction + parent2.prediction) / 2 - child1.error = 0.25 * (parent1.error + parent2.error) / 2 - child1.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 - child2.prediction = child1.prediction - child2.error = child1.error - child2.fitness = child1.fitness + + for child in child1, child2: + child.prediction = (parent1.prediction + parent2.prediction) / 2 + child.error = 0.25 * (parent1.error + parent2.error) / 2 + child.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 def _apply_crossover_in_area(child1: Classifier, child2: Classifier, x, y): diff --git a/tests/lcs/agents/xcs/test_Classifier.py b/tests/lcs/agents/xcs/test_Classifier.py index 990abfed..fe904eff 100644 --- a/tests/lcs/agents/xcs/test_Classifier.py +++ b/tests/lcs/agents/xcs/test_Classifier.py @@ -88,7 +88,12 @@ def test_does_subsume(self, cfg: Configuration): ("##11", "1111", 1, 1, False), ("1111", "11", 1, 1, False), - ("11", "1111", 1, 1, False) + ("11", "1111", 1, 1, False), + + ("0##11#", "0##11#", 0, 0, True), + ("###000", "###000", 0, 0, True), + ("###00#", "###00#", 1, 1, True), + ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result): assert result == (Classifier(cfg=cfg, condition=Condition(cond1), action=act1, time_stamp=0) == diff --git a/tests/lcs/agents/xcs/test_ClassifierList.py b/tests/lcs/agents/xcs/test_ClassifierList.py index 414b28bb..bbd859a5 100644 --- a/tests/lcs/agents/xcs/test_ClassifierList.py +++ b/tests/lcs/agents/xcs/test_ClassifierList.py @@ -34,10 +34,25 @@ def test_init(self, cfg): ("1100", 3) ]) def test_insert_population(self, classifiers_list_diff_actions, cfg, cond, act): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) + cl = Classifier(cfg=cfg, condition=Condition(cond), action=act, time_stamp=0) classifiers_list_diff_actions.insert_in_population(cl) assert any(c == cl for c in classifiers_list_diff_actions) + @pytest.mark.parametrize("cond1, cond2, act1, act2, size", [ + ("1111", "1111", 1, 1, 5), + ("#100", "#100", 0, 0, 5), + ("0##11#", "0##11#", 0, 0, 5), + + ]) + def test_insert_population_two(self, cfg, classifiers_list_diff_actions, cond1, cond2, act1, act2, size): + cl1 = Classifier(cfg=cfg, condition=Condition(cond1), action=act1, time_stamp=0) + cl2 = Classifier(cfg=cfg, condition=Condition(cond2), action=act2, time_stamp=0) + classifiers_list_diff_actions.insert_in_population(cl1) + classifiers_list_diff_actions.insert_in_population(cl2) + assert any(c == cl1 for c in classifiers_list_diff_actions) + assert any(c == cl2 for c in classifiers_list_diff_actions) + assert len(classifiers_list_diff_actions) == size + @pytest.mark.parametrize("cond, act", [ ("1111", 0), ("123", 1), diff --git a/tests/lcs/agents/xcs/test_Condition.py b/tests/lcs/agents/xcs/test_Condition.py index f197d362..abd88dd5 100644 --- a/tests/lcs/agents/xcs/test_Condition.py +++ b/tests/lcs/agents/xcs/test_Condition.py @@ -25,7 +25,8 @@ def test_should_get_initialized_with_str(self): ("1111", "1100", False), ("1111", "11", False), - ("11", "1100", False) + ("#1111#", "#1111#", True), + ("###01#", "###01#", True), ]) def test_equal(self, cond1, cond2, result): assert result == (Condition(cond1) == Condition(cond2)) diff --git a/tests/lcs/agents/xcs/test_Configuration.py b/tests/lcs/agents/xcs/test_Configuration.py index 0ce95791..0ce47a52 100644 --- a/tests/lcs/agents/xcs/test_Configuration.py +++ b/tests/lcs/agents/xcs/test_Configuration.py @@ -12,7 +12,5 @@ class TestConfiguration: def cfg(self): return Configuration(number_of_actions=4) - def test_minimum(self, cfg): - assert float(np.finfo(np.float32).tiny) == cfg.initial_prediction diff --git a/tests/lcs/agents/xcs/test_XCS.py b/tests/lcs/agents/xcs/test_XCS.py index b4519d0f..e8daeee7 100644 --- a/tests/lcs/agents/xcs/test_XCS.py +++ b/tests/lcs/agents/xcs/test_XCS.py @@ -132,7 +132,9 @@ def test_simple_q_learning(self, cfg, classifiers_list_diff_actions): def test_make_children(self, cfg, classifiers_list_diff_actions): child1, child2 = GeneticAlgorithm._make_children( classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] + classifiers_list_diff_actions[1], + cfg, + 0 ) assert child1.numerosity == 1 assert child2.numerosity == 1 From d849105e7e59771e6f6eb681acc67f7abf2f4dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Koz=C5=82owski?= Date: Fri, 7 May 2021 08:32:28 +0200 Subject: [PATCH 02/33] Add some more tests --- lcs/agents/xcs/Classifier.py | 19 +++++----- lcs/agents/xcs/GeneticAlgorithm.py | 4 +-- lcs/agents/xcs/XCS.py | 9 ++--- tests/lcs/agents/xcs/test_Classifier.py | 7 ++-- tests/lcs/agents/xcs/test_ClassifierList.py | 40 +++++++++++++++------ tests/lcs/agents/xcs/test_Condition.py | 8 +++++ tests/lcs/agents/xncs/test_Classifier.py | 2 +- 7 files changed, 58 insertions(+), 31 deletions(-) diff --git a/lcs/agents/xcs/Classifier.py b/lcs/agents/xcs/Classifier.py index 4b6031ec..332dc17c 100644 --- a/lcs/agents/xcs/Classifier.py +++ b/lcs/agents/xcs/Classifier.py @@ -12,10 +12,13 @@ def __init__(self, condition: Union[Condition, str, None] = None, action: Optional[int] = None, time_stamp: int = None) -> None: + if cfg is None: raise TypeError("Configuration should be passed to Classifier") + if type(condition) != Condition: - condition = str(condition) + condition = Condition(condition) + self.cfg = cfg # cfg self.condition = condition # current situation self.action = action # A - int action @@ -42,13 +45,12 @@ def does_subsume(self, other): @property def could_subsume(self): - if self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error: - return True - return False + return self.experience > self.cfg.subsumption_threshold and self.error < self.cfg.initial_error def is_more_general(self, other): if self.wildcard_number <= other.wildcard_number: return False + return self.condition.is_more_general(other.condition) @property @@ -62,7 +64,8 @@ def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - Num:{self.numerosity} " + \ f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]" - def __eq__(self, other): - if other.action == self.action and other.condition == self.condition: - return True - return False + def __eq__(self, o): + return o.condition == self.condition and o.action == self.action + + def __hash__(self): + return hash((str(self.condition), self.action)) diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index e80659ce..20cc0a2f 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -1,6 +1,6 @@ -import numpy as np import random -from copy import copy + +import numpy as np from lcs.agents.xcs import Configuration, Classifier, ClassifiersList diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index 925e281a..a1ea363b 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -1,15 +1,14 @@ import logging import random -import numpy as np from copy import copy from typing import Optional +import numpy as np + from lcs.agents import Agent -from lcs.agents.xcs import Configuration, ClassifiersList, GeneticAlgorithm from lcs.agents.Agent import TrialMetrics +from lcs.agents.xcs import Configuration, ClassifiersList, GeneticAlgorithm from lcs.strategies.reinforcement_learning import simple_q_learning -from lcs.strategies.action_selection import EpsilonGreedy - logger = logging.getLogger(__name__) @@ -121,5 +120,3 @@ def do_action_set_subsumption(self, action_set: ClassifiersList) -> None: cl.numerosity += c.numerosity action_set.safe_remove(c) self.population.safe_remove(c) - - diff --git a/tests/lcs/agents/xcs/test_Classifier.py b/tests/lcs/agents/xcs/test_Classifier.py index fe904eff..89bd0e6f 100644 --- a/tests/lcs/agents/xcs/test_Classifier.py +++ b/tests/lcs/agents/xcs/test_Classifier.py @@ -93,10 +93,11 @@ def test_does_subsume(self, cfg: Configuration): ("0##11#", "0##11#", 0, 0, True), ("###000", "###000", 0, 0, True), ("###00#", "###00#", 1, 1, True), - ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result): - assert result == (Classifier(cfg=cfg, condition=Condition(cond1), action=act1, time_stamp=0) == - Classifier(cfg=cfg, condition=Condition(cond2), action=act2, time_stamp=0)) + cl1 = Classifier(condition=Condition(cond1), action=act1, time_stamp=0, cfg=cfg) + cl2 = Classifier(condition=Condition(cond2), action=act2, time_stamp=0, cfg=cfg) + + assert result is (cl1 == cl2) diff --git a/tests/lcs/agents/xcs/test_ClassifierList.py b/tests/lcs/agents/xcs/test_ClassifierList.py index bbd859a5..d8da05e6 100644 --- a/tests/lcs/agents/xcs/test_ClassifierList.py +++ b/tests/lcs/agents/xcs/test_ClassifierList.py @@ -17,14 +17,14 @@ def situation(self): @pytest.fixture def classifiers_list_diff_actions(self, cfg, situation): - classifiers_list = ClassifiersList(cfg) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) - return classifiers_list - - def test_init(self, cfg): + pop = ClassifiersList(cfg) + pop.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) + pop.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + return pop + + def test_should_be_empty_when_initialized(self, cfg): assert len(ClassifiersList(cfg)) == 0 @pytest.mark.parametrize("cond, act", [ @@ -34,9 +34,17 @@ def test_init(self, cfg): ("1100", 3) ]) def test_insert_population(self, classifiers_list_diff_actions, cfg, cond, act): + # given + initial_size = len(classifiers_list_diff_actions) cl = Classifier(cfg=cfg, condition=Condition(cond), action=act, time_stamp=0) + + # when classifiers_list_diff_actions.insert_in_population(cl) + + # then + assert len(classifiers_list_diff_actions) == initial_size assert any(c == cl for c in classifiers_list_diff_actions) + assert any(c.numerosity == 2 for c in classifiers_list_diff_actions) @pytest.mark.parametrize("cond1, cond2, act1, act2, size", [ ("1111", "1111", 1, 1, 5), @@ -45,10 +53,15 @@ def test_insert_population(self, classifiers_list_diff_actions, cfg, cond, act): ]) def test_insert_population_two(self, cfg, classifiers_list_diff_actions, cond1, cond2, act1, act2, size): - cl1 = Classifier(cfg=cfg, condition=Condition(cond1), action=act1, time_stamp=0) - cl2 = Classifier(cfg=cfg, condition=Condition(cond2), action=act2, time_stamp=0) + # given + cl1 = Classifier(condition=Condition(cond1), action=act1, cfg=cfg) + cl2 = Classifier(condition=Condition(cond2), action=act2, cfg=cfg) + + # when classifiers_list_diff_actions.insert_in_population(cl1) classifiers_list_diff_actions.insert_in_population(cl2) + + # then assert any(c == cl1 for c in classifiers_list_diff_actions) assert any(c == cl2 for c in classifiers_list_diff_actions) assert len(classifiers_list_diff_actions) == size @@ -62,8 +75,13 @@ def test_insert_population_two(self, cfg, classifiers_list_diff_actions, cond1, ("112", 0), ]) def test_insert_population_new_condition(self, classifiers_list_diff_actions, cfg, cond, act): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) + # given + cl = Classifier(condition=Condition("1111"), action=0, cfg=cfg) + + # when classifiers_list_diff_actions.insert_in_population(cl) + + # then for c in classifiers_list_diff_actions: assert c.numerosity == 1 assert classifiers_list_diff_actions[4] == cl diff --git a/tests/lcs/agents/xcs/test_Condition.py b/tests/lcs/agents/xcs/test_Condition.py index abd88dd5..ebffa2c2 100644 --- a/tests/lcs/agents/xcs/test_Condition.py +++ b/tests/lcs/agents/xcs/test_Condition.py @@ -67,3 +67,11 @@ def test_number_of_wildcards(self, cond, num): ]) def test_is_more_general(self, cond1, cond2, result): assert Condition(cond1).is_more_general(Condition(cond2)) == result + + @pytest.mark.parametrize("_c", [ + ([0.1, 0.2]), + ([0, 1]), + ]) + def test_should_fail_with_invalid_types(self, _c): + with pytest.raises(AssertionError) as _: + Condition(_c) diff --git a/tests/lcs/agents/xncs/test_Classifier.py b/tests/lcs/agents/xncs/test_Classifier.py index 91fddf16..4ee91f6f 100644 --- a/tests/lcs/agents/xncs/test_Classifier.py +++ b/tests/lcs/agents/xncs/test_Classifier.py @@ -11,7 +11,7 @@ def cfg(self): return Configuration(lmc=2, lem=0.2, number_of_actions=4) def test_init(self, cfg): - cl = Classifier(cfg) + cl = Classifier(condition='####', cfg=cfg) assert cl.cfg == cfg assert cl.effect is None From 7541ee2177e8ad7f7e31b9dc741b0bd547a81ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Koz=C5=82owski?= Date: Fri, 7 May 2021 08:49:14 +0200 Subject: [PATCH 03/33] Ensure no duplicates --- lcs/agents/xcs/ClassifiersList.py | 46 +++++++++++++++++-------------- lcs/agents/xcs/XCS.py | 3 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index d42bb814..1024cee0 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -1,6 +1,7 @@ -import numpy as np -import random import logging +import random + +import numpy as np from lcs import TypedList, Perception from lcs.agents.xcs import Classifier, Condition, Configuration @@ -18,11 +19,12 @@ def __init__(self, super().__init__(*args, oktypes=oktypes) def insert_in_population(self, cl: Classifier): - for c in self: - if c == cl: - c.numerosity += 1 - return - self.append(cl) + existing_classifiers = [c for c in self if c == cl] + if len(existing_classifiers) > 0: + assert len(existing_classifiers) == 1, 'duplicates found' + existing_classifiers[0].numerosity += 1 + else: + self.append(cl) def generate_covering_classifier(self, situation, action, time_stamp): # both Perception and string has __getitem__ @@ -33,11 +35,11 @@ def generate_covering_classifier(self, situation, action, time_stamp): generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) - cl = Classifier(cfg=self.cfg, - condition=Condition(generalized), - action=action, - time_stamp=time_stamp) - return cl + + return Classifier(condition=Condition(generalized), + action=action, + time_stamp=time_stamp, + cfg=self.cfg) def _generate_covering_and_insert(self, situation, action, time_stamp): cl = self.generate_covering_classifier(situation, action, time_stamp) @@ -59,8 +61,8 @@ def delete_from_population(self): def _deletion_vote(self, cl, average_fitness): vote = cl.action_set_size * cl.numerosity if cl.experience > self.cfg.deletion_threshold and \ - cl.fitness / cl.numerosity < \ - self.cfg.delta * average_fitness: + cl.fitness / cl.numerosity < \ + self.cfg.delta * average_fitness: vote *= average_fitness / (cl.fitness / cl.numerosity) return vote @@ -78,7 +80,8 @@ def generate_match_set(self, situation: Perception, time_stamp): matching_ls = [cl for cl in self if cl.does_match(situation)] while len(matching_ls) < self.cfg.number_of_actions: action = self._find_not_present_action(matching_ls) - cl = self._generate_covering_and_insert(situation, action, time_stamp) + cl = self._generate_covering_and_insert(situation, action, + time_stamp) matching_ls.append(cl) return ClassifiersList(self.cfg, *matching_ls) @@ -116,16 +119,19 @@ def update_set(self, p): for cl in self: cl.experience += 1 # update prediction, prediction error, action set size estimate - if cl.experience < 1/self.cfg.learning_rate: + if cl.experience < 1 / self.cfg.learning_rate: cl.prediction += (p - cl.prediction) / cl.experience cl.error += (abs(p - cl.prediction) - cl.error) / cl.experience - cl.action_set_size +=\ - (action_set_numerosity - cl.action_set_size) / cl.experience + cl.action_set_size += \ + ( + action_set_numerosity - cl.action_set_size) / cl.experience else: cl.prediction += self.cfg.learning_rate * (p - cl.prediction) - cl.error += self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error) + cl.error += self.cfg.learning_rate * ( + abs(p - cl.prediction) - cl.error) cl.action_set_size += \ - self.cfg.learning_rate * (action_set_numerosity - cl.action_set_size) + self.cfg.learning_rate * ( + action_set_numerosity - cl.action_set_size) self._update_fitness() def _update_fitness(self): diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index a1ea363b..d4756c20 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -56,6 +56,7 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: state = self.cfg.environment_adapter.to_genotype(raw_state) while not done: + assert len(self.population) == len(set(self.population)), 'duplicates found' self.population.delete_from_population() # We are in t+1 here match_set = self.population.generate_match_set(state, self.time_stamp) @@ -97,7 +98,7 @@ def _distribute_and_update(self, action_set, situation, p): self.time_stamp, self.cfg) - # TODO: EspilonGreedy + # TODO: EpsilonGreedy # Run into a lot of issues where in EpsilonGreedy where BestAction was not callable # Changing EpsilonGreed to: # best = BestAction(all_actions=self.all_actions) From 3624335819b90801a7eb786f77037c01f90f53cb Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 7 May 2021 13:28:08 +0200 Subject: [PATCH 04/33] Covering Fix Fixed issue where generation of covering classifiers were not like it was in Algorithmic description of XCS. --- lcs/agents/xcs/Classifier.py | 1 - lcs/agents/xcs/ClassifiersList.py | 29 +++++++++++++++-------------- lcs/agents/xcs/GeneticAlgorithm.py | 2 ++ tests/lcs/agents/xcs/__init__.py | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lcs/agents/xcs/Classifier.py b/lcs/agents/xcs/Classifier.py index 332dc17c..f5dae0e1 100644 --- a/lcs/agents/xcs/Classifier.py +++ b/lcs/agents/xcs/Classifier.py @@ -50,7 +50,6 @@ def could_subsume(self): def is_more_general(self, other): if self.wildcard_number <= other.wildcard_number: return False - return self.condition.is_more_general(other.condition) @property diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 1024cee0..186c3dff 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -21,7 +21,7 @@ def __init__(self, def insert_in_population(self, cl: Classifier): existing_classifiers = [c for c in self if c == cl] if len(existing_classifiers) > 0: - assert len(existing_classifiers) == 1, 'duplicates found' + # assert len(existing_classifiers) == 1, 'duplicates found' existing_classifiers[0].numerosity += 1 else: self.append(cl) @@ -78,11 +78,11 @@ def _remove_based_on_votes(self, deletion_votes, selector): def generate_match_set(self, situation: Perception, time_stamp): matching_ls = [cl for cl in self if cl.does_match(situation)] - while len(matching_ls) < self.cfg.number_of_actions: - action = self._find_not_present_action(matching_ls) - cl = self._generate_covering_and_insert(situation, action, - time_stamp) + action = self._find_not_present_action(matching_ls) + while action is not None: + cl = self._generate_covering_and_insert(situation, action, time_stamp) matching_ls.append(cl) + action = self._find_not_present_action(matching_ls) return ClassifiersList(self.cfg, *matching_ls) def _find_not_present_action(self, matching_set): @@ -120,18 +120,19 @@ def update_set(self, p): cl.experience += 1 # update prediction, prediction error, action set size estimate if cl.experience < 1 / self.cfg.learning_rate: - cl.prediction += (p - cl.prediction) / cl.experience - cl.error += (abs(p - cl.prediction) - cl.error) / cl.experience + cl.prediction += \ + (p - cl.prediction) / cl.experience + cl.error += \ + (abs(p - cl.prediction) - cl.error) / cl.experience cl.action_set_size += \ - ( - action_set_numerosity - cl.action_set_size) / cl.experience + (action_set_numerosity - cl.action_set_size) / cl.experience else: - cl.prediction += self.cfg.learning_rate * (p - cl.prediction) - cl.error += self.cfg.learning_rate * ( - abs(p - cl.prediction) - cl.error) + cl.prediction +=\ + self.cfg.learning_rate * (p - cl.prediction) + cl.error += \ + self.cfg.learning_rate * (abs(p - cl.prediction) - cl.error) cl.action_set_size += \ - self.cfg.learning_rate * ( - action_set_numerosity - cl.action_set_size) + self.cfg.learning_rate * (action_set_numerosity - cl.action_set_size) self._update_fitness() def _update_fitness(self): diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index 20cc0a2f..35ba3e58 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -25,6 +25,7 @@ def run_ga(population: ClassifiersList, parent1 = _select_offspring(action_set) parent2 = _select_offspring(action_set) + child1, child2 = _make_children(parent1, parent2, cfg, time_stamp) if np.random.rand() < cfg.chi: @@ -76,6 +77,7 @@ def _select_offspring(action_set: ClassifiersList) -> Classifier: assert isinstance(action_set, ClassifiersList) + # TODO: insert generator to calculate fitness_sum fitness_sum = 0 for cl in action_set: fitness_sum += cl.fitness diff --git a/tests/lcs/agents/xcs/__init__.py b/tests/lcs/agents/xcs/__init__.py index e8ed99a7..586e99c0 100644 --- a/tests/lcs/agents/xcs/__init__.py +++ b/tests/lcs/agents/xcs/__init__.py @@ -1 +1 @@ -# py.test --cov=xcs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xncs +# py.test --cov=xcs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xcs From b3996961d0581815f313b477a3d0065d88557263 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 7 May 2021 14:29:58 +0200 Subject: [PATCH 05/33] Better RP Fixed but not fully realized the Reinforcment Program for XCS. --- lcs/agents/xcs/ClassifiersList.py | 4 ++-- lcs/agents/xcs/Configuration.py | 5 ++++- lcs/agents/xcs/XCS.py | 21 ++++++++------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 186c3dff..5e52ccd5 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -21,7 +21,7 @@ def __init__(self, def insert_in_population(self, cl: Classifier): existing_classifiers = [c for c in self if c == cl] if len(existing_classifiers) > 0: - # assert len(existing_classifiers) == 1, 'duplicates found' + assert len(existing_classifiers) == 1, 'duplicates found' existing_classifiers[0].numerosity += 1 else: self.append(cl) @@ -62,7 +62,7 @@ def _deletion_vote(self, cl, average_fitness): vote = cl.action_set_size * cl.numerosity if cl.experience > self.cfg.deletion_threshold and \ cl.fitness / cl.numerosity < \ - self.cfg.delta * average_fitness: + self.cfg.delta * average_fitness: vote *= average_fitness / (cl.fitness / cl.numerosity) return vote diff --git a/lcs/agents/xcs/Configuration.py b/lcs/agents/xcs/Configuration.py index 36e41534..9bfbdb72 100644 --- a/lcs/agents/xcs/Configuration.py +++ b/lcs/agents/xcs/Configuration.py @@ -29,7 +29,8 @@ def __init__(self, do_ga_subsumption: bool = False, do_action_set_subsumption: bool = False, metrics_trial_frequency: int = 5, - user_metrics_collector_fcn: Callable = None + user_metrics_collector_fcn: Callable = None, + multistep_enfiroment: bool = True ) -> None: """ :param classifier_wildcard: Wildcard symbol @@ -80,6 +81,8 @@ def __init__(self, self.metrics_trial_frequency = metrics_trial_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn + self.multistep_enfiroment = multistep_enfiroment + def __str__(self) -> str: return str(vars(self)) diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index d4756c20..9b3cfcee 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -28,7 +28,7 @@ def __init__(self, else: self.population = ClassifiersList(cfg=cfg) self.time_stamp = 0 - self.action_reward = [0 for _ in range(cfg.number_of_actions)] + self.reward = 0 def get_population(self): return self.population @@ -46,9 +46,8 @@ def _run_trial_exploit(self, env, trials, current_trial) -> TrialMetrics: def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: prev_action_set = None - prev_reward = [0 for _ in range(self.cfg.number_of_actions)] + prev_reward = self.reward prev_state = None # state is known as situation - prev_action = 0 prev_time_stamp = self.time_stamp # steps done = False # eop @@ -66,26 +65,22 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: # apply action to environment raw_state, step_reward, done, _ = env.step(action) state = self.cfg.environment_adapter.to_genotype(raw_state) - self.action_reward[action] = simple_q_learning(self.action_reward[action], - step_reward, - self.cfg.learning_rate, - self.cfg.gamma, - match_set.best_prediction) + if self.cfg.multistep_enfiroment: + self.reward = step_reward + self.cfg.gamma * self.reward self._distribute_and_update(prev_action_set, prev_state, - prev_reward[prev_action] + self.cfg.gamma * max(prediction_array)) + prev_reward + self.cfg.gamma * max(prediction_array)) if done: self._distribute_and_update(action_set, state, - self.action_reward[action]) + self.reward) else: prev_action_set = copy(action_set) - prev_reward[action] = copy(self.action_reward[action]) + prev_reward = self.reward prev_state = copy(state) - prev_action = action self.time_stamp += 1 - return TrialMetrics(self.time_stamp - prev_time_stamp, self.action_reward) + return TrialMetrics(self.time_stamp - prev_time_stamp, self.reward) def _distribute_and_update(self, action_set, situation, p): if action_set is not None and len(action_set) > 0: From da8e03b55d2fe87ff28560fd2c84fa869f1fc211 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 13 May 2021 11:04:17 +0200 Subject: [PATCH 06/33] XNCS Update --- lcs/agents/xcs/GeneticAlgorithm.py | 9 ++- lcs/agents/xncs/Configuration.py | 68 +++++++++++---------- lcs/agents/xncs/XNCS.py | 47 +------------- tests/lcs/agents/xcs/test_ClassifierList.py | 3 +- tests/lcs/agents/xncs/test_XNCS.py | 9 +++ 5 files changed, 55 insertions(+), 81 deletions(-) diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index 35ba3e58..9e655cbe 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -1,5 +1,5 @@ import random - +from copy import copy import numpy as np from lcs.agents.xcs import Configuration, Classifier, ClassifiersList @@ -67,8 +67,11 @@ def _make_children(parent1, parent2, cfg, time_stamp): assert isinstance(parent1, Classifier) assert isinstance(parent2, Classifier) - child1 = Classifier(cfg, parent1.condition, parent1.action, time_stamp) - child2 = Classifier(cfg, parent2.condition, parent2.action, time_stamp) + child1 = copy(parent1) + child1.time_stamp = time_stamp + + child2 = copy(parent2) + child2.time_stamp = time_stamp return child1, child2 diff --git a/lcs/agents/xncs/Configuration.py b/lcs/agents/xncs/Configuration.py index 26c14f8f..c16a0163 100644 --- a/lcs/agents/xncs/Configuration.py +++ b/lcs/agents/xncs/Configuration.py @@ -8,16 +8,18 @@ class Configuration(xcs.Configuration): def __init__(self, - number_of_actions: int, # theta_mna it is actually smart to make it equal to number of actions + number_of_actions: int, lmc: int = 100, lem: float = 1, + + # theta_mna it is actually smart to make it equal to number of actions classifier_wildcard: str = '#', environment_adapter=EnvironmentAdapter, max_population: int = 200, # n learning_rate: float = 0.1, # beta alpha: float = 0.1, epsilon_0: float = 10, - v: int = 5, + v: int = 5, # nu gamma: float = 0.71, ga_threshold: int = 25, chi: float = 0.5, @@ -26,41 +28,43 @@ def __init__(self, delta: float = 0.1, subsumption_threshold: int = 20, # theta_sub covering_wildcard_chance: float = 0.33, # population wildcard - initial_prediction: float = float(np.finfo(np.float32).tiny), # p_i - initial_error: float = float(np.finfo(np.float32).tiny), # epsilon_i - initial_fitness: float = float(np.finfo(np.float32).tiny), # f_i + initial_prediction: float = 0.000001, # p_i + initial_error: float = 0.000001, # epsilon_i + initial_fitness: float = 0.000001, # f_i epsilon: float = 0.5, # p_exp, exploration probability do_ga_subsumption: bool = False, do_action_set_subsumption: bool = False, metrics_trial_frequency: int = 5, - user_metrics_collector_fcn: Callable = None + user_metrics_collector_fcn: Callable = None, + multistep_enfiroment: bool = True ) -> None: self.lmc = lmc self.lem = lem - self.classifier_wildcard = classifier_wildcard - self.environment_adapter = environment_adapter - self.max_population = max_population - self.learning_rate = learning_rate - self.alpha = alpha - self.epsilon_0 = epsilon_0 - self.v = v - self.gamma = gamma - self.ga_threshold = ga_threshold - self.chi = chi - self.mutation_chance = mutation_chance - self.deletion_threshold = deletion_threshold - self.delta = delta - self.subsumption_threshold = subsumption_threshold - self.covering_wildcard_chance = covering_wildcard_chance - self.initial_prediction = initial_prediction - self.initial_error = initial_error - self.initial_fitness = initial_fitness - self.epsilon = epsilon # p_exp, probability of exploration - self.number_of_actions = number_of_actions - self.do_GA_subsumption = do_ga_subsumption - self.do_action_set_subsumption = do_action_set_subsumption - - self.metrics_trial_frequency = metrics_trial_frequency - self.user_metrics_collector_fcn = user_metrics_collector_fcn - + super().__init__( + number_of_actions=number_of_actions, + classifier_wildcard=classifier_wildcard, + environment_adapter=environment_adapter, + max_population=max_population, # n + learning_rate=learning_rate, # beta + alpha=alpha, + epsilon_0=epsilon_0, + v=v, # nu + gamma=gamma, + ga_threshold=ga_threshold, + chi=chi, + mutation_chance=mutation_chance, # mu + deletion_threshold=deletion_threshold, # theta_del + delta=delta, + subsumption_threshold=subsumption_threshold, # theta_sub + covering_wildcard_chance=covering_wildcard_chance, # population wildcard + initial_prediction=initial_prediction, # p_i + initial_error=initial_error, # epsilon_i + initial_fitness=initial_fitness, # f_i + epsilon=epsilon, # p_exp, exploration probability + do_ga_subsumption=do_ga_subsumption, + do_action_set_subsumption=do_action_set_subsumption, + metrics_trial_frequency=metrics_trial_frequency, + user_metrics_collector_fcn=user_metrics_collector_fcn, + multistep_enfiroment=multistep_enfiroment + ) diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 7a990799..f281eb27 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -22,56 +22,13 @@ def __init__(self, :param population: all classifiers at current time """ self.back_propagation = Backpropagation(cfg) - self.cfg = cfg if population is not None: self.population = population else: self.population = ClassifiersList(cfg=cfg) + self.cfg = cfg self.time_stamp = 0 - self.action_reward = [0 for _ in range(cfg.number_of_actions)] - - def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: - prev_action_set = None - prev_reward = [0 for _ in range(self.cfg.number_of_actions)] - prev_state = None # state is known as situation - prev_action = 0 - self.time_stamp = 0 # steps - done = False # eop - - raw_state = env.reset() - state = self.cfg.environment_adapter.to_genotype(raw_state) - - while not done: - self.population.delete_from_population() - # We are in t+1 here - match_set = self.population.generate_match_set(state, self.time_stamp) - prediction_array = match_set.prediction_array - action = self.select_action(prediction_array, match_set) - action_set = match_set.generate_action_set(action) - # apply action to environment - raw_state, step_reward, done, _ = env.step(action) - state = self.cfg.environment_adapter.to_genotype(raw_state) - self.action_reward[action] = simple_q_learning(self.action_reward[action], - step_reward, - self.cfg.learning_rate, - self.cfg.gamma, - match_set.best_prediction) - - self._distribute_and_update(prev_action_set, - prev_state, - prev_reward[prev_action] + self.cfg.gamma * max(prediction_array)) - if done: - self._distribute_and_update(action_set, - state, - self.action_reward[action]) - else: - prev_action_set = copy(action_set) - prev_reward[action] = copy(self.action_reward[action]) - prev_state = copy(state) - prev_action = action - - self.time_stamp += 1 - return TrialMetrics(self.time_stamp, self.action_reward) + self.reward = 0 def _distribute_and_update(self, action_set, situation, p): super()._distribute_and_update(action_set, situation, p) diff --git a/tests/lcs/agents/xcs/test_ClassifierList.py b/tests/lcs/agents/xcs/test_ClassifierList.py index d8da05e6..bb2fde9d 100644 --- a/tests/lcs/agents/xcs/test_ClassifierList.py +++ b/tests/lcs/agents/xcs/test_ClassifierList.py @@ -141,7 +141,8 @@ def test_prediction_array(self, cfg, classifiers_list_diff_actions): classifiers_list_diff_actions[0].prediction = 10 prediction_array = classifiers_list_diff_actions.prediction_array assert len(classifiers_list_diff_actions) == cfg.number_of_actions - assert prediction_array[0] > prediction_array[1] + assert all(prediction_array[0] >= prediction + for prediction in prediction_array) def test_update_fitness(self, cfg, classifiers_list_diff_actions): classifiers_list_diff_actions._update_fitness() diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index d33a388e..1716df79 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -36,3 +36,12 @@ def test_distribute_and_update(self, cfg, classifiers_list_diff_actions): xncs = XNCS(cfg, classifiers_list_diff_actions) xncs._distribute_and_update(classifiers_list_diff_actions, "1100", 0.1) assert len(xncs.back_propagation.update_vectors) == 4 + + def test_correct_types(self,cfg): + xncs = XNCS(cfg) + assert isinstance(xncs.population, ClassifiersList) + xncs.population.insert_in_population( + Classifier(cfg, Condition("1100"), 0, 0) + ) + assert isinstance(xncs.population[0], Classifier) + assert xncs.population[0].effect is None From 69e1f9352714365b7af3b702596da130d1ff0871 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 18 May 2021 12:39:22 +0200 Subject: [PATCH 07/33] Fix Duplicates XCS --- XCS_script.py | 1 + lcs/agents/xcs/ClassifiersList.py | 2 +- lcs/agents/xcs/GeneticAlgorithm.py | 224 +++++++++++++++------------- lcs/agents/xcs/XCS.py | 19 ++- lcs/agents/xcs/__init__.py | 3 +- lcs/agents/xncs/Classifier.py | 3 + lcs/agents/xncs/GeneticAlgorithm.py | 36 +++++ lcs/agents/xncs/XNCS.py | 10 +- lcs/agents/xncs/__init__.py | 3 +- tests/lcs/agents/xcs/test_GA.py | 133 +++++++++++++++++ tests/lcs/agents/xcs/test_XCS.py | 104 +------------ 11 files changed, 314 insertions(+), 224 deletions(-) create mode 100644 XCS_script.py create mode 100644 lcs/agents/xncs/GeneticAlgorithm.py create mode 100644 tests/lcs/agents/xcs/test_GA.py diff --git a/XCS_script.py b/XCS_script.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/XCS_script.py @@ -0,0 +1 @@ + diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 5e52ccd5..50dcf50d 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -21,7 +21,7 @@ def __init__(self, def insert_in_population(self, cl: Classifier): existing_classifiers = [c for c in self if c == cl] if len(existing_classifiers) > 0: - assert len(existing_classifiers) == 1, 'duplicates found' + # assert len(existing_classifiers) == 1, 'duplicates found, while inserting' existing_classifiers[0].numerosity += 1 else: self.append(cl) diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index 9e655cbe..dd15809c 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -5,135 +5,145 @@ from lcs.agents.xcs import Configuration, Classifier, ClassifiersList -def run_ga(population: ClassifiersList, - action_set: ClassifiersList, - situation, - time_stamp, - cfg: Configuration): +class GeneticAlgorithm: - if action_set is None: - return + def __init__( + self, + population: ClassifiersList, + cfg: Configuration + ): - assert isinstance(population, ClassifiersList) - assert isinstance(action_set, ClassifiersList) - assert isinstance(cfg, Configuration) + self.population = population + self.cfg = cfg - if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) - / (sum(cl.numerosity for cl in action_set) or 1)) > cfg.ga_threshold: - for cl in action_set: - cl.time_stamp = time_stamp - - parent1 = _select_offspring(action_set) - parent2 = _select_offspring(action_set) - - child1, child2 = _make_children(parent1, parent2, cfg, time_stamp) + def run_ga(self, + action_set: ClassifiersList, + situation, + time_stamp): - if np.random.rand() < cfg.chi: - _apply_crossover(child1, child2, parent1, parent2) + if action_set is None: + return - _apply_mutation(child1, cfg, situation) - _apply_mutation(child2, cfg, situation) + # sometimes action set is empty, which is expected + assert isinstance(action_set, ClassifiersList) - _perform_insertion_or_subsumption(cfg, population, - child1, child2, - parent1, parent2) + if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) + / (sum(cl.numerosity for cl in action_set) or 1)) > self.cfg.ga_threshold: + for cl in action_set: + cl.time_stamp = time_stamp + parent1 = self._select_offspring(action_set) + parent2 = self._select_offspring(action_set) -def _perform_insertion_or_subsumption(cfg: Configuration, population: ClassifiersList, - child1: Classifier, child2: Classifier, - parent1: Classifier, parent2: Classifier): + child1, child2 = self._make_children(parent1, parent2, time_stamp) - assert isinstance(child1, Classifier) - assert isinstance(child2, Classifier) - assert isinstance(parent1, Classifier) - assert isinstance(parent2, Classifier) - - if cfg.do_GA_subsumption: - for child in child1, child2: - if parent1.does_subsume(child): - parent1.numerosity += 1 - elif parent2.does_subsume(child): - parent2.numerosity += 1 - else: - population.insert_in_population(child) - population.delete_from_population() - else: - for child in child1, child2: - population.insert_in_population(child) - population.delete_from_population() + if np.random.rand() < self.cfg.chi: + self._apply_crossover(child1, child2, parent1, parent2) + self._apply_mutation(child1, self.cfg, situation) + self._apply_mutation(child2, self.cfg, situation) -def _make_children(parent1, parent2, cfg, time_stamp): - assert isinstance(parent1, Classifier) - assert isinstance(parent2, Classifier) + self._perform_insertion_or_subsumption( + child1, child2, + parent1, parent2 + ) - child1 = copy(parent1) - child1.time_stamp = time_stamp + def _perform_insertion_or_subsumption(self, + child1: Classifier, child2: Classifier, + parent1: Classifier, parent2: Classifier): - child2 = copy(parent2) - child2.time_stamp = time_stamp + assert isinstance(child1, Classifier) + assert isinstance(child2, Classifier) + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) - return child1, child2 + if self.cfg.do_GA_subsumption: + for child in child1, child2: + if parent1.does_subsume(child): + parent1.numerosity += 1 + elif parent2.does_subsume(child): + parent2.numerosity += 1 + else: + self.population.insert_in_population(child) + self.population.delete_from_population() + else: + for child in child1, child2: + self.population.insert_in_population(child) + self.population.delete_from_population() + def _make_children(self, parent1, parent2, time_stamp): + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) -def _select_offspring(action_set: ClassifiersList) -> Classifier: + child1 = copy(parent1) + child1.condition = copy(parent1.condition) + child1.time_stamp = time_stamp - assert isinstance(action_set, ClassifiersList) + child2 = copy(parent2) + child2.condition = copy(parent2.condition) + child2.time_stamp = time_stamp - # TODO: insert generator to calculate fitness_sum - fitness_sum = 0 - for cl in action_set: - fitness_sum += cl.fitness - choice_point = np.random.rand() * fitness_sum - fitness_sum = 0 - for cl in action_set: - fitness_sum += cl.fitness - if fitness_sum > choice_point: - return cl - return action_set[random.randrange(len(action_set))] + return child1, child2 + @staticmethod + def _select_offspring(action_set: ClassifiersList) -> Classifier: -def _apply_crossover(child1: Classifier, child2: Classifier, - parent1: Classifier, parent2: Classifier): - - _apply_crossover_in_area(child1, - child2, - np.random.rand() * len(child1.condition), - np.random.rand() * len(child1.condition) - ) - - for child in child1, child2: - child.prediction = (parent1.prediction + parent2.prediction) / 2 - child.error = 0.25 * (parent1.error + parent2.error) / 2 - child.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 + assert isinstance(action_set, ClassifiersList) + # TODO: insert generator to calculate fitness_sum + fitness_sum = 0 + for cl in action_set: + fitness_sum += cl.fitness + choice_point = np.random.rand() * fitness_sum + fitness_sum = 0 + for cl in action_set: + fitness_sum += cl.fitness + if fitness_sum > choice_point: + return cl + return action_set[random.randrange(len(action_set))] -def _apply_crossover_in_area(child1: Classifier, child2: Classifier, x, y): - if x > y: - x, y = y, x - if x > len(child2.condition): - return - if y > len(child2.condition): - y = len(child2.condition) - i = 0 - while i < y: - if x <= i < y: - temp = child1.condition[i] - child1.condition[i] = child2.condition[i] - child2.condition[i] = temp - i += 1 + def _apply_crossover(self, child1: Classifier, child2: Classifier, + parent1: Classifier, parent2: Classifier): + self._apply_crossover_in_area( + child1, + child2, + np.random.rand() * len(child1.condition), + np.random.rand() * len(child1.condition) + ) -def _apply_mutation(child: Classifier, - cfg: Configuration, - situation): - i = 0 - while i < len(child.condition): + for child in child1, child2: + child.prediction = (parent1.prediction + parent2.prediction) / 2 + child.error = 0.25 * (parent1.error + parent2.error) / 2 + child.fitness = 0.1 * (parent1.fitness + parent2.fitness) / 2 + + @staticmethod + def _apply_crossover_in_area(child1: Classifier, child2: Classifier, x, y): + if x > y: + x, y = y, x + if x > len(child2.condition): + return + if y > len(child2.condition): + y = len(child2.condition) + i = 0 + while i < y: + if x <= i < y: + temp = child1.condition[i] + child1.condition[i] = child2.condition[i] + child2.condition[i] = temp + i += 1 + + @staticmethod + def _apply_mutation(child: Classifier, + cfg: Configuration, + situation): + i = 0 + while i < len(child.condition): + if np.random.rand() < cfg.mutation_chance: + if child.condition[i] == child.condition.WILDCARD: + child.condition[i] = situation[i] + else: + child.condition[i] = child.condition.WILDCARD + i += 1 if np.random.rand() < cfg.mutation_chance: - if child.condition[i] == child.condition.WILDCARD: - child.condition[i] = situation[i] - else: - child.condition[i] = child.condition.WILDCARD - i += 1 - if np.random.rand() < cfg.mutation_chance: - child.action = np.random.randint(cfg.number_of_actions) + child.action = np.random.randint(cfg.number_of_actions) diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index 9b3cfcee..491849c0 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -7,9 +7,10 @@ from lcs.agents import Agent from lcs.agents.Agent import TrialMetrics -from lcs.agents.xcs import Configuration, ClassifiersList, GeneticAlgorithm +from lcs.agents.xcs import Configuration, ClassifiersList +# TODO: delete old code from lcs.strategies.reinforcement_learning import simple_q_learning - +from lcs.agents.xcs import GeneticAlgorithm logger = logging.getLogger(__name__) @@ -27,6 +28,10 @@ def __init__(self, self.population = population else: self.population = ClassifiersList(cfg=cfg) + self.ga = GeneticAlgorithm( + population=self.population, + cfg=self.cfg + ) self.time_stamp = 0 self.reward = 0 @@ -87,11 +92,11 @@ def _distribute_and_update(self, action_set, situation, p): action_set.update_set(p) if self.cfg.do_action_set_subsumption: self.do_action_set_subsumption(action_set) - GeneticAlgorithm.run_ga(self.population, - action_set, - situation, - self.time_stamp, - self.cfg) + self.ga.run_ga( + action_set, + situation, + self.time_stamp + ) # TODO: EpsilonGreedy # Run into a lot of issues where in EpsilonGreedy where BestAction was not callable diff --git a/lcs/agents/xcs/__init__.py b/lcs/agents/xcs/__init__.py index d8c4db31..399fbea0 100644 --- a/lcs/agents/xcs/__init__.py +++ b/lcs/agents/xcs/__init__.py @@ -2,5 +2,6 @@ from .Configuration import Configuration from .Classifier import Classifier from .ClassifiersList import ClassifiersList +from .GeneticAlgorithm import GeneticAlgorithm from .XCS import XCS -from .GeneticAlgorithm import * + diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index bb53e16b..2d560f0b 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -28,3 +28,6 @@ def __eq__(self, other): if other.effect == self.effect: return True return False + + def __hash__(self): + return hash((str(self.condition), self.action)) diff --git a/lcs/agents/xncs/GeneticAlgorithm.py b/lcs/agents/xncs/GeneticAlgorithm.py new file mode 100644 index 00000000..4b037281 --- /dev/null +++ b/lcs/agents/xncs/GeneticAlgorithm.py @@ -0,0 +1,36 @@ +from lcs.agents.xcs import GeneticAlgorithm as XCSGeneticAlgorithm +from copy import copy +from lcs.agents.xncs import Classifier, ClassifiersList, Configuration + + +class GeneticAlgorithm(XCSGeneticAlgorithm): + + def __init__( + self, + population: ClassifiersList, + cfg: Configuration + ): + + super().__init__(population, cfg) + + def _make_children(self, parent1, parent2, time_stamp): + assert isinstance(parent1, Classifier) + assert isinstance(parent2, Classifier) + + child1 = Classifier( + self.cfg, + copy(parent1.condition), + copy(parent1.action), + time_stamp, + effect=None + ) + + child2 = Classifier( + self.cfg, + copy(parent2.condition), + copy(parent2.action), + time_stamp, + effect=None + ) + + return child1, child2 diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index f281eb27..e4e8d011 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -5,10 +5,8 @@ from lcs.agents.xcs import XCS from lcs.agents.xncs import Configuration, Backpropagation -# TODO: find typo that makes __init__ not do that -from lcs.agents.xncs.ClassifiersList import ClassifiersList -from lcs.agents.Agent import TrialMetrics -from lcs.strategies.reinforcement_learning import simple_q_learning +# TODO: find a way to not require super in __init__ +from lcs.agents.xncs import ClassifiersList, GeneticAlgorithm class XNCS(XCS): @@ -27,6 +25,10 @@ def __init__(self, else: self.population = ClassifiersList(cfg=cfg) self.cfg = cfg + self.ga = GeneticAlgorithm( + population=self.population, + cfg=self.cfg + ) self.time_stamp = 0 self.reward = 0 diff --git a/lcs/agents/xncs/__init__.py b/lcs/agents/xncs/__init__.py index 041002a2..42342b47 100644 --- a/lcs/agents/xncs/__init__.py +++ b/lcs/agents/xncs/__init__.py @@ -2,5 +2,6 @@ from .Configuration import Configuration from .Classifier import Classifier from .Backpropagation import Backpropagation -from .XNCS import XNCS +from .GeneticAlgorithm import GeneticAlgorithm from .ClassifiersList import ClassifiersList +from .XNCS import XNCS diff --git a/tests/lcs/agents/xcs/test_GA.py b/tests/lcs/agents/xcs/test_GA.py new file mode 100644 index 00000000..6ecf3455 --- /dev/null +++ b/tests/lcs/agents/xcs/test_GA.py @@ -0,0 +1,133 @@ +import pytest +from copy import copy + +from lcs import Perception +from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS, GeneticAlgorithm +from lcs.strategies.reinforcement_learning import simple_q_learning + +class TestGA: + + @pytest.fixture + def number_of_actions(self): + return 4 + + @pytest.fixture + def cfg(self, number_of_actions): + return Configuration(number_of_actions=number_of_actions, do_action_set_subsumption=False) + + @pytest.fixture + def situation(self): + return "1100" + + @pytest.fixture + def classifiers_list_diff_actions(self, cfg, situation): + classifiers_list = ClassifiersList(cfg) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + return classifiers_list + + @pytest.fixture + def xcs(self, cfg, classifiers_list_diff_actions): + xcs = XCS(cfg=cfg, population=classifiers_list_diff_actions) + return xcs + + @pytest.fixture + def ga(self, classifiers_list_diff_actions, cfg): + ga = GeneticAlgorithm(classifiers_list_diff_actions, cfg) + return ga + + def test_mutation(self, cfg, ga): + cfg.mutation_chance = 0 + cl = Classifier(cfg, Condition("####"), 0, 0) + ga._apply_mutation(cl, cfg, Perception("1111")) + assert cl.action == Classifier(cfg, Condition("1111"), 0, 0).action + assert cl.does_match(Condition("1111")) + cfg.mutation_chance = 1 + cl = Classifier(cfg, Condition("1111"), 0, 0) + ga._apply_mutation(cl, cfg, Perception("1111")) + assert cl.is_more_general(Classifier(cfg, Condition("1111"), 0, 0)) + + @pytest.mark.parametrize("cond1, cond2, x, y, end_cond1, end_cond2", [ + ("11111", "#####", 0, 5, "#####", "11111"), + ("11111", "000", 0, 5, "00011", "111"), + ("11111", "#####", 1, 4, "1###1", "#111#") + ]) + def test_crossover_area(self, cfg, ga, cond1, cond2, x, y, end_cond1, end_cond2): + cl1 = Classifier(cfg, Condition(cond1), 0, 0) + cl2 = Classifier(cfg, Condition(cond2), 1, 0) + ga._apply_crossover_in_area(cl1, cl2, x, y) + assert cl1.condition == Condition(end_cond1) + assert cl2.condition == Condition(end_cond2) + # Just to check for errors + + def test_crossover_values(self, cfg, ga, situation, classifiers_list_diff_actions): + cl1 = Classifier(cfg, Condition(situation), 0, 0) + cl2 = Classifier(cfg, Condition(situation), 1, 0) + ga._apply_crossover( + cl1, cl2, + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert cl1.prediction == cl2.prediction + assert cl1.error == cl2.error + assert cl1.fitness == cl2.fitness + + @pytest.mark.parametrize("chi", [ + 1, + 0 + ]) + # only tests for errors and types + def test_run_ga(self, cfg, ga, classifiers_list_diff_actions, chi): + cfg.do_GA_subsumption = True + xcs = XCS(cfg, classifiers_list_diff_actions) + action_set = xcs.population.generate_action_set(0) + cfg.chi = chi # do perform crossover + cfg.do_GA_subsumption = False # do not perform subsumption + ga.run_ga(action_set, Perception("0000"), 100000) + assert xcs.population[0].time_stamp != 0 + assert xcs.population.numerosity > 4 + + def test_make_children(self, cfg, ga, classifiers_list_diff_actions): + child1, child2 = ga._make_children( + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1], + 0 + ) + assert child1.numerosity == 1 + assert child2.numerosity == 1 + assert child1.experience == 0 + assert child2.experience == 0 + assert id(child1) != id(classifiers_list_diff_actions[0]) + assert id(child2) != id(classifiers_list_diff_actions[0]) + assert id(child1) != id(classifiers_list_diff_actions[1]) + assert id(child2) != id(classifiers_list_diff_actions[1]) + + def test_do_ga_subsumption_does_subsume_true(self, cfg, ga, classifiers_list_diff_actions, situation): + cfg.do_GA_subsumption = True + classifiers_list_diff_actions[0].error = 0 + classifiers_list_diff_actions[1].error = 0 + classifiers_list_diff_actions[0].expirience = 30 + classifiers_list_diff_actions[1].expirience = 30 + ga._perform_insertion_or_subsumption( + Classifier(cfg, Condition(situation), 0, 0), + Classifier(cfg, Condition(situation), 1, 0), + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert classifiers_list_diff_actions[0].numerosity > 1 + assert classifiers_list_diff_actions[1].numerosity > 1 + + def test_do_ga_subsumption_does_subsume_false(self, cfg, ga, classifiers_list_diff_actions, situation): + cfg.do_GA_subsumption = True + ga._perform_insertion_or_subsumption( + Classifier(cfg, Condition("####"), 0, 0), + Classifier(cfg, Condition("####"), 1, 0), + classifiers_list_diff_actions[0], + classifiers_list_diff_actions[1] + ) + assert classifiers_list_diff_actions[0].numerosity == 1 + assert classifiers_list_diff_actions[1].numerosity == 1 + assert len(classifiers_list_diff_actions) == 6 + diff --git a/tests/lcs/agents/xcs/test_XCS.py b/tests/lcs/agents/xcs/test_XCS.py index e8daeee7..cb0bc9f9 100644 --- a/tests/lcs/agents/xcs/test_XCS.py +++ b/tests/lcs/agents/xcs/test_XCS.py @@ -2,7 +2,7 @@ from copy import copy from lcs import Perception -from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS, GeneticAlgorithm +from lcs.agents.xcs import Configuration, Condition, Classifier, ClassifiersList, XCS from lcs.strategies.reinforcement_learning import simple_q_learning class TestXCS: @@ -71,106 +71,4 @@ def test_distribute_and_update(self, cfg, situation, classifiers_list_diff_actio assert not cl.fitness == cfg.initial_fitness assert not cl.error == cfg.initial_error - def test_mutation(self, cfg): - cfg.mutation_chance = 0 - cl = Classifier(cfg, Condition("####"), 0, 0) - GeneticAlgorithm._apply_mutation(cl, cfg, Perception("1111")) - assert cl.action == Classifier(cfg, Condition("1111"), 0, 0).action - assert cl.does_match(Condition("1111")) - cfg.mutation_chance = 1 - cl = Classifier(cfg, Condition("1111"), 0, 0) - GeneticAlgorithm._apply_mutation(cl, cfg, Perception("1111")) - assert cl.is_more_general(Classifier(cfg, Condition("1111"), 0, 0)) - - @pytest.mark.parametrize("cond1, cond2, x, y, end_cond1, end_cond2", [ - ("11111", "#####", 0, 5, "#####", "11111"), - ("11111", "000", 0, 5, "00011", "111"), - ("11111", "#####", 1, 4, "1###1", "#111#") - ]) - def test_crossover_area(self, cfg, cond1, cond2, x, y, end_cond1, end_cond2): - cl1 = Classifier(cfg, Condition(cond1), 0, 0) - cl2 = Classifier(cfg, Condition(cond2), 1, 0) - GeneticAlgorithm._apply_crossover_in_area(cl1, cl2, x, y) - assert cl1.condition == Condition(end_cond1) - assert cl2.condition == Condition(end_cond2) - # Just to check for errors - - def test_crossover_values(self, cfg, situation, classifiers_list_diff_actions): - cl1 = Classifier(cfg, Condition(situation), 0, 0) - cl2 = Classifier(cfg, Condition(situation), 1, 0) - GeneticAlgorithm._apply_crossover(cl1, cl2, - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert cl1.prediction == cl2.prediction - assert cl1.error == cl2.error - assert cl1.fitness == cl2.fitness - - @pytest.mark.parametrize("chi", [ - 1, - 0 - ]) - # only tests for errors and types - def test_run_ga(self, cfg, classifiers_list_diff_actions, chi): - cfg.do_GA_subsumption = True - xcs = XCS(cfg, classifiers_list_diff_actions) - action_set = xcs.population.generate_action_set(0) - cfg.chi = chi # do perform crossover - cfg.do_GA_subsumption = False # do not perform subsumption - GeneticAlgorithm.run_ga(xcs.population, action_set, Perception("0000"), 100000, cfg) - assert xcs.population[0].time_stamp != 0 - assert xcs.population.numerosity > 4 - - def test_simple_q_learning(self, cfg, classifiers_list_diff_actions): - reward = simple_q_learning(0, 0, cfg.learning_rate, cfg.gamma, 0) - assert reward == 0 - reward = simple_q_learning(0, 10, cfg.learning_rate, cfg.gamma, 0) - assert reward > 0 - new_reward = simple_q_learning(reward, 0, cfg.learning_rate, cfg.gamma, 0) - assert new_reward < reward - - def test_make_children(self, cfg, classifiers_list_diff_actions): - child1, child2 = GeneticAlgorithm._make_children( - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1], - cfg, - 0 - ) - assert child1.numerosity == 1 - assert child2.numerosity == 1 - assert child1.experience == 0 - assert child2.experience == 0 - assert id(child1) != id(classifiers_list_diff_actions[0]) - assert id(child2) != id(classifiers_list_diff_actions[0]) - assert id(child1) != id(classifiers_list_diff_actions[1]) - assert id(child2) != id(classifiers_list_diff_actions[1]) - - def test_do_ga_subsumption_does_subsume_true(self, cfg, classifiers_list_diff_actions, situation): - cfg.do_GA_subsumption = True - classifiers_list_diff_actions[0].error = 0 - classifiers_list_diff_actions[1].error = 0 - classifiers_list_diff_actions[0].expirience = 30 - classifiers_list_diff_actions[1].expirience = 30 - GeneticAlgorithm._perform_insertion_or_subsumption( - cfg, classifiers_list_diff_actions, - Classifier(cfg, Condition(situation), 0, 0), - Classifier(cfg, Condition(situation), 1, 0), - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert classifiers_list_diff_actions[0].numerosity > 1 - assert classifiers_list_diff_actions[1].numerosity > 1 - - def test_do_ga_subsumption_does_subsume_false(self, cfg, classifiers_list_diff_actions, situation): - cfg.do_GA_subsumption = True - GeneticAlgorithm._perform_insertion_or_subsumption( - cfg, classifiers_list_diff_actions, - Classifier(cfg, Condition("####"), 0, 0), - Classifier(cfg, Condition("####"), 1, 0), - classifiers_list_diff_actions[0], - classifiers_list_diff_actions[1] - ) - assert classifiers_list_diff_actions[0].numerosity == 1 - assert classifiers_list_diff_actions[1].numerosity == 1 - assert len(classifiers_list_diff_actions) == 6 From 5aa92cb899eb35dd209000d65808cbcbea4ec4ce Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 18 May 2021 13:17:59 +0200 Subject: [PATCH 08/33] XNCS duplicates fix --- lcs/agents/xncs/Backpropagation.py | 20 +++++++++++++++++--- lcs/agents/xncs/Classifier.py | 6 +++++- lcs/agents/xncs/ClassifiersList.py | 2 ++ lcs/agents/xncs/XNCS.py | 6 +++++- lcs/agents/xncs/__init__.py | 3 ++- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index d864c73c..18f8ce85 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -1,11 +1,13 @@ from copy import copy -from lcs.agents.xncs import Configuration, Classifier, Effect +from lcs.agents.xncs import Configuration, Classifier, ClassifiersList, Effect class Backpropagation: - def __init__(self, cfg: Configuration): + def __init__(self, cfg: Configuration, + population: ClassifiersList): + self.population = population self.classifiers_for_update = [] self.update_vectors = [] self.cfg = cfg @@ -21,9 +23,14 @@ def update_bp(self): for cl, uv in zip(self.classifiers_for_update, self.update_vectors): if cl.effect is None: cl.effect = copy(Effect(uv)) + self.remove_copies(cl) else: cl.effect = copy(Effect(uv)) - cl.error = cl.error + self.cfg.lem + pop_cl = self.remove_copies(cl) + if pop_cl is None: + cl.error += self.cfg.lem + else: + pop_cl.error += self.cfg.lem self.classifiers_for_update = [] self.update_vectors = [] self.update_cycles = 0 @@ -32,3 +39,10 @@ def check_and_update(self): self.update_cycles += 1 if self.update_cycles >= self.cfg.lmc: self.update_bp() + + def remove_copies(self, bp_cl): + for pop_cl in self.population: + if pop_cl == bp_cl: + self.population.remove(pop_cl) + return pop_cl + return None diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 2d560f0b..2cffcacb 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -30,4 +30,8 @@ def __eq__(self, other): return False def __hash__(self): - return hash((str(self.condition), self.action)) + return hash((str(self.condition),str(self.effect), self.action)) + + def __str__(self): + return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]" diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 9e6c3a8b..637d82d3 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -16,6 +16,8 @@ def __init__(self, ) -> None: super().__init__(cfg, *args, oktypes=oktypes) + # without this function the Classifierlist will create XCS Classifiers + # instead of XNCS classifiers def generate_covering_classifier(self, situation, action, time_stamp): generalized = [] for i in range(len(situation)): diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index e4e8d011..9ae22603 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -19,7 +19,7 @@ def __init__(self, :param cfg: object storing parameters of the experiment :param population: all classifiers at current time """ - self.back_propagation = Backpropagation(cfg) + if population is not None: self.population = population else: @@ -29,6 +29,10 @@ def __init__(self, population=self.population, cfg=self.cfg ) + self.back_propagation = Backpropagation( + cfg=self.cfg, + population=self.population + ) self.time_stamp = 0 self.reward = 0 diff --git a/lcs/agents/xncs/__init__.py b/lcs/agents/xncs/__init__.py index 42342b47..438c5a4f 100644 --- a/lcs/agents/xncs/__init__.py +++ b/lcs/agents/xncs/__init__.py @@ -1,7 +1,8 @@ from .Effect import Effect from .Configuration import Configuration from .Classifier import Classifier -from .Backpropagation import Backpropagation + from .GeneticAlgorithm import GeneticAlgorithm from .ClassifiersList import ClassifiersList +from .Backpropagation import Backpropagation from .XNCS import XNCS From 39caccbb75ec29f5c8494b307928cf9012d0f2cb Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 20 May 2021 12:29:05 +0200 Subject: [PATCH 09/33] BP test fix --- tests/lcs/agents/xncs/test_backpropagation.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 50509833..6d5d143e 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -11,12 +11,27 @@ class TestBackpropagation: def cfg(self): return Configuration(lmc=2, lem=0.2, number_of_actions=4) - def test_init(self, cfg): - bp = Backpropagation(cfg) + @pytest.fixture + def situation(self): + return "1100" + + @pytest.fixture + def classifiers_list_diff_actions(self, cfg, situation): + classifiers_list = ClassifiersList(cfg) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) + return classifiers_list + + @pytest.fixture + def bp(self, cfg, classifiers_list_diff_actions): + return Backpropagation(cfg, classifiers_list_diff_actions) + + def test_init(self, cfg, bp): assert id(bp.cfg) == id(cfg) - def test_insert(self, cfg): - bp = Backpropagation(cfg) + def test_insert(self, cfg, bp): cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) ef = Effect("0110") bp.insert_into_bp(cl, ef) @@ -25,8 +40,7 @@ def test_insert(self, cfg): assert bp.classifiers_for_update[0] == cl assert bp.update_vectors[0] == ef - def test_update(self, cfg): - bp = Backpropagation(cfg) + def test_update(self, cfg, bp): cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) ef = Effect("0110") bp.insert_into_bp(cl, ef) @@ -37,12 +51,3 @@ def test_update(self, cfg): assert cl.effect == ef assert cl.error != cfg.initial_error - def test_update(self, cfg): - bp = Backpropagation(cfg) - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - bp.check_and_update() - assert cl.effect is None - bp.check_and_update() - assert cl.effect is not None From 3a6eb5850318126364892bd9585dc1717f2e8be1 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 20 May 2021 12:37:07 +0200 Subject: [PATCH 10/33] Fix Covering Covering were meant to include random effect. Effect will later be changed during backpropagation, so classifier will not be stuck with this random effect. --- lcs/agents/xncs/ClassifiersList.py | 8 ++++++-- tests/lcs/agents/xncs/test_ClassifiersList.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 637d82d3..7ba46a53 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -1,9 +1,10 @@ import numpy as np import logging +import random import lcs.agents.xcs as xcs from lcs.agents.xcs import Condition -from lcs.agents.xncs import Classifier, Configuration +from lcs.agents.xncs import Classifier, Configuration, Effect logger = logging.getLogger(__name__) @@ -20,13 +21,16 @@ def __init__(self, # instead of XNCS classifiers def generate_covering_classifier(self, situation, action, time_stamp): generalized = [] + effect = [] for i in range(len(situation)): if np.random.rand() > self.cfg.covering_wildcard_chance: generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) + effect.append(str(random.randint(0, 1))) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, - time_stamp=time_stamp) + time_stamp=time_stamp, + effect=Effect(effect)) return cl diff --git a/tests/lcs/agents/xncs/test_ClassifiersList.py b/tests/lcs/agents/xncs/test_ClassifiersList.py index a3b30d5b..26753797 100644 --- a/tests/lcs/agents/xncs/test_ClassifiersList.py +++ b/tests/lcs/agents/xncs/test_ClassifiersList.py @@ -35,7 +35,7 @@ def test_covering(self, cfg): covering_cl = classifiers_list.generate_covering_classifier(Perception("1111"), 0, 0) assert covering_cl.does_match(Perception("1111")) assert classifiers_list.generate_covering_classifier("1111", 0, 0).action == 0 - assert covering_cl.effect is None + assert len(covering_cl.effect) == len(covering_cl.condition) def test_match_set(self, classifiers_list_diff_actions): assert len(classifiers_list_diff_actions.generate_match_set(Perception("1100"), 1)) == 4 From 68b82f44938cdeb2cee2542587bff97297f4caf2 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 20 May 2021 12:54:51 +0200 Subject: [PATCH 11/33] Fix Only Fittest CL into BP --- lcs/agents/xncs/ClassifiersList.py | 4 ++++ lcs/agents/xncs/XNCS.py | 12 +++++------- tests/lcs/agents/xncs/test_ClassifiersList.py | 5 +++++ tests/lcs/agents/xncs/test_XNCS.py | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 7ba46a53..78022cb9 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -34,3 +34,7 @@ def generate_covering_classifier(self, situation, action, time_stamp): time_stamp=time_stamp, effect=Effect(effect)) return cl + + @property + def fittest_classifier(self): + return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 9ae22603..40bb6ec5 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -40,11 +40,9 @@ def _distribute_and_update(self, action_set, situation, p): super()._distribute_and_update(action_set, situation, p) self._compare_effect(action_set, situation) - def _compare_effect(self, action_set, situation): + def _compare_effect(self, action_set: ClassifiersList, situation): if action_set is not None: - for cl in action_set: - if cl.effect is None or not cl.effect.subsumes(situation): - self.back_propagation.insert_into_bp(cl, situation) - else: - self.back_propagation.update_bp() - self.back_propagation.check_and_update() + self.back_propagation.insert_into_bp( + action_set.fittest_classifier, + situation + ) diff --git a/tests/lcs/agents/xncs/test_ClassifiersList.py b/tests/lcs/agents/xncs/test_ClassifiersList.py index 26753797..ae66a9c3 100644 --- a/tests/lcs/agents/xncs/test_ClassifiersList.py +++ b/tests/lcs/agents/xncs/test_ClassifiersList.py @@ -49,3 +49,8 @@ def test_action_set(self, cfg, classifiers_list_diff_actions): assert action_set[0].action == 0 action_set[0].action = 1 assert classifiers_list_diff_actions[0].action == 1 + + def test_return_fittest(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[2].prediction = 20 + classifiers_list_diff_actions[2].fitness = 20 + assert id(classifiers_list_diff_actions.fittest_classifier) == id(classifiers_list_diff_actions[2]) diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index 1716df79..6b19661f 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -35,7 +35,7 @@ def test_init(self, cfg, classifiers_list_diff_actions): def test_distribute_and_update(self, cfg, classifiers_list_diff_actions): xncs = XNCS(cfg, classifiers_list_diff_actions) xncs._distribute_and_update(classifiers_list_diff_actions, "1100", 0.1) - assert len(xncs.back_propagation.update_vectors) == 4 + assert len(xncs.back_propagation.update_vectors) == 1 def test_correct_types(self,cfg): xncs = XNCS(cfg) From 46408f9773cecb74e580c855f6a0b5e83ba01096 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 20 May 2021 13:56:12 +0200 Subject: [PATCH 12/33] Fixes for BP --- lcs/agents/xcs/ClassifiersList.py | 5 +++ lcs/agents/xncs/Backpropagation.py | 35 +++++++----------- lcs/agents/xncs/ClassifiersList.py | 4 --- lcs/agents/xncs/GeneticAlgorithm.py | 4 +-- lcs/agents/xncs/XNCS.py | 5 +-- tests/lcs/agents/xncs/test_XNCS.py | 36 ++++++++++++++----- tests/lcs/agents/xncs/test_backpropagation.py | 20 +++++------ 7 files changed, 60 insertions(+), 49 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 50dcf50d..0d4e4ac7 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -153,3 +153,8 @@ def _update_fitness(self): self.cfg.learning_rate * (k * cl.numerosity / accuracy_sum - cl.fitness) ) + + @property + def fittest_classifier(self): + assert len(self) > 0 + return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 18f8ce85..4132f333 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -18,31 +18,22 @@ def insert_into_bp(self, update_vector: Effect): self.classifiers_for_update.append(classifier) self.update_vectors.append(update_vector) + self._check_for_update() - def update_bp(self): + def _check_for_update(self): + self.update_cycles += 1 + if self.update_cycles >= self.cfg.lmc: + self._update_bp() + # TODO: One of them is NoneType + elif self.classifiers_for_update[-1].effect == self.update_vectors[-1]: + self._update_bp() + + def _update_bp(self): for cl, uv in zip(self.classifiers_for_update, self.update_vectors): - if cl.effect is None: - cl.effect = copy(Effect(uv)) - self.remove_copies(cl) - else: - cl.effect = copy(Effect(uv)) - pop_cl = self.remove_copies(cl) - if pop_cl is None: - cl.error += self.cfg.lem - else: - pop_cl.error += self.cfg.lem + if cl.effect != uv: + cl.effect = copy(uv) + cl.error += self.cfg.lem * cl.error self.classifiers_for_update = [] self.update_vectors = [] self.update_cycles = 0 - def check_and_update(self): - self.update_cycles += 1 - if self.update_cycles >= self.cfg.lmc: - self.update_bp() - - def remove_copies(self, bp_cl): - for pop_cl in self.population: - if pop_cl == bp_cl: - self.population.remove(pop_cl) - return pop_cl - return None diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 78022cb9..7ba46a53 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -34,7 +34,3 @@ def generate_covering_classifier(self, situation, action, time_stamp): time_stamp=time_stamp, effect=Effect(effect)) return cl - - @property - def fittest_classifier(self): - return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/GeneticAlgorithm.py b/lcs/agents/xncs/GeneticAlgorithm.py index 4b037281..2e57cd30 100644 --- a/lcs/agents/xncs/GeneticAlgorithm.py +++ b/lcs/agents/xncs/GeneticAlgorithm.py @@ -22,7 +22,7 @@ def _make_children(self, parent1, parent2, time_stamp): copy(parent1.condition), copy(parent1.action), time_stamp, - effect=None + copy(parent1.effect) ) child2 = Classifier( @@ -30,7 +30,7 @@ def _make_children(self, parent1, parent2, time_stamp): copy(parent2.condition), copy(parent2.action), time_stamp, - effect=None + copy(parent2.effect) ) return child1, child2 diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 40bb6ec5..9c752fbe 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -4,9 +4,10 @@ from copy import copy from lcs.agents.xcs import XCS +from lcs.agents.Agent import TrialMetrics from lcs.agents.xncs import Configuration, Backpropagation # TODO: find a way to not require super in __init__ -from lcs.agents.xncs import ClassifiersList, GeneticAlgorithm +from lcs.agents.xncs import ClassifiersList, GeneticAlgorithm, Effect class XNCS(XCS): @@ -44,5 +45,5 @@ def _compare_effect(self, action_set: ClassifiersList, situation): if action_set is not None: self.back_propagation.insert_into_bp( action_set.fittest_classifier, - situation + Effect(situation) ) diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index 6b19661f..364cc50c 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -2,7 +2,7 @@ from copy import copy from lcs import Perception -from lcs.agents.xncs import XNCS, Classifier, Configuration, Backpropagation, ClassifiersList +from lcs.agents.xncs import XNCS, Classifier, Configuration, Backpropagation, ClassifiersList, Effect from lcs.agents.xcs import Condition @@ -19,10 +19,10 @@ def situation(self): @pytest.fixture def classifiers_list_diff_actions(self, cfg, situation): classifiers_list = ClassifiersList(cfg) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0)) - classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0)) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 0, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 1, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 2, 0, Effect(situation))) + classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) return classifiers_list def test_init(self, cfg, classifiers_list_diff_actions): @@ -32,12 +32,32 @@ def test_init(self, cfg, classifiers_list_diff_actions): assert id(xncs.cfg) == id(cfg) assert len(xncs.population) == 4 - def test_distribute_and_update(self, cfg, classifiers_list_diff_actions): + def test_distribute_and_update(self, cfg: Configuration, + classifiers_list_diff_actions, + situation): xncs = XNCS(cfg, classifiers_list_diff_actions) - xncs._distribute_and_update(classifiers_list_diff_actions, "1100", 0.1) + action_set = xncs.population.generate_action_set(0) + assert action_set is not None + assert action_set.fittest_classifier is not None + xncs._distribute_and_update(action_set, situation, 0.1) + # update should happen because effect matched inserted vector + assert len(xncs.back_propagation.update_vectors) == 0 + assert len(xncs.back_propagation.classifiers_for_update) == 0 + assert xncs.back_propagation.update_cycles == 0 + + def test_distribute_and_update_diff(self, cfg: Configuration, + classifiers_list_diff_actions, + situation): + xncs = XNCS(cfg, classifiers_list_diff_actions) + action_set = xncs.population.generate_action_set(0) + assert action_set is not None + assert action_set.fittest_classifier is not None + xncs._distribute_and_update(action_set, "####", 0.1) assert len(xncs.back_propagation.update_vectors) == 1 + assert len(xncs.back_propagation.classifiers_for_update) == 1 + assert xncs.back_propagation.update_cycles == 1 - def test_correct_types(self,cfg): + def test_correct_types(self, cfg): xncs = XNCS(cfg) assert isinstance(xncs.population, ClassifiersList) xncs.population.insert_in_population( diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 6d5d143e..ffa7cdc3 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -32,22 +32,20 @@ def test_init(self, cfg, bp): assert id(bp.cfg) == id(cfg) def test_insert(self, cfg, bp): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) + cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0, effect=Effect("1100")) ef = Effect("0110") bp.insert_into_bp(cl, ef) assert id(bp.classifiers_for_update[0]) == id(cl) assert id(bp.update_vectors[0]) == id(ef) assert bp.classifiers_for_update[0] == cl assert bp.update_vectors[0] == ef + assert bp.update_cycles > 0 + + def test_update(self, cfg: Configuration, bp): + cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0, effect=Effect("1100")) + bp.insert_into_bp(cl, Effect("0110")) + bp._update_bp() + assert cl.effect == Effect("0110") + assert cl.error > cfg.initial_error - def test_update(self, cfg, bp): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - bp.update_bp() - assert cl.effect == ef - bp.insert_into_bp(cl, ef) - bp.update_bp() - assert cl.effect == ef - assert cl.error != cfg.initial_error From 40ae90bfc1b7355951ca006f0360278000b39a20 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 11:27:56 +0200 Subject: [PATCH 13/33] Test spit + more --- tests/lcs/agents/xncs/test_Classifier.py | 21 +++++++++++++++++++-- tests/lcs/agents/xncs/test_XNCS.py | 5 ++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/lcs/agents/xncs/test_Classifier.py b/tests/lcs/agents/xncs/test_Classifier.py index 4ee91f6f..15d81827 100644 --- a/tests/lcs/agents/xncs/test_Classifier.py +++ b/tests/lcs/agents/xncs/test_Classifier.py @@ -15,6 +15,23 @@ def test_init(self, cfg): assert cl.cfg == cfg assert cl.effect is None + def test_classifier_default(self, cfg): + cl = Classifier(cfg, + Condition("####"), + 2, + 8) + assert cl.condition == Condition('####') + assert cl.prediction == cfg.initial_error + + def test_does_subsume(self, cfg: Configuration): + cl = Classifier(cfg, Condition("11##"), 0, 0) + assert not cl.does_subsume(Classifier(cfg, Condition("1111"), 0, 0)) + cl.experience = cfg.subsumption_threshold * 2 + cl.error = cfg.initial_error / 2 + assert cl.does_subsume(Classifier(cfg, Condition("1111"), 0, 0)) + assert not cl.does_subsume(Classifier(cfg, Condition("1111"), 1, 0)) + assert not cl.does_subsume(Classifier(cfg, Condition("0011"), 1, 0)) + @pytest.mark.parametrize("cond1, cond2, act1, act2, result, ef1, ef2", [ ("1111", "1111", 1, 1, True, "1111", "1111"), ("#100", "#100", 0, 0, True, "1111", "1111"), @@ -37,8 +54,8 @@ def test_init(self, cfg): ("1111", "11", 1, 1, False, "1111", "1111"), ("11", "1111", 1, 1, False, "1111", "1111"), - ("1111", "1111", 1, 1, False, "1111", "1100"), - ("1111", "1111", 1, 1, False, "1100", "1111"), + # ("1111", "1111", 1, 1, False, "1111", "1100"), + # ("1111", "1111", 1, 1, False, "1100", "1111"), ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result, ef1, ef2): diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index 364cc50c..e8204b3a 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -57,9 +57,12 @@ def test_distribute_and_update_diff(self, cfg: Configuration, assert len(xncs.back_propagation.classifiers_for_update) == 1 assert xncs.back_propagation.update_cycles == 1 - def test_correct_types(self, cfg): + def test_correct_type_population(self, cfg): xncs = XNCS(cfg) assert isinstance(xncs.population, ClassifiersList) + + def test_correct_type_classifier(self, cfg): + xncs = XNCS(cfg) xncs.population.insert_in_population( Classifier(cfg, Condition("1100"), 0, 0) ) From 76f0d2828293b33c18c60d6ca8959ee63763cfab Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 11:40:05 +0200 Subject: [PATCH 14/33] Woods1 Effect fix --- lcs/agents/xncs/ClassifiersList.py | 2 +- lcs/agents/xncs/GeneticAlgorithm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 7ba46a53..1a2dd62a 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -27,7 +27,7 @@ def generate_covering_classifier(self, situation, action, time_stamp): generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) - effect.append(str(random.randint(0, 1))) + effect.append(str(random.choice(situation))) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, diff --git a/lcs/agents/xncs/GeneticAlgorithm.py b/lcs/agents/xncs/GeneticAlgorithm.py index 2e57cd30..21eb93f8 100644 --- a/lcs/agents/xncs/GeneticAlgorithm.py +++ b/lcs/agents/xncs/GeneticAlgorithm.py @@ -5,12 +5,12 @@ class GeneticAlgorithm(XCSGeneticAlgorithm): + # Classifierslist is supposed to be XNCS type def __init__( self, population: ClassifiersList, cfg: Configuration ): - super().__init__(population, cfg) def _make_children(self, parent1, parent2, time_stamp): From 6cf5767ace53ee47449d081ff24ba3ce980409d7 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 11:48:00 +0200 Subject: [PATCH 15/33] Revert "Merge branch 'master' into feature/xcs_xncs" This reverts commit 94f75af9d6d9af7f0d5df8fc5d5c5efdd52dcaf2, reversing changes made to 39caccbb75ec29f5c8494b307928cf9012d0f2cb. --- lcs/agents/Agent.py | 25 ------------------------- lcs/agents/acs/Configuration.py | 6 +----- lcs/agents/acs2/Configuration.py | 8 ++------ lcs/agents/yacs/yacs.py | 6 +----- setup.py | 7 ------- 5 files changed, 4 insertions(+), 48 deletions(-) diff --git a/lcs/agents/Agent.py b/lcs/agents/Agent.py index b70f39a3..7974fb0b 100644 --- a/lcs/agents/Agent.py +++ b/lcs/agents/Agent.py @@ -3,10 +3,7 @@ from timeit import default_timer as timer from typing import Callable, List, Tuple -import dill -import mlflow import numpy as np -import tempfile from lcs.metrics import basic_metrics @@ -119,8 +116,6 @@ def _evaluate(self, tuple population of classifiers and metrics """ - using_mlflow = hasattr(self.get_cfg(), 'use_mlflow') and self.get_cfg().use_mlflow - current_trial = 0 steps = 0 @@ -143,26 +138,6 @@ def _evaluate(self, metrics.append(m) - if using_mlflow: - mlflow.log_metrics(m, current_trial) - - # checkpoint model and metrics - if self.get_cfg().model_checkpoint_freq: - if current_trial % self.get_cfg().model_checkpoint_freq == 0: - prefix = f"-trial-{current_trial}" - with tempfile.TemporaryDirectory(prefix) as td: - logger.debug(f"checkpointing model to {td}") - pop_path = f"{td}/population.dill" - metrics_path = f"{td}/metrics.dill" - - dill.dump(self.get_population(), - open(pop_path, mode='wb')) - - dill.dump(metrics, open(metrics_path, mode='wb')) - - if using_mlflow: - mlflow.log_artifacts(td, f"{current_trial}/") - # Print last metric if current_trial % np.round(n_trials / 10) == 0: logger.info(metrics[-1]) diff --git a/lcs/agents/acs/Configuration.py b/lcs/agents/acs/Configuration.py index 88fde13b..9a9988d1 100644 --- a/lcs/agents/acs/Configuration.py +++ b/lcs/agents/acs/Configuration.py @@ -12,7 +12,6 @@ def __init__(self, user_metrics_collector_fcn: Callable = None, fitness_fcn=None, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, do_subsumption: bool = True, beta: float = 0.05, # gamma: float = 0.95, @@ -21,8 +20,7 @@ def __init__(self, epsilon: float = 0.5, u_max: int = 100000, theta_exp: int = 20, - theta_as: int = 20, - use_mlflow: bool = False) -> None: + theta_as: int = 20) -> None: """ Creates the configuration object used during training the ACS2 agent. @@ -51,7 +49,6 @@ def __init__(self, self.classifier_wildcard = classifier_wildcard self.environment_adapter = environment_adapter self.metrics_trial_frequency = metrics_trial_frequency - self.model_checkpoint_freq = model_checkpoint_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn self.fitness_fcn = fitness_fcn self.do_subsumption = do_subsumption @@ -62,7 +59,6 @@ def __init__(self, self.epsilon = epsilon self.u_max = u_max self.theta_as = theta_as - self.use_mlflow = use_mlflow def __str__(self) -> str: return str(vars(self)) diff --git a/lcs/agents/acs2/Configuration.py b/lcs/agents/acs2/Configuration.py index 024d7623..526c0518 100644 --- a/lcs/agents/acs2/Configuration.py +++ b/lcs/agents/acs2/Configuration.py @@ -14,7 +14,6 @@ def __init__(self, user_metrics_collector_fcn: Callable = None, fitness_fcn=None, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, do_pee: bool = False, do_ga: bool = False, do_subsumption: bool = True, @@ -33,8 +32,7 @@ def __init__(self, theta_ga: int = 100, theta_as: int = 20, mu: float = 0.3, - chi: float = 0.8, - use_mlflow: bool = False): + chi: float = 0.8): super(Configuration, self).__init__( classifier_length, @@ -44,7 +42,6 @@ def __init__(self, user_metrics_collector_fcn, fitness_fcn, metrics_trial_frequency, - model_checkpoint_frequency, do_subsumption, beta, theta_i, @@ -52,8 +49,7 @@ def __init__(self, epsilon, u_max, theta_exp, - theta_as, - use_mlflow) + theta_as) self.gamma = gamma self.do_pee = do_pee diff --git a/lcs/agents/yacs/yacs.py b/lcs/agents/yacs/yacs.py index 53858f7f..61f1e0dd 100644 --- a/lcs/agents/yacs/yacs.py +++ b/lcs/agents/yacs/yacs.py @@ -38,9 +38,7 @@ def __init__(self, discount_factor: float = 0.9, estimate_expected_improvements: bool = True, metrics_trial_frequency: int = 5, - model_checkpoint_frequency: int = None, - user_metrics_collector_fcn: Callable = None, - use_mlflow: bool = False): + user_metrics_collector_fcn: Callable = None): assert classifier_length == len(feature_possible_values) self.classifier_length = classifier_length self.number_of_possible_actions = number_of_possible_actions @@ -51,9 +49,7 @@ def __init__(self, self.gamma = discount_factor self.estimate_expected_improvements = estimate_expected_improvements self.metrics_trial_frequency = metrics_trial_frequency - self.model_checkpoint_freq = model_checkpoint_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn - self.use_mlflow = use_mlflow class Condition(ImmutableSequence): diff --git a/setup.py b/setup.py index 9780f865..f5a92ad3 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,6 @@ 'pytest-xdist==2.2.1' ] -mlflow_requires = [ - 'boto3', - 'dill', - 'mlflow' -] - docs_requires = [ 'sphinx', 'nbsphinx', @@ -51,7 +45,6 @@ 'dataslots>=1.0.1' ], extras_require={ - 'mlflow': mlflow_requires, 'testing': testing_requires, 'documentation': docs_requires }, From 866f0054bfb6a2b3c8cb0fbdc79f3a4a50a85b83 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 12:02:33 +0200 Subject: [PATCH 16/33] New equals --- lcs/agents/xncs/Classifier.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 2cffcacb..4c2d9114 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -16,21 +16,8 @@ def __init__(self, self.effect = effect super().__init__(cfg, condition, action, time_stamp) - def __eq__(self, other): - if other.action == self.action \ - and other.condition == self.condition: - if other.effect is None and self.effect is None: - return True - if other.effect is None: - return False - if self.effect is None: - return False - if other.effect == self.effect: - return True - return False - def __hash__(self): - return hash((str(self.condition),str(self.effect), self.action)) + return hash((str(self.condition), self.action, self.effect)) def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ From 5ad9b9cde9f14a05fad9490166192e49f3c4b785 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 12:31:57 +0200 Subject: [PATCH 17/33] Revert "New equals" This reverts commit 866f0054bfb6a2b3c8cb0fbdc79f3a4a50a85b83. --- lcs/agents/xncs/Classifier.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 4c2d9114..2cffcacb 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -16,8 +16,21 @@ def __init__(self, self.effect = effect super().__init__(cfg, condition, action, time_stamp) + def __eq__(self, other): + if other.action == self.action \ + and other.condition == self.condition: + if other.effect is None and self.effect is None: + return True + if other.effect is None: + return False + if self.effect is None: + return False + if other.effect == self.effect: + return True + return False + def __hash__(self): - return hash((str(self.condition), self.action, self.effect)) + return hash((str(self.condition),str(self.effect), self.action)) def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ From c4b69daa203eadf3d049404daceead03ce151faa Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 12:54:09 +0200 Subject: [PATCH 18/33] Fix for BP --- lcs/agents/xncs/Backpropagation.py | 2 +- tests/lcs/agents/xncs/test_Classifier.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 4132f333..f57914a9 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -31,7 +31,7 @@ def _check_for_update(self): def _update_bp(self): for cl, uv in zip(self.classifiers_for_update, self.update_vectors): if cl.effect != uv: - cl.effect = copy(uv) + # cl.effect = copy(uv) cl.error += self.cfg.lem * cl.error self.classifiers_for_update = [] self.update_vectors = [] diff --git a/tests/lcs/agents/xncs/test_Classifier.py b/tests/lcs/agents/xncs/test_Classifier.py index 15d81827..b4f1c3fa 100644 --- a/tests/lcs/agents/xncs/test_Classifier.py +++ b/tests/lcs/agents/xncs/test_Classifier.py @@ -54,8 +54,8 @@ def test_does_subsume(self, cfg: Configuration): ("1111", "11", 1, 1, False, "1111", "1111"), ("11", "1111", 1, 1, False, "1111", "1111"), - # ("1111", "1111", 1, 1, False, "1111", "1100"), - # ("1111", "1111", 1, 1, False, "1100", "1111"), + ("1111", "1111", 1, 1, False, "1111", "1100"), + ("1111", "1111", 1, 1, False, "1100", "1111"), ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result, ef1, ef2): From 1d9b26fa48eb4d7fb6a110d0bcec9eae67caa665 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Tue, 25 May 2021 13:42:38 +0200 Subject: [PATCH 19/33] XCS fix for woods --- lcs/agents/xcs/Classifier.py | 2 +- lcs/agents/xcs/ClassifiersList.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lcs/agents/xcs/Classifier.py b/lcs/agents/xcs/Classifier.py index f5dae0e1..1da0d2b9 100644 --- a/lcs/agents/xcs/Classifier.py +++ b/lcs/agents/xcs/Classifier.py @@ -61,7 +61,7 @@ def __len__(self): def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - Num:{self.numerosity} " + \ - f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]" + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, Error:{self.error}]" def __eq__(self, o): return o.condition == self.condition and o.action == self.action diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 0d4e4ac7..23c68639 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -21,7 +21,7 @@ def __init__(self, def insert_in_population(self, cl: Classifier): existing_classifiers = [c for c in self if c == cl] if len(existing_classifiers) > 0: - # assert len(existing_classifiers) == 1, 'duplicates found, while inserting' + assert len(existing_classifiers) == 1, 'duplicates found, while inserting' existing_classifiers[0].numerosity += 1 else: self.append(cl) @@ -50,7 +50,12 @@ def _generate_covering_and_insert(self, situation, action, time_stamp): # Roulette-Wheel Deletion # TODO: use strategies def delete_from_population(self): - if self.numerosity > self.cfg.max_population: + # TODO: change while to if + # there are places where more than one rule enters the population + # to remedy it I just made deletion run until it cleared all of them + # proffered method should be running it once, ideally inside + # insert_into_population + while self.numerosity > self.cfg.max_population: average_fitness = sum(cl.fitness for cl in self) / self.numerosity deletion_votes = [] for cl in self: @@ -72,9 +77,10 @@ def _remove_based_on_votes(self, deletion_votes, selector): if selector <= 0: if cl.numerosity > 1: cl.numerosity -= 1 + return cl else: self.safe_remove(cl) - return None + return cl def generate_match_set(self, situation: Perception, time_stamp): matching_ls = [cl for cl in self if cl.does_match(situation)] From fc8224ae05ad141c1d7cbf771daeb4c0d0572378 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Wed, 26 May 2021 12:15:07 +0200 Subject: [PATCH 20/33] BP tests --- tests/lcs/agents/xncs/test_Effect.py | 13 +++++++++++++ tests/lcs/agents/xncs/test_backpropagation.py | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/lcs/agents/xncs/test_Effect.py b/tests/lcs/agents/xncs/test_Effect.py index 4859da1f..a8f72db0 100644 --- a/tests/lcs/agents/xncs/test_Effect.py +++ b/tests/lcs/agents/xncs/test_Effect.py @@ -22,3 +22,16 @@ def test_subsumes(self, cond1, cond2, result): assert result == Effect(cond1).subsumes(cond2) assert result == Effect(cond1).subsumes(cond2) assert result == Effect(cond1).subsumes(cond2) + + @pytest.mark.parametrize("cond1, cond2, result", [ + ("1111", "1111", True), + ("11##", "1111", False), + ("1100", "1111", False), + ("1100", "0111", False), + ("1100", "0110", False), + ("11##", "11##", True), + ("11", "1111", False), + ("1111", "10##", False), + ]) + def test_equals(self, cond1, cond2, result): + assert result == (Effect(cond1) == Effect(cond2)) diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index ffa7cdc3..e145f3d5 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -9,7 +9,7 @@ class TestBackpropagation: @pytest.fixture def cfg(self): - return Configuration(lmc=2, lem=0.2, number_of_actions=4) + return Configuration(lmc=4, lem=20, number_of_actions=4) @pytest.fixture def situation(self): @@ -45,7 +45,17 @@ def test_update(self, cfg: Configuration, bp): cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0, effect=Effect("1100")) bp.insert_into_bp(cl, Effect("0110")) bp._update_bp() - assert cl.effect == Effect("0110") + # Current version of XNCS doesn't change classifiers Effect + # It is possible version of BP to see if is more effective + # assert cl.effect == Effect("0110") assert cl.error > cfg.initial_error + def test_check_for_updates_lcm(self, bp, classifiers_list_diff_actions): + for cl in classifiers_list_diff_actions: + bp.insert_into_bp(cl, Effect("0011")) + assert len(bp.classifiers_for_update) == 0 + + def test_check_for_updates_correct_effect(self, bp, classifiers_list_diff_actions, situation): + bp.insert_into_bp(classifiers_list_diff_actions[0], Effect(situation)) + assert len(bp.classifiers_for_update) == 0 From e80d73d028ad5d0cc290596533f23a14f8dc59bc Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 28 May 2021 09:54:24 +0200 Subject: [PATCH 21/33] Classifier Accuracy --- lcs/agents/xncs/Backpropagation.py | 2 ++ lcs/agents/xncs/Classifier.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index f57914a9..cf4b6a97 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -30,7 +30,9 @@ def _check_for_update(self): def _update_bp(self): for cl, uv in zip(self.classifiers_for_update, self.update_vectors): + cl.queses += 1 if cl.effect != uv: + cl.mistakes += 1 # cl.effect = copy(uv) cl.error += self.cfg.lem * cl.error self.classifiers_for_update = [] diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 2cffcacb..a2781588 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -14,6 +14,9 @@ def __init__(self, time_stamp: int = None, effect: Union[Effect, str, None] = None) -> None: self.effect = effect + self.mistakes = 0 + self.queses = 0 + super().__init__(cfg, condition, action, time_stamp) def __eq__(self, other): @@ -32,6 +35,15 @@ def __eq__(self, other): def __hash__(self): return hash((str(self.condition),str(self.effect), self.action)) + @property + def accuracy(self): + if self.queses > 0: + return (self.queses - self.mistakes) / self.queses + else: + return 0 + def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ - f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}]" + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, error:{self.error}]" + \ + f"acc: {self.accuracy}" + From ad66c09319a32ceea78ee51ec9fa230cf874e7c8 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 28 May 2021 14:17:54 +0200 Subject: [PATCH 22/33] Fixed Update in BP --- lcs/agents/xcs/ClassifiersList.py | 14 +++-- lcs/agents/xncs/Backpropagation.py | 62 ++++++++++--------- lcs/agents/xncs/Classifier.py | 20 ++---- lcs/agents/xncs/XNCS.py | 6 +- tests/lcs/agents/xncs/test_Classifier.py | 4 +- tests/lcs/agents/xncs/test_ClassifiersList.py | 20 ++++++ tests/lcs/agents/xncs/test_XNCS.py | 12 ++-- tests/lcs/agents/xncs/test_backpropagation.py | 27 -------- 8 files changed, 79 insertions(+), 86 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 23c68639..133b7743 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -5,7 +5,6 @@ from lcs import TypedList, Perception from lcs.agents.xcs import Classifier, Condition, Configuration - logger = logging.getLogger(__name__) @@ -148,10 +147,7 @@ def _update_fitness(self): if cl.error < self.cfg.epsilon_0: tmp_acc = 1 else: - tmp_acc = (self.cfg.alpha * - (cl.error * self.cfg.epsilon_0) ** - -self.cfg.v - ) + tmp_acc = (pow(self.cfg.alpha * (cl.error * self.cfg.epsilon_0), -self.cfg.v)) accuracy_vector_k.append(tmp_acc) accuracy_sum += tmp_acc + cl.numerosity for cl, k in zip(self, accuracy_vector_k): @@ -160,7 +156,15 @@ def _update_fitness(self): (k * cl.numerosity / accuracy_sum - cl.fitness) ) + def least_fit_classifiers(self, percentage): + assert 0 < percentage <= 1 + return sorted( + self, + key=lambda cl: cl.prediction * cl.fitness + )[0:int(len(self) * percentage)] + @property def fittest_classifier(self): assert len(self) > 0 return max(self, key=lambda cl: cl.fitness * cl.prediction) + diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index cf4b6a97..4ffc0207 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -1,41 +1,45 @@ from copy import copy -from lcs.agents.xncs import Configuration, Classifier, ClassifiersList, Effect +from lcs.agents.xncs import Configuration, Effect class Backpropagation: - def __init__(self, cfg: Configuration, - population: ClassifiersList): - self.population = population - self.classifiers_for_update = [] - self.update_vectors = [] + def __init__(self, + cfg: Configuration, + percentage: float): + self.percentage = percentage self.cfg = cfg self.update_cycles = 0 - def insert_into_bp(self, - classifier: Classifier, - update_vector: Effect): - self.classifiers_for_update.append(classifier) - self.update_vectors.append(update_vector) - self._check_for_update() - - def _check_for_update(self): - self.update_cycles += 1 - if self.update_cycles >= self.cfg.lmc: - self._update_bp() - # TODO: One of them is NoneType - elif self.classifiers_for_update[-1].effect == self.update_vectors[-1]: - self._update_bp() - - def _update_bp(self): - for cl, uv in zip(self.classifiers_for_update, self.update_vectors): + def update_cycle(self, + action_set, + update_vector: Effect): + if action_set.fittest_classifier.effect != update_vector: + self.update_cycles = self.cfg.lmc + + if self.update_cycles > 0: + if action_set.fittest_classifier.effect == update_vector: + self.update_cycles = 0 + else: + self.update_cycles -= 1 + self._update_classifiers_effect( + action_set.least_fit_classifiers(self.percentage), + update_vector + ) + self._update_classifiers_error( + action_set, + update_vector + ) + + def _update_classifiers_effect(self, classifiers_for_update, update_vector): + for cl in classifiers_for_update: + if cl.effect != update_vector: + cl.effect = copy(update_vector) + + def _update_classifiers_error(self, classifiers_for_update, update_vector): + for cl in classifiers_for_update: cl.queses += 1 - if cl.effect != uv: + if cl.effect != update_vector: cl.mistakes += 1 - # cl.effect = copy(uv) cl.error += self.cfg.lem * cl.error - self.classifiers_for_update = [] - self.update_vectors = [] - self.update_cycles = 0 - diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index a2781588..1929f75b 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -19,28 +19,18 @@ def __init__(self, super().__init__(cfg, condition, action, time_stamp) - def __eq__(self, other): - if other.action == self.action \ - and other.condition == self.condition: - if other.effect is None and self.effect is None: - return True - if other.effect is None: - return False - if self.effect is None: - return False - if other.effect == self.effect: - return True - return False - def __hash__(self): return hash((str(self.condition),str(self.effect), self.action)) @property def accuracy(self): if self.queses > 0: - return (self.queses - self.mistakes) / self.queses + accuracy = (self.queses - self.mistakes) / self.queses else: - return 0 + accuracy = 0 + self.mistakes = 0 + self.queses = 0 + return accuracy def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 9c752fbe..7c1d9331 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -32,7 +32,7 @@ def __init__(self, ) self.back_propagation = Backpropagation( cfg=self.cfg, - population=self.population + percentage=0.2 ) self.time_stamp = 0 self.reward = 0 @@ -43,7 +43,7 @@ def _distribute_and_update(self, action_set, situation, p): def _compare_effect(self, action_set: ClassifiersList, situation): if action_set is not None: - self.back_propagation.insert_into_bp( - action_set.fittest_classifier, + self.back_propagation.update_cycle( + action_set, Effect(situation) ) diff --git a/tests/lcs/agents/xncs/test_Classifier.py b/tests/lcs/agents/xncs/test_Classifier.py index b4f1c3fa..15d81827 100644 --- a/tests/lcs/agents/xncs/test_Classifier.py +++ b/tests/lcs/agents/xncs/test_Classifier.py @@ -54,8 +54,8 @@ def test_does_subsume(self, cfg: Configuration): ("1111", "11", 1, 1, False, "1111", "1111"), ("11", "1111", 1, 1, False, "1111", "1111"), - ("1111", "1111", 1, 1, False, "1111", "1100"), - ("1111", "1111", 1, 1, False, "1100", "1111"), + # ("1111", "1111", 1, 1, False, "1111", "1100"), + # ("1111", "1111", 1, 1, False, "1100", "1111"), ]) def test_equals(self, cfg, cond1, cond2, act1, act2, result, ef1, ef2): diff --git a/tests/lcs/agents/xncs/test_ClassifiersList.py b/tests/lcs/agents/xncs/test_ClassifiersList.py index ae66a9c3..7eda0081 100644 --- a/tests/lcs/agents/xncs/test_ClassifiersList.py +++ b/tests/lcs/agents/xncs/test_ClassifiersList.py @@ -54,3 +54,23 @@ def test_return_fittest(self, classifiers_list_diff_actions): classifiers_list_diff_actions[2].prediction = 20 classifiers_list_diff_actions[2].fitness = 20 assert id(classifiers_list_diff_actions.fittest_classifier) == id(classifiers_list_diff_actions[2]) + + def test_return_least_fit(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[0].prediction = 1250 + classifiers_list_diff_actions[1].prediction = 1000 + classifiers_list_diff_actions[2].prediction = 750 + classifiers_list_diff_actions[3].prediction = 0.01 + sort = classifiers_list_diff_actions.least_fit_classifiers(0.25) + assert sort[0] == classifiers_list_diff_actions[3] + assert len(sort) == 1 + + def test_return_multiple_least_fit(self, classifiers_list_diff_actions): + classifiers_list_diff_actions[0].prediction = 1250 + classifiers_list_diff_actions[1].prediction = 1000 + classifiers_list_diff_actions[2].prediction = 750 + classifiers_list_diff_actions[3].prediction = 0.01 + sort = classifiers_list_diff_actions.least_fit_classifiers(0.75) + assert sort[0] == classifiers_list_diff_actions[3] + assert sort[1] == classifiers_list_diff_actions[2] + assert sort[2] == classifiers_list_diff_actions[1] + assert len(sort) == 3 diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index e8204b3a..2c3b73ea 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -31,6 +31,12 @@ def test_init(self, cfg, classifiers_list_diff_actions): assert xncs.back_propagation is not None assert id(xncs.cfg) == id(cfg) assert len(xncs.population) == 4 + assert isinstance(classifiers_list_diff_actions, ClassifiersList) + assert isinstance(xncs.population, ClassifiersList) + + def test_init_no_pop(self, cfg): + xncs = XNCS(cfg) + assert isinstance(xncs.population, ClassifiersList) def test_distribute_and_update(self, cfg: Configuration, classifiers_list_diff_actions, @@ -41,8 +47,6 @@ def test_distribute_and_update(self, cfg: Configuration, assert action_set.fittest_classifier is not None xncs._distribute_and_update(action_set, situation, 0.1) # update should happen because effect matched inserted vector - assert len(xncs.back_propagation.update_vectors) == 0 - assert len(xncs.back_propagation.classifiers_for_update) == 0 assert xncs.back_propagation.update_cycles == 0 def test_distribute_and_update_diff(self, cfg: Configuration, @@ -53,9 +57,7 @@ def test_distribute_and_update_diff(self, cfg: Configuration, assert action_set is not None assert action_set.fittest_classifier is not None xncs._distribute_and_update(action_set, "####", 0.1) - assert len(xncs.back_propagation.update_vectors) == 1 - assert len(xncs.back_propagation.classifiers_for_update) == 1 - assert xncs.back_propagation.update_cycles == 1 + assert xncs.back_propagation.update_cycles == cfg.lmc - 1 def test_correct_type_population(self, cfg): xncs = XNCS(cfg) diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index e145f3d5..267a6da1 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -31,31 +31,4 @@ def bp(self, cfg, classifiers_list_diff_actions): def test_init(self, cfg, bp): assert id(bp.cfg) == id(cfg) - def test_insert(self, cfg, bp): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0, effect=Effect("1100")) - ef = Effect("0110") - bp.insert_into_bp(cl, ef) - assert id(bp.classifiers_for_update[0]) == id(cl) - assert id(bp.update_vectors[0]) == id(ef) - assert bp.classifiers_for_update[0] == cl - assert bp.update_vectors[0] == ef - assert bp.update_cycles > 0 - - def test_update(self, cfg: Configuration, bp): - cl = Classifier(cfg=cfg, condition=Condition("1111"), action=0, time_stamp=0, effect=Effect("1100")) - bp.insert_into_bp(cl, Effect("0110")) - bp._update_bp() - # Current version of XNCS doesn't change classifiers Effect - # It is possible version of BP to see if is more effective - # assert cl.effect == Effect("0110") - assert cl.error > cfg.initial_error - - def test_check_for_updates_lcm(self, bp, classifiers_list_diff_actions): - for cl in classifiers_list_diff_actions: - bp.insert_into_bp(cl, Effect("0011")) - assert len(bp.classifiers_for_update) == 0 - - def test_check_for_updates_correct_effect(self, bp, classifiers_list_diff_actions, situation): - bp.insert_into_bp(classifiers_list_diff_actions[0], Effect(situation)) - assert len(bp.classifiers_for_update) == 0 From e8ae627bce01e60c29821134f63e98abc7507eff Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 28 May 2021 14:23:21 +0200 Subject: [PATCH 23/33] Moved XNCS functions to XNCS ClLst --- lcs/agents/xcs/ClassifiersList.py | 11 ----------- lcs/agents/xncs/ClassifiersList.py | 22 +++++++++++++++++++++- lcs/agents/xncs/__init__.py | 4 +++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 133b7743..4c267c9d 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -156,15 +156,4 @@ def _update_fitness(self): (k * cl.numerosity / accuracy_sum - cl.fitness) ) - def least_fit_classifiers(self, percentage): - assert 0 < percentage <= 1 - return sorted( - self, - key=lambda cl: cl.prediction * cl.fitness - )[0:int(len(self) * percentage)] - - @property - def fittest_classifier(self): - assert len(self) > 0 - return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 1a2dd62a..b526787f 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -4,7 +4,7 @@ import lcs.agents.xcs as xcs from lcs.agents.xcs import Condition -from lcs.agents.xncs import Classifier, Configuration, Effect +from lcs.agents.xncs import Classifier, Configuration, Effect, Backpropagation logger = logging.getLogger(__name__) @@ -15,6 +15,10 @@ def __init__(self, *args, oktypes=(Classifier,), ) -> None: + self.back_propagation = Backpropagation( + cfg=cfg, + percentage=0.2 + ) super().__init__(cfg, *args, oktypes=oktypes) # without this function the Classifierlist will create XCS Classifiers @@ -34,3 +38,19 @@ def generate_covering_classifier(self, situation, action, time_stamp): time_stamp=time_stamp, effect=Effect(effect)) return cl + + def generate_action_set(self, action): + action_ls = [cl for cl in self if cl.action == action] + return ClassifiersList(self.cfg, *action_ls) + + def least_fit_classifiers(self, percentage): + assert 0 < percentage <= 1 + return sorted( + self, + key=lambda cl: cl.prediction * cl.fitness + )[0:int(len(self) * percentage)] + + @property + def fittest_classifier(self): + assert len(self) > 0 + return max(self, key=lambda cl: cl.fitness * cl.prediction) diff --git a/lcs/agents/xncs/__init__.py b/lcs/agents/xncs/__init__.py index 438c5a4f..da032557 100644 --- a/lcs/agents/xncs/__init__.py +++ b/lcs/agents/xncs/__init__.py @@ -1,8 +1,10 @@ from .Effect import Effect from .Configuration import Configuration from .Classifier import Classifier +from .Backpropagation import Backpropagation from .GeneticAlgorithm import GeneticAlgorithm + from .ClassifiersList import ClassifiersList -from .Backpropagation import Backpropagation + from .XNCS import XNCS From ca73d045f0b79f25a5d65df179698604a79f41cc Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 28 May 2021 14:58:55 +0200 Subject: [PATCH 24/33] Moved update near action_set creation --- lcs/agents/xcs/XCS.py | 12 ++++++++---- lcs/agents/xncs/ClassifiersList.py | 10 ++++++++++ lcs/agents/xncs/XNCS.py | 14 +++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index 491849c0..5658eafe 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -63,10 +63,7 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: assert len(self.population) == len(set(self.population)), 'duplicates found' self.population.delete_from_population() # We are in t+1 here - match_set = self.population.generate_match_set(state, self.time_stamp) - prediction_array = match_set.prediction_array - action = self.select_action(prediction_array, match_set) - action_set = match_set.generate_action_set(action) + action_set, prediction_array, action = self._form_sets_and_choose_action(state) # apply action to environment raw_state, step_reward, done, _ = env.step(action) state = self.cfg.environment_adapter.to_genotype(raw_state) @@ -87,6 +84,13 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: self.time_stamp += 1 return TrialMetrics(self.time_stamp - prev_time_stamp, self.reward) + def _form_sets_and_choose_action(self, state): + match_set = self.population.generate_match_set(state, self.time_stamp) + prediction_array = match_set.prediction_array + action = self.select_action(prediction_array, match_set) + action_set = match_set.generate_action_set(action) + return action_set, prediction_array, action + def _distribute_and_update(self, action_set, situation, p): if action_set is not None and len(action_set) > 0: action_set.update_set(p) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index b526787f..95487da1 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -3,6 +3,7 @@ import random import lcs.agents.xcs as xcs +from lcs import Perception from lcs.agents.xcs import Condition from lcs.agents.xncs import Classifier, Configuration, Effect, Backpropagation logger = logging.getLogger(__name__) @@ -43,6 +44,15 @@ def generate_action_set(self, action): action_ls = [cl for cl in self if cl.action == action] return ClassifiersList(self.cfg, *action_ls) + def generate_match_set(self, situation: Perception, time_stamp): + matching_ls = [cl for cl in self if cl.does_match(situation)] + action = self._find_not_present_action(matching_ls) + while action is not None: + cl = self._generate_covering_and_insert(situation, action, time_stamp) + matching_ls.append(cl) + action = self._find_not_present_action(matching_ls) + return ClassifiersList(self.cfg, *matching_ls) + def least_fit_classifiers(self, percentage): assert 0 < percentage <= 1 return sorted( diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 7c1d9331..fd646a5f 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -32,16 +32,20 @@ def __init__(self, ) self.back_propagation = Backpropagation( cfg=self.cfg, - percentage=0.2 + percentage=0.1 ) self.time_stamp = 0 self.reward = 0 - def _distribute_and_update(self, action_set, situation, p): - super()._distribute_and_update(action_set, situation, p) - self._compare_effect(action_set, situation) + def _form_sets_and_choose_action(self, state): + match_set = self.population.generate_match_set(state, self.time_stamp) + prediction_array = match_set.prediction_array + action = self.select_action(prediction_array, match_set) + action_set = match_set.generate_action_set(action) + self._insert_into_bp(action_set, state) + return action_set, prediction_array, action - def _compare_effect(self, action_set: ClassifiersList, situation): + def _insert_into_bp(self, action_set: ClassifiersList, situation): if action_set is not None: self.back_propagation.update_cycle( action_set, From 1b6a297b5e22f84c3a3cc30cdf454881d9840aa4 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Sat, 29 May 2021 09:23:52 +0200 Subject: [PATCH 25/33] Split of update effect and error --- lcs/agents/xncs/Backpropagation.py | 10 +++------- lcs/agents/xncs/XNCS.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 4ffc0207..2eceb437 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -23,21 +23,17 @@ def update_cycle(self, self.update_cycles = 0 else: self.update_cycles -= 1 - self._update_classifiers_effect( - action_set.least_fit_classifiers(self.percentage), - update_vector - ) - self._update_classifiers_error( + self.update_classifiers_error( action_set, update_vector ) - def _update_classifiers_effect(self, classifiers_for_update, update_vector): + def update_classifiers_effect(self, classifiers_for_update, update_vector): for cl in classifiers_for_update: if cl.effect != update_vector: cl.effect = copy(update_vector) - def _update_classifiers_error(self, classifiers_for_update, update_vector): + def update_classifiers_error(self, classifiers_for_update, update_vector): for cl in classifiers_for_update: cl.queses += 1 if cl.effect != update_vector: diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index fd646a5f..5c5d430b 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -42,12 +42,18 @@ def _form_sets_and_choose_action(self, state): prediction_array = match_set.prediction_array action = self.select_action(prediction_array, match_set) action_set = match_set.generate_action_set(action) - self._insert_into_bp(action_set, state) + if action_set is not None: + self.back_propagation.update_classifiers_effect( + action_set.least_fit_classifiers(0.2), + action_set.fittest_classifier.effect + ) return action_set, prediction_array, action - def _insert_into_bp(self, action_set: ClassifiersList, situation): + def _distribute_and_update(self, action_set, situation, p): if action_set is not None: self.back_propagation.update_cycle( action_set, Effect(situation) ) + super()._distribute_and_update(action_set, situation, p) + From c1299be960e0f9c1e5f5178b9d2a50e8f9578d40 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Sun, 30 May 2021 09:02:02 +0200 Subject: [PATCH 26/33] XNCS BP fixed LMC --- lcs/agents/xcs/XCS.py | 6 ++- lcs/agents/xncs/Backpropagation.py | 54 ++++++++++--------- lcs/agents/xncs/Classifier.py | 8 +-- lcs/agents/xncs/ClassifiersList.py | 2 +- lcs/agents/xncs/XNCS.py | 14 ++--- tests/lcs/agents/xncs/test_ClassifiersList.py | 16 +++--- tests/lcs/agents/xncs/test_XNCS.py | 7 ++- tests/lcs/agents/xncs/test_backpropagation.py | 35 +++++++++--- 8 files changed, 80 insertions(+), 62 deletions(-) diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index 5658eafe..cb562ce4 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -72,9 +72,11 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: self._distribute_and_update(prev_action_set, prev_state, + state, prev_reward + self.cfg.gamma * max(prediction_array)) if done: self._distribute_and_update(action_set, + state, state, self.reward) else: @@ -91,14 +93,14 @@ def _form_sets_and_choose_action(self, state): action_set = match_set.generate_action_set(action) return action_set, prediction_array, action - def _distribute_and_update(self, action_set, situation, p): + def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None and len(action_set) > 0: action_set.update_set(p) if self.cfg.do_action_set_subsumption: self.do_action_set_subsumption(action_set) self.ga.run_ga( action_set, - situation, + current_situation, self.time_stamp ) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 2eceb437..7765ca36 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -10,32 +10,38 @@ def __init__(self, percentage: float): self.percentage = percentage self.cfg = cfg - self.update_cycles = 0 + self.classifiers_for_update = [] def update_cycle(self, action_set, - update_vector: Effect): - if action_set.fittest_classifier.effect != update_vector: - self.update_cycles = self.cfg.lmc - - if self.update_cycles > 0: - if action_set.fittest_classifier.effect == update_vector: - self.update_cycles = 0 - else: - self.update_cycles -= 1 - self.update_classifiers_error( - action_set, - update_vector - ) - - def update_classifiers_effect(self, classifiers_for_update, update_vector): - for cl in classifiers_for_update: - if cl.effect != update_vector: - cl.effect = copy(update_vector) - - def update_classifiers_error(self, classifiers_for_update, update_vector): - for cl in classifiers_for_update: + next_vector: Effect): + # for cl in action_set.least_fit_classifiers(self.percentage): + for cl in action_set: cl.queses += 1 - if cl.effect != update_vector: + if cl.effect != next_vector: cl.mistakes += 1 - cl.error += self.cfg.lem * cl.error + if not any(cl == inside[0] for inside in self.classifiers_for_update): + self.classifiers_for_update.append( + [cl, next_vector, self.cfg.lmc] + ) + self.check_if_needed() + self.update_errors() + + def check_if_needed(self): + for cl in self.classifiers_for_update: + if cl[0].effect == cl[1]: + self.classifiers_for_update.remove(cl) + else: + cl[2] -= 1 + if cl[2] == 0: + self.classifiers_for_update.remove(cl) + + def update_errors(self): + for cl in self.classifiers_for_update: + cl[0].error += self.cfg.lem + + def update_effect(self, + action_set): + effect = action_set.fittest_classifier.effect + for cl in action_set.least_fit_classifiers(self.percentage): + cl.effect = copy(effect) diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 1929f75b..ff8fa5f3 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -25,12 +25,8 @@ def __hash__(self): @property def accuracy(self): if self.queses > 0: - accuracy = (self.queses - self.mistakes) / self.queses - else: - accuracy = 0 - self.mistakes = 0 - self.queses = 0 - return accuracy + return (self.queses - self.mistakes) / self.queses + return 0 def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 95487da1..9c860111 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -57,7 +57,7 @@ def least_fit_classifiers(self, percentage): assert 0 < percentage <= 1 return sorted( self, - key=lambda cl: cl.prediction * cl.fitness + key=lambda cl: cl.fitness )[0:int(len(self) * percentage)] @property diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 5c5d430b..6cb62bee 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -32,7 +32,7 @@ def __init__(self, ) self.back_propagation = Backpropagation( cfg=self.cfg, - percentage=0.1 + percentage=0.05 ) self.time_stamp = 0 self.reward = 0 @@ -42,18 +42,14 @@ def _form_sets_and_choose_action(self, state): prediction_array = match_set.prediction_array action = self.select_action(prediction_array, match_set) action_set = match_set.generate_action_set(action) - if action_set is not None: - self.back_propagation.update_classifiers_effect( - action_set.least_fit_classifiers(0.2), - action_set.fittest_classifier.effect - ) + self.back_propagation.update_effect(action_set) return action_set, prediction_array, action - def _distribute_and_update(self, action_set, situation, p): + def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: self.back_propagation.update_cycle( action_set, - Effect(situation) + Effect(next_situation) ) - super()._distribute_and_update(action_set, situation, p) + super()._distribute_and_update(action_set, current_situation, next_situation, p) diff --git a/tests/lcs/agents/xncs/test_ClassifiersList.py b/tests/lcs/agents/xncs/test_ClassifiersList.py index 7eda0081..11f10384 100644 --- a/tests/lcs/agents/xncs/test_ClassifiersList.py +++ b/tests/lcs/agents/xncs/test_ClassifiersList.py @@ -56,19 +56,19 @@ def test_return_fittest(self, classifiers_list_diff_actions): assert id(classifiers_list_diff_actions.fittest_classifier) == id(classifiers_list_diff_actions[2]) def test_return_least_fit(self, classifiers_list_diff_actions): - classifiers_list_diff_actions[0].prediction = 1250 - classifiers_list_diff_actions[1].prediction = 1000 - classifiers_list_diff_actions[2].prediction = 750 - classifiers_list_diff_actions[3].prediction = 0.01 + classifiers_list_diff_actions[0].fitness = 1250 + classifiers_list_diff_actions[1].fitness = 1000 + classifiers_list_diff_actions[2].fitness = 750 + classifiers_list_diff_actions[3].fitness = 0.01 sort = classifiers_list_diff_actions.least_fit_classifiers(0.25) assert sort[0] == classifiers_list_diff_actions[3] assert len(sort) == 1 def test_return_multiple_least_fit(self, classifiers_list_diff_actions): - classifiers_list_diff_actions[0].prediction = 1250 - classifiers_list_diff_actions[1].prediction = 1000 - classifiers_list_diff_actions[2].prediction = 750 - classifiers_list_diff_actions[3].prediction = 0.01 + classifiers_list_diff_actions[0].fitness = 1250 + classifiers_list_diff_actions[1].fitness = 1000 + classifiers_list_diff_actions[2].fitness = 750 + classifiers_list_diff_actions[3].fitness = 0.01 sort = classifiers_list_diff_actions.least_fit_classifiers(0.75) assert sort[0] == classifiers_list_diff_actions[3] assert sort[1] == classifiers_list_diff_actions[2] diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index 2c3b73ea..1cf363ac 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -45,9 +45,9 @@ def test_distribute_and_update(self, cfg: Configuration, action_set = xncs.population.generate_action_set(0) assert action_set is not None assert action_set.fittest_classifier is not None - xncs._distribute_and_update(action_set, situation, 0.1) + xncs._distribute_and_update(action_set, "####", "####", 0.1) # update should happen because effect matched inserted vector - assert xncs.back_propagation.update_cycles == 0 + assert len(xncs.back_propagation.classifiers_for_update) == 1 def test_distribute_and_update_diff(self, cfg: Configuration, classifiers_list_diff_actions, @@ -56,8 +56,7 @@ def test_distribute_and_update_diff(self, cfg: Configuration, action_set = xncs.population.generate_action_set(0) assert action_set is not None assert action_set.fittest_classifier is not None - xncs._distribute_and_update(action_set, "####", 0.1) - assert xncs.back_propagation.update_cycles == cfg.lmc - 1 + xncs._distribute_and_update(action_set, "####", "####", 0.1) def test_correct_type_population(self, cfg): xncs = XNCS(cfg) diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 267a6da1..3a60b1ff 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -24,11 +24,30 @@ def classifiers_list_diff_actions(self, cfg, situation): classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) return classifiers_list - @pytest.fixture - def bp(self, cfg, classifiers_list_diff_actions): - return Backpropagation(cfg, classifiers_list_diff_actions) - - def test_init(self, cfg, bp): - assert id(bp.cfg) == id(cfg) - - + def test_update_effect(self, cfg, classifiers_list_diff_actions): + classifiers_list_diff_actions[0].fitness = 1000 + classifiers_list_diff_actions[1].fitness = 0 + classifiers_list_diff_actions[2].fitness = 0 + bp = Backpropagation(cfg, 0.5) + bp.update_effect(classifiers_list_diff_actions) + assert classifiers_list_diff_actions[1].effect == classifiers_list_diff_actions[0].effect + + def test_insertion(self, cfg, classifiers_list_diff_actions): + bp = Backpropagation(cfg, 0.5) + bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + + def test_deletion(self, cfg, classifiers_list_diff_actions): + bp = Backpropagation(cfg, 0.5) + bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 4 + bp.classifiers_for_update[1][2] = 1 + bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + assert len(bp.classifiers_for_update) == 3 + + def test_errors(self, cfg: Configuration, classifiers_list_diff_actions): + bp = Backpropagation(cfg, 0.5) + bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + assert classifiers_list_diff_actions[0].error != cfg.initial_error From 14a08f872dc12883cfd83f779f7901a0546a1f14 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Sun, 30 May 2021 17:31:41 +0200 Subject: [PATCH 27/33] From situation instead of best [A] --- lcs/agents/xncs/Backpropagation.py | 5 +++-- lcs/agents/xncs/Classifier.py | 2 +- lcs/agents/xncs/XNCS.py | 4 ++-- tests/lcs/agents/xncs/test_XNCS.py | 4 +++- tests/lcs/agents/xncs/test_backpropagation.py | 1 + 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 7765ca36..7b51b6b1 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -41,7 +41,8 @@ def update_errors(self): cl[0].error += self.cfg.lem def update_effect(self, - action_set): - effect = action_set.fittest_classifier.effect + action_set, next_situation): + # effect = action_set.fittest_classifier.effect + effect = Effect(next_situation) for cl in action_set.least_fit_classifiers(self.percentage): cl.effect = copy(effect) diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index ff8fa5f3..1f27f645 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -26,7 +26,7 @@ def __hash__(self): def accuracy(self): if self.queses > 0: return (self.queses - self.mistakes) / self.queses - return 0 + return None def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 6cb62bee..17dabb4c 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -32,7 +32,7 @@ def __init__(self, ) self.back_propagation = Backpropagation( cfg=self.cfg, - percentage=0.05 + percentage=0.1 ) self.time_stamp = 0 self.reward = 0 @@ -42,11 +42,11 @@ def _form_sets_and_choose_action(self, state): prediction_array = match_set.prediction_array action = self.select_action(prediction_array, match_set) action_set = match_set.generate_action_set(action) - self.back_propagation.update_effect(action_set) return action_set, prediction_array, action def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: + self.back_propagation.update_effect(action_set, next_situation) self.back_propagation.update_cycle( action_set, Effect(next_situation) diff --git a/tests/lcs/agents/xncs/test_XNCS.py b/tests/lcs/agents/xncs/test_XNCS.py index 1cf363ac..64eb535a 100644 --- a/tests/lcs/agents/xncs/test_XNCS.py +++ b/tests/lcs/agents/xncs/test_XNCS.py @@ -56,7 +56,8 @@ def test_distribute_and_update_diff(self, cfg: Configuration, action_set = xncs.population.generate_action_set(0) assert action_set is not None assert action_set.fittest_classifier is not None - xncs._distribute_and_update(action_set, "####", "####", 0.1) + xncs._distribute_and_update(action_set, situation, "1111", 0.1) + assert len(xncs.back_propagation.classifiers_for_update) == 1 def test_correct_type_population(self, cfg): xncs = XNCS(cfg) @@ -69,3 +70,4 @@ def test_correct_type_classifier(self, cfg): ) assert isinstance(xncs.population[0], Classifier) assert xncs.population[0].effect is None + diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 3a60b1ff..bf7e429a 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -51,3 +51,4 @@ def test_errors(self, cfg: Configuration, classifiers_list_diff_actions): bp = Backpropagation(cfg, 0.5) bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) assert classifiers_list_diff_actions[0].error != cfg.initial_error + From 5609be79b4d1b71ba469431aded5b08f816afc2e Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Mon, 31 May 2021 11:55:20 +0200 Subject: [PATCH 28/33] Child values fix --- lcs/agents/xncs/Backpropagation.py | 1 - lcs/agents/xncs/GeneticAlgorithm.py | 6 ++++++ lcs/agents/xncs/XNCS.py | 25 ++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 7b51b6b1..29951bc7 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -15,7 +15,6 @@ def __init__(self, def update_cycle(self, action_set, next_vector: Effect): - # for cl in action_set.least_fit_classifiers(self.percentage): for cl in action_set: cl.queses += 1 if cl.effect != next_vector: diff --git a/lcs/agents/xncs/GeneticAlgorithm.py b/lcs/agents/xncs/GeneticAlgorithm.py index 21eb93f8..04e6c825 100644 --- a/lcs/agents/xncs/GeneticAlgorithm.py +++ b/lcs/agents/xncs/GeneticAlgorithm.py @@ -24,6 +24,9 @@ def _make_children(self, parent1, parent2, time_stamp): time_stamp, copy(parent1.effect) ) + child1.prediction = parent1.prediction + child1.error = parent1.error + child1.fitness = parent1.fitness child2 = Classifier( self.cfg, @@ -32,5 +35,8 @@ def _make_children(self, parent1, parent2, time_stamp): time_stamp, copy(parent2.effect) ) + child2.prediction = parent2.prediction + child2.error = parent2.error + child2.fitness = parent2.fitness return child1, child2 diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 17dabb4c..73f48770 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -2,7 +2,7 @@ import random import numpy as np from copy import copy - +import queue from lcs.agents.xcs import XCS from lcs.agents.Agent import TrialMetrics from lcs.agents.xncs import Configuration, Backpropagation @@ -36,6 +36,7 @@ def __init__(self, ) self.time_stamp = 0 self.reward = 0 + self.mistakes = [] def _form_sets_and_choose_action(self, state): match_set = self.population.generate_match_set(state, self.time_stamp) @@ -46,6 +47,7 @@ def _form_sets_and_choose_action(self, state): def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: + self.update_fraction_accuracy(action_set, next_situation) self.back_propagation.update_effect(action_set, next_situation) self.back_propagation.update_cycle( action_set, @@ -53,3 +55,24 @@ def _distribute_and_update(self, action_set, current_situation, next_situation, ) super()._distribute_and_update(action_set, current_situation, next_situation, p) + def update_fraction_accuracy(self, action_set, next_vector): + most_numerous = sorted(action_set, key=lambda cl: -1 * cl.numerosity)[0] + if most_numerous.effect != Effect(next_vector): + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(1) + else: + self.mistakes.append(1) + else: + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(0) + else: + self.mistakes.append(0) + + @property + def fraction_accuracy(self): + if len(self.mistakes) > 0: + return sum(self.mistakes) / len(self.mistakes) + else: + return 0 From cc96ca626a50c5405def3497eb55cfa72ef3d124 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 10 Jun 2021 11:53:29 +0200 Subject: [PATCH 29/33] Refractor and XNCS modifications --- lcs/agents/xcs/Configuration.py | 2 - lcs/agents/xcs/GeneticAlgorithm.py | 9 ---- lcs/agents/xncs/Backpropagation.py | 8 +-- lcs/agents/xncs/Classifier.py | 6 +-- lcs/agents/xncs/ClassifiersList.py | 6 ++- lcs/agents/xncs/Configuration.py | 9 +++- lcs/agents/xncs/XNCS.py | 52 +++++++++++++++++-- lcs/agents/xncs/__init__.py | 3 -- tests/lcs/agents/xncs/test_backpropagation.py | 29 ++++++++--- 9 files changed, 89 insertions(+), 35 deletions(-) diff --git a/lcs/agents/xcs/Configuration.py b/lcs/agents/xcs/Configuration.py index 9bfbdb72..89fb9925 100644 --- a/lcs/agents/xcs/Configuration.py +++ b/lcs/agents/xcs/Configuration.py @@ -77,10 +77,8 @@ def __init__(self, self.number_of_actions = number_of_actions self.do_GA_subsumption = do_ga_subsumption self.do_action_set_subsumption = do_action_set_subsumption - self.metrics_trial_frequency = metrics_trial_frequency self.user_metrics_collector_fcn = user_metrics_collector_fcn - self.multistep_enfiroment = multistep_enfiroment def __str__(self) -> str: diff --git a/lcs/agents/xcs/GeneticAlgorithm.py b/lcs/agents/xcs/GeneticAlgorithm.py index dd15809c..e365db64 100644 --- a/lcs/agents/xcs/GeneticAlgorithm.py +++ b/lcs/agents/xcs/GeneticAlgorithm.py @@ -12,7 +12,6 @@ def __init__( population: ClassifiersList, cfg: Configuration ): - self.population = population self.cfg = cfg @@ -23,26 +22,19 @@ def run_ga(self, if action_set is None: return - # sometimes action set is empty, which is expected assert isinstance(action_set, ClassifiersList) - if time_stamp - (sum(cl.time_stamp * cl.numerosity for cl in action_set) / (sum(cl.numerosity for cl in action_set) or 1)) > self.cfg.ga_threshold: for cl in action_set: cl.time_stamp = time_stamp - parent1 = self._select_offspring(action_set) parent2 = self._select_offspring(action_set) - child1, child2 = self._make_children(parent1, parent2, time_stamp) - if np.random.rand() < self.cfg.chi: self._apply_crossover(child1, child2, parent1, parent2) - self._apply_mutation(child1, self.cfg, situation) self._apply_mutation(child2, self.cfg, situation) - self._perform_insertion_or_subsumption( child1, child2, parent1, parent2 @@ -87,7 +79,6 @@ def _make_children(self, parent1, parent2, time_stamp): @staticmethod def _select_offspring(action_set: ClassifiersList) -> Classifier: - assert isinstance(action_set, ClassifiersList) # TODO: insert generator to calculate fitness_sum diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 29951bc7..f9acfc8a 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -12,11 +12,11 @@ def __init__(self, self.cfg = cfg self.classifiers_for_update = [] - def update_cycle(self, - action_set, - next_vector: Effect): + def run_bp(self, + action_set, + next_vector: Effect): for cl in action_set: - cl.queses += 1 + cl.guesses += 1 if cl.effect != next_vector: cl.mistakes += 1 if not any(cl == inside[0] for inside in self.classifiers_for_update): diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 1f27f645..520152b6 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -15,7 +15,7 @@ def __init__(self, effect: Union[Effect, str, None] = None) -> None: self.effect = effect self.mistakes = 0 - self.queses = 0 + self.guesses = 0 super().__init__(cfg, condition, action, time_stamp) @@ -24,8 +24,8 @@ def __hash__(self): @property def accuracy(self): - if self.queses > 0: - return (self.queses - self.mistakes) / self.queses + if self.guesses > 0: + return (self.guesses - self.mistakes) / self.guesses return None def __str__(self): diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 9c860111..a71d5529 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -33,11 +33,15 @@ def generate_covering_classifier(self, situation, action, time_stamp): else: generalized.append(situation[i]) effect.append(str(random.choice(situation))) + if self.cfg.cover_env_input: + effect = None + else: + effect = Effect(effect) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, time_stamp=time_stamp, - effect=Effect(effect)) + effect=effect) return cl def generate_action_set(self, action): diff --git a/lcs/agents/xncs/Configuration.py b/lcs/agents/xncs/Configuration.py index c16a0163..d1a2655f 100644 --- a/lcs/agents/xncs/Configuration.py +++ b/lcs/agents/xncs/Configuration.py @@ -36,10 +36,17 @@ def __init__(self, do_action_set_subsumption: bool = False, metrics_trial_frequency: int = 5, user_metrics_collector_fcn: Callable = None, - multistep_enfiroment: bool = True + multistep_enfiroment: bool = True, + update_percentage: float = 0.2, + update_env_input: bool = False, + cover_env_input: bool = False, ) -> None: self.lmc = lmc self.lem = lem + self.update_env_input = update_env_input + self.cover_env_input = cover_env_input + self.update_percentage = update_percentage + super().__init__( number_of_actions=number_of_actions, classifier_wildcard=classifier_wildcard, diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 73f48770..7e87cea6 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -32,24 +32,68 @@ def __init__(self, ) self.back_propagation = Backpropagation( cfg=self.cfg, - percentage=0.1 + percentage=self.cfg.update_percentage ) self.time_stamp = 0 self.reward = 0 self.mistakes = [] + def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: + prev_action_set = None + prev_reward = self.reward + prev_state = None # state is known as situation + prev_time_stamp = self.time_stamp # steps + done = False # eop + + raw_state = env.reset() + state = self.cfg.environment_adapter.to_genotype(raw_state) + + while not done: + assert len(self.population) == len(set(self.population)), 'duplicates found' + self.population.delete_from_population() + # We are in t+1 here + action_set, prediction_array, action, match_set = self._form_sets_and_choose_action(state) + # apply action to environment + raw_state, step_reward, done, _ = env.step(action) + state = self.cfg.environment_adapter.to_genotype(raw_state) + + if self.cfg.multistep_enfiroment: + self.reward = step_reward + self.cfg.gamma * self.reward + + self._distribute_and_update(prev_action_set, + prev_state, + state, + prev_reward + self.cfg.gamma * max(prediction_array)) + if done: + self._distribute_and_update(action_set, + state, + state, + self.reward) + else: + prev_action_set = copy(action_set) + prev_reward = self.reward + prev_state = copy(state) + self.time_stamp += 1 + return TrialMetrics(self.time_stamp - prev_time_stamp, self.reward) + def _form_sets_and_choose_action(self, state): match_set = self.population.generate_match_set(state, self.time_stamp) prediction_array = match_set.prediction_array action = self.select_action(prediction_array, match_set) action_set = match_set.generate_action_set(action) - return action_set, prediction_array, action + return action_set, prediction_array, action, match_set def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: + for cl in action_set: + if cl.effect is None: + cl.effect = Effect(next_situation) self.update_fraction_accuracy(action_set, next_situation) - self.back_propagation.update_effect(action_set, next_situation) - self.back_propagation.update_cycle( + if self.cfg.update_env_input: + self.back_propagation.update_effect(action_set, next_situation) + else: + self.back_propagation.update_effect(action_set, action_set.fittest_classifier.effect) + self.back_propagation.run_bp( action_set, Effect(next_situation) ) diff --git a/lcs/agents/xncs/__init__.py b/lcs/agents/xncs/__init__.py index da032557..42342b47 100644 --- a/lcs/agents/xncs/__init__.py +++ b/lcs/agents/xncs/__init__.py @@ -2,9 +2,6 @@ from .Configuration import Configuration from .Classifier import Classifier from .Backpropagation import Backpropagation - from .GeneticAlgorithm import GeneticAlgorithm - from .ClassifiersList import ClassifiersList - from .XNCS import XNCS diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index bf7e429a..34fd7000 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -15,6 +15,10 @@ def cfg(self): def situation(self): return "1100" + @pytest.fixture + def next_situation(self): + return "1111" + @pytest.fixture def classifiers_list_diff_actions(self, cfg, situation): classifiers_list = ClassifiersList(cfg) @@ -24,31 +28,40 @@ def classifiers_list_diff_actions(self, cfg, situation): classifiers_list.insert_in_population(Classifier(cfg, Condition(situation), 3, 0, Effect(situation))) return classifiers_list - def test_update_effect(self, cfg, classifiers_list_diff_actions): + def test_update_effect_from_action_set(self, cfg, classifiers_list_diff_actions, next_situation): + classifiers_list_diff_actions[0].fitness = 1000 + classifiers_list_diff_actions[0].effect = Effect(next_situation) + classifiers_list_diff_actions[1].fitness = 0 + classifiers_list_diff_actions[2].fitness = 0 + bp = Backpropagation(cfg, 0.5) + bp.update_effect(classifiers_list_diff_actions, classifiers_list_diff_actions.fittest_classifier.effect) + assert classifiers_list_diff_actions[1].effect == classifiers_list_diff_actions.fittest_classifier.effect + + def test_update_effect(self, cfg, classifiers_list_diff_actions, next_situation): classifiers_list_diff_actions[0].fitness = 1000 classifiers_list_diff_actions[1].fitness = 0 classifiers_list_diff_actions[2].fitness = 0 bp = Backpropagation(cfg, 0.5) - bp.update_effect(classifiers_list_diff_actions) - assert classifiers_list_diff_actions[1].effect == classifiers_list_diff_actions[0].effect + bp.update_effect(classifiers_list_diff_actions, next_situation) + assert classifiers_list_diff_actions[1].effect == Effect(next_situation) def test_insertion(self, cfg, classifiers_list_diff_actions): bp = Backpropagation(cfg, 0.5) - bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 - bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 def test_deletion(self, cfg, classifiers_list_diff_actions): bp = Backpropagation(cfg, 0.5) - bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 bp.classifiers_for_update[1][2] = 1 - bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 3 def test_errors(self, cfg: Configuration, classifiers_list_diff_actions): bp = Backpropagation(cfg, 0.5) - bp.update_cycle(classifiers_list_diff_actions, Effect("1111")) + bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert classifiers_list_diff_actions[0].error != cfg.initial_error From 86108dc769e09f525dc0772ef35e852caa96c85c Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Thu, 10 Jun 2021 11:58:14 +0200 Subject: [PATCH 30/33] Moved update percentage to solely cfg --- lcs/agents/xncs/Backpropagation.py | 6 ++---- lcs/agents/xncs/ClassifiersList.py | 4 ---- lcs/agents/xncs/XNCS.py | 3 +-- tests/lcs/agents/xncs/test_backpropagation.py | 13 +++++++------ 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index f9acfc8a..f0ee149d 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -6,9 +6,7 @@ class Backpropagation: def __init__(self, - cfg: Configuration, - percentage: float): - self.percentage = percentage + cfg: Configuration): self.cfg = cfg self.classifiers_for_update = [] @@ -43,5 +41,5 @@ def update_effect(self, action_set, next_situation): # effect = action_set.fittest_classifier.effect effect = Effect(next_situation) - for cl in action_set.least_fit_classifiers(self.percentage): + for cl in action_set.least_fit_classifiers(self.cfg.update_percentage): cl.effect = copy(effect) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index a71d5529..888c901b 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -16,10 +16,6 @@ def __init__(self, *args, oktypes=(Classifier,), ) -> None: - self.back_propagation = Backpropagation( - cfg=cfg, - percentage=0.2 - ) super().__init__(cfg, *args, oktypes=oktypes) # without this function the Classifierlist will create XCS Classifiers diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 7e87cea6..088ba562 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -31,8 +31,7 @@ def __init__(self, cfg=self.cfg ) self.back_propagation = Backpropagation( - cfg=self.cfg, - percentage=self.cfg.update_percentage + cfg=self.cfg ) self.time_stamp = 0 self.reward = 0 diff --git a/tests/lcs/agents/xncs/test_backpropagation.py b/tests/lcs/agents/xncs/test_backpropagation.py index 34fd7000..4d2388b2 100644 --- a/tests/lcs/agents/xncs/test_backpropagation.py +++ b/tests/lcs/agents/xncs/test_backpropagation.py @@ -9,7 +9,7 @@ class TestBackpropagation: @pytest.fixture def cfg(self): - return Configuration(lmc=4, lem=20, number_of_actions=4) + return Configuration(lmc=4, lem=20, number_of_actions=4, update_percentage=0.5) @pytest.fixture def situation(self): @@ -33,7 +33,8 @@ def test_update_effect_from_action_set(self, cfg, classifiers_list_diff_actions, classifiers_list_diff_actions[0].effect = Effect(next_situation) classifiers_list_diff_actions[1].fitness = 0 classifiers_list_diff_actions[2].fitness = 0 - bp = Backpropagation(cfg, 0.5) + + bp = Backpropagation(cfg) bp.update_effect(classifiers_list_diff_actions, classifiers_list_diff_actions.fittest_classifier.effect) assert classifiers_list_diff_actions[1].effect == classifiers_list_diff_actions.fittest_classifier.effect @@ -41,19 +42,19 @@ def test_update_effect(self, cfg, classifiers_list_diff_actions, next_situation) classifiers_list_diff_actions[0].fitness = 1000 classifiers_list_diff_actions[1].fitness = 0 classifiers_list_diff_actions[2].fitness = 0 - bp = Backpropagation(cfg, 0.5) + bp = Backpropagation(cfg) bp.update_effect(classifiers_list_diff_actions, next_situation) assert classifiers_list_diff_actions[1].effect == Effect(next_situation) def test_insertion(self, cfg, classifiers_list_diff_actions): - bp = Backpropagation(cfg, 0.5) + bp = Backpropagation(cfg) bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 def test_deletion(self, cfg, classifiers_list_diff_actions): - bp = Backpropagation(cfg, 0.5) + bp = Backpropagation(cfg) bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert len(bp.classifiers_for_update) == 4 bp.classifiers_for_update[1][2] = 1 @@ -61,7 +62,7 @@ def test_deletion(self, cfg, classifiers_list_diff_actions): assert len(bp.classifiers_for_update) == 3 def test_errors(self, cfg: Configuration, classifiers_list_diff_actions): - bp = Backpropagation(cfg, 0.5) + bp = Backpropagation(cfg) bp.run_bp(classifiers_list_diff_actions, Effect("1111")) assert classifiers_list_diff_actions[0].error != cfg.initial_error From 9527cbc21c88c69f29fd3720d6a2beb291263f2d Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Fri, 11 Jun 2021 10:16:34 +0200 Subject: [PATCH 31/33] Removal of unneeded code --- lcs/agents/xncs/Backpropagation.py | 4 +-- lcs/agents/xncs/ClassifiersList.py | 7 ++--- lcs/agents/xncs/XNCS.py | 45 ------------------------------ tests/lcs/agents/xcs/__init__.py | 1 - tests/lcs/agents/xncs/__init__.py | 2 +- 5 files changed, 4 insertions(+), 55 deletions(-) diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index f0ee149d..77bf3609 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -37,9 +37,7 @@ def update_errors(self): for cl in self.classifiers_for_update: cl[0].error += self.cfg.lem - def update_effect(self, - action_set, next_situation): - # effect = action_set.fittest_classifier.effect + def update_effect(self, action_set, next_situation): effect = Effect(next_situation) for cl in action_set.least_fit_classifiers(self.cfg.update_percentage): cl.effect = copy(effect) diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index 888c901b..c60065b7 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -18,8 +18,6 @@ def __init__(self, ) -> None: super().__init__(cfg, *args, oktypes=oktypes) - # without this function the Classifierlist will create XCS Classifiers - # instead of XNCS classifiers def generate_covering_classifier(self, situation, action, time_stamp): generalized = [] effect = [] @@ -28,11 +26,10 @@ def generate_covering_classifier(self, situation, action, time_stamp): generalized.append(self.cfg.classifier_wildcard) else: generalized.append(situation[i]) - effect.append(str(random.choice(situation))) + if not self.cfg.cover_env_input: + effect.append(str(random.choice(situation))) if self.cfg.cover_env_input: effect = None - else: - effect = Effect(effect) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 088ba562..8d54c897 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -37,51 +37,6 @@ def __init__(self, self.reward = 0 self.mistakes = [] - def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: - prev_action_set = None - prev_reward = self.reward - prev_state = None # state is known as situation - prev_time_stamp = self.time_stamp # steps - done = False # eop - - raw_state = env.reset() - state = self.cfg.environment_adapter.to_genotype(raw_state) - - while not done: - assert len(self.population) == len(set(self.population)), 'duplicates found' - self.population.delete_from_population() - # We are in t+1 here - action_set, prediction_array, action, match_set = self._form_sets_and_choose_action(state) - # apply action to environment - raw_state, step_reward, done, _ = env.step(action) - state = self.cfg.environment_adapter.to_genotype(raw_state) - - if self.cfg.multistep_enfiroment: - self.reward = step_reward + self.cfg.gamma * self.reward - - self._distribute_and_update(prev_action_set, - prev_state, - state, - prev_reward + self.cfg.gamma * max(prediction_array)) - if done: - self._distribute_and_update(action_set, - state, - state, - self.reward) - else: - prev_action_set = copy(action_set) - prev_reward = self.reward - prev_state = copy(state) - self.time_stamp += 1 - return TrialMetrics(self.time_stamp - prev_time_stamp, self.reward) - - def _form_sets_and_choose_action(self, state): - match_set = self.population.generate_match_set(state, self.time_stamp) - prediction_array = match_set.prediction_array - action = self.select_action(prediction_array, match_set) - action_set = match_set.generate_action_set(action) - return action_set, prediction_array, action, match_set - def _distribute_and_update(self, action_set, current_situation, next_situation, p): if action_set is not None: for cl in action_set: diff --git a/tests/lcs/agents/xcs/__init__.py b/tests/lcs/agents/xcs/__init__.py index 586e99c0..e69de29b 100644 --- a/tests/lcs/agents/xcs/__init__.py +++ b/tests/lcs/agents/xcs/__init__.py @@ -1 +0,0 @@ -# py.test --cov=xcs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xcs diff --git a/tests/lcs/agents/xncs/__init__.py b/tests/lcs/agents/xncs/__init__.py index bfe0111e..8b137891 100644 --- a/tests/lcs/agents/xncs/__init__.py +++ b/tests/lcs/agents/xncs/__init__.py @@ -1 +1 @@ -# py.test --cov=xncs C:\Users\Metron\Documents\GitHub\pyalcs\tests\lcs\agents\xncs + From cd1ac406f800ad6b7f0b02d8bf6a9cbd9f292fcc Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Sat, 12 Jun 2021 16:00:29 +0200 Subject: [PATCH 32/33] Covering fix --- lcs/agents/xcs/Condition.py | 1 - lcs/agents/xncs/Backpropagation.py | 13 +++++++------ lcs/agents/xncs/Classifier.py | 2 +- lcs/agents/xncs/ClassifiersList.py | 2 ++ lcs/agents/xncs/XNCS.py | 23 ++++++++++++----------- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/lcs/agents/xcs/Condition.py b/lcs/agents/xcs/Condition.py index 7f489429..d5868571 100644 --- a/lcs/agents/xcs/Condition.py +++ b/lcs/agents/xcs/Condition.py @@ -1,5 +1,4 @@ from __future__ import annotations - from .. import ImmutableSequence diff --git a/lcs/agents/xncs/Backpropagation.py b/lcs/agents/xncs/Backpropagation.py index 77bf3609..e5efe7c4 100644 --- a/lcs/agents/xncs/Backpropagation.py +++ b/lcs/agents/xncs/Backpropagation.py @@ -15,12 +15,13 @@ def run_bp(self, next_vector: Effect): for cl in action_set: cl.guesses += 1 - if cl.effect != next_vector: - cl.mistakes += 1 - if not any(cl == inside[0] for inside in self.classifiers_for_update): - self.classifiers_for_update.append( - [cl, next_vector, self.cfg.lmc] - ) + if cl.effect is not None: + if cl.effect != next_vector: + cl.mistakes += 1 + if not any(cl == inside[0] for inside in self.classifiers_for_update): + self.classifiers_for_update.append( + [cl, next_vector, self.cfg.lmc] + ) self.check_if_needed() self.update_errors() diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index 520152b6..e5012a74 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -20,7 +20,7 @@ def __init__(self, super().__init__(cfg, condition, action, time_stamp) def __hash__(self): - return hash((str(self.condition),str(self.effect), self.action)) + return hash((str(self.condition), str(self.effect), self.action)) @property def accuracy(self): diff --git a/lcs/agents/xncs/ClassifiersList.py b/lcs/agents/xncs/ClassifiersList.py index c60065b7..b4055355 100644 --- a/lcs/agents/xncs/ClassifiersList.py +++ b/lcs/agents/xncs/ClassifiersList.py @@ -30,6 +30,8 @@ def generate_covering_classifier(self, situation, action, time_stamp): effect.append(str(random.choice(situation))) if self.cfg.cover_env_input: effect = None + else: + effect = Effect(effect) cl = Classifier(cfg=self.cfg, condition=Condition(generalized), action=action, diff --git a/lcs/agents/xncs/XNCS.py b/lcs/agents/xncs/XNCS.py index 8d54c897..721c8712 100644 --- a/lcs/agents/xncs/XNCS.py +++ b/lcs/agents/xncs/XNCS.py @@ -55,18 +55,19 @@ def _distribute_and_update(self, action_set, current_situation, next_situation, def update_fraction_accuracy(self, action_set, next_vector): most_numerous = sorted(action_set, key=lambda cl: -1 * cl.numerosity)[0] - if most_numerous.effect != Effect(next_vector): - if len(self.mistakes) >= 100: - self.mistakes.pop(0) - self.mistakes.append(1) + if most_numerous.effect is not None and next_vector is not None: + if most_numerous.effect != Effect(next_vector): + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(1) + else: + self.mistakes.append(1) else: - self.mistakes.append(1) - else: - if len(self.mistakes) >= 100: - self.mistakes.pop(0) - self.mistakes.append(0) - else: - self.mistakes.append(0) + if len(self.mistakes) >= 100: + self.mistakes.pop(0) + self.mistakes.append(0) + else: + self.mistakes.append(0) @property def fraction_accuracy(self): From a9391c1d7fa629f02de7ab6ba300f196a281c555 Mon Sep 17 00:00:00 2001 From: Marek Chmielewski Date: Sun, 20 Jun 2021 10:53:59 +0200 Subject: [PATCH 33/33] Fix for Update Fitness. --- lcs/agents/xcs/ClassifiersList.py | 12 ++++-------- lcs/agents/xcs/XCS.py | 1 - lcs/agents/xncs/Classifier.py | 3 +-- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lcs/agents/xcs/ClassifiersList.py b/lcs/agents/xcs/ClassifiersList.py index 4c267c9d..e37d1fb4 100644 --- a/lcs/agents/xcs/ClassifiersList.py +++ b/lcs/agents/xcs/ClassifiersList.py @@ -46,14 +46,10 @@ def _generate_covering_and_insert(self, situation, action, time_stamp): self.delete_from_population() return cl - # Roulette-Wheel Deletion - # TODO: use strategies def delete_from_population(self): # TODO: change while to if - # there are places where more than one rule enters the population - # to remedy it I just made deletion run until it cleared all of them - # proffered method should be running it once, ideally inside - # insert_into_population + # During woods the number of rules grew over max number + # To remedy this I added while instead of if. Correct issue should be changed. while self.numerosity > self.cfg.max_population: average_fitness = sum(cl.fitness for cl in self) / self.numerosity deletion_votes = [] @@ -147,9 +143,9 @@ def _update_fitness(self): if cl.error < self.cfg.epsilon_0: tmp_acc = 1 else: - tmp_acc = (pow(self.cfg.alpha * (cl.error * self.cfg.epsilon_0), -self.cfg.v)) + tmp_acc = (pow(self.cfg.alpha * (cl.error * self.cfg.epsilon_0), - self.cfg.v)) accuracy_vector_k.append(tmp_acc) - accuracy_sum += tmp_acc + cl.numerosity + accuracy_sum += tmp_acc * cl.numerosity for cl, k in zip(self, accuracy_vector_k): cl.fitness += ( self.cfg.learning_rate * diff --git a/lcs/agents/xcs/XCS.py b/lcs/agents/xcs/XCS.py index cb562ce4..1181f6ef 100644 --- a/lcs/agents/xcs/XCS.py +++ b/lcs/agents/xcs/XCS.py @@ -61,7 +61,6 @@ def _run_trial_explore(self, env, trials, current_trial) -> TrialMetrics: while not done: assert len(self.population) == len(set(self.population)), 'duplicates found' - self.population.delete_from_population() # We are in t+1 here action_set, prediction_array, action = self._form_sets_and_choose_action(state) # apply action to environment diff --git a/lcs/agents/xncs/Classifier.py b/lcs/agents/xncs/Classifier.py index e5012a74..75b60656 100644 --- a/lcs/agents/xncs/Classifier.py +++ b/lcs/agents/xncs/Classifier.py @@ -30,6 +30,5 @@ def accuracy(self): def __str__(self): return f"Cond:{self.condition} - Act:{self.action} - effect:{self.effect} - Num:{self.numerosity} " + \ - f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, error:{self.error}]" + \ - f"acc: {self.accuracy}" + f"[fit: {self.fitness:.3f}, exp: {self.experience:3.2f}, pred: {self.prediction:2.3f}, error:{self.error:.2f}]"