Set associative backup

Methodology.md

@@ -0,0 +1,31 @@
+# Documentation on the methodology work for DRAM cache experiments
+
+This is not a very detailed or formal document, but tries to at least point to
+the relevant links or paths and important notes.
+
+- DRAM cache component is part of the stdlib in this repo ("src/python/gem5/components/memory/dcache.py").
+- Octopi-cache is a three level MESI cache hiearchy 
+
+
+## gem5 scripts used to take checkpoints
+
+- **NPB D Class:**  Octopi-cache/riscv-2channel-1ccd-checkpoint-timing.py
+
+- **NPB C Class:** ctopi-cache/riscv-2channel-1ccd-checkpoint-timing.py
+
+- **GAPBS (2^25 vertices):** Octopi-cache/riscv-2channel-1ccd-checkpoint-timing-gapbs.py
+
+
+## Bash scripts used (will show the directory where the checkpoints and results are)
+
+- **NPB:** `npb_checkpoint.sh` and `npb_checkpoint_c_class.sh`
+- **GAPBS:** `gapbs_checkpoint.sh`
+
+The gem5 directory I have been using is here: `/home/aakahlow/Dcache_Src/dramCacheController`.
+D Class and GAPBS checkpoints will go there. The C class checkpoints will be stored in /projects (exact path in the npb_checkpoint_c_class.sh script).
+
+## Looppoint stuff
+
+- The branch with O3 loop point analysis stuff: https://github.com/darchr/gem5/commits/looppointAnalysis-Ayaz
+- example.py script in the above branch is a modified version of Zhantong's example script to use looppoint analysis probe and print most recent PC + count.
+

Octopi-cache/components/Octopi.py

@@ -0,0 +1,162 @@
+from gem5.components.cachehierarchies.ruby.abstract_ruby_cache_hierarchy import AbstractRubyCacheHierarchy
+from gem5.components.cachehierarchies.abstract_three_level_cache_hierarchy import (
+    AbstractThreeLevelCacheHierarchy,
+)
+from gem5.coherence_protocol import CoherenceProtocol
+from gem5.isas import ISA
+from gem5.components.boards.abstract_board import AbstractBoard
+from gem5.utils.requires import requires
+
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.directory import Directory
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.dma_controller import DMAController
+
+from m5.objects import RubySystem, DMASequencer, RubyPortProxy, SimpleMemory, AddrRange
+
+from .core_complex import CoreComplex
+from .octopi_network import OctopiNetwork
+from .ruby_network_components import RubyNetworkComponent, RubyRouter, RubyExtLink, RubyIntLink
+
+# CoreComplex sub-systems own the L1, L2, L3 controllers
+# OctopiCache owns the directory controllers
+# RubySystem owns the DMA Controllers
+class OctopiCache(AbstractRubyCacheHierarchy, AbstractThreeLevelCacheHierarchy):
+    def __init__(
+        self,
+        l1i_size: str,
+        l1i_assoc: int,
+        l1d_size: str,
+        l1d_assoc: int,
+        l2_size: str,
+        l2_assoc: int,
+        l3_size: str,
+        l3_assoc: int,
+        num_core_complexes: int,
+        is_fullsystem: bool
+    ):
+        AbstractRubyCacheHierarchy.__init__(self=self)
+        AbstractThreeLevelCacheHierarchy.__init__(
+            self=self,
+            l1i_size=l1i_size,
+            l1i_assoc=l1i_assoc,
+            l1d_size=l1d_size,
+            l1d_assoc=l1d_assoc,
+            l2_size=l2_size,
+            l2_assoc=l2_assoc,
+            l3_size=l3_size,
+            l3_assoc=l3_assoc,
+        )
+
+        self._directory_controllers = []
+        self._dma_controllers = []
+        self._io_controllers = []
+        self._core_complexes = []
+        self._num_core_complexes = num_core_complexes
+        self._is_fullsystem = is_fullsystem
+
+    def incorporate_cache(self, board: AbstractBoard) -> None:
+
+        requires(
+            coherence_protocol_required=CoherenceProtocol.MESI_THREE_LEVEL
+        )
+
+        cache_line_size = board.get_cache_line_size()
+
+        self.ruby_system = RubySystem()
+        # MESI_Three_Level needs 3 virtual networks
+        self.ruby_system.number_of_virtual_networks = 3
+        self.ruby_system.network = OctopiNetwork(self.ruby_system)
+        self.ruby_system.access_backing_store = True
+
+        # Get the first range and the range part.
+        addr_range_0 = board.get_mem_ports()[0][0]
+        self.ruby_system.phys_mem = SimpleMemory(
+            range=AddrRange(start=addr_range_0.start,
+                            end=addr_range_0.end),
+            in_addr_map=False)
+
+        # Setting up the core complex
+        all_cores = board.get_processor().get_cores()
+        num_cores_per_core_complex = len(all_cores) // self._num_core_complexes
+
+        self.core_complexes = [CoreComplex(
+                board = board,
+                cores = all_cores[core_complex_idx*num_cores_per_core_complex:(core_complex_idx + 1) * num_cores_per_core_complex],
+                ruby_system = self.ruby_system,
+                l1i_size = self._l1i_size,
+                l1i_assoc = self._l1i_assoc,
+                l1d_size = self._l1d_size,
+                l1d_assoc = self._l1d_assoc,
+                l2_size = self._l2_size,
+                l2_assoc = self._l2_assoc,
+                l3_size = self._l3_size,
+                l3_assoc = self._l3_assoc,
+        ) for core_complex_idx in range(self._num_core_complexes)]
+
+        self.ruby_system.network.incorporate_ccds(self.core_complexes)
+
+        self._create_directory_controllers(board)
+        self._create_dma_controllers(board, self.ruby_system)
+
+        self.ruby_system.num_of_sequencers = len(all_cores) + len(self._dma_controllers) + len(self._io_controllers)
+        # SimpleNetwork requires .int_links and .routers to exist
+        # if we want to call SimpleNetwork.setup_buffers()
+        self.ruby_system.network.int_links = self.ruby_system.network._int_links
+        self.ruby_system.network.ext_links = self.ruby_system.network._ext_links
+        self.ruby_system.network.routers = self.ruby_system.network._routers
+        self.ruby_system.network.setup_buffers()
+
+        # Set up a proxy port for the system_port. Used for load binaries and
+        # other functional-only things.
+        self.ruby_system.sys_port_proxy = RubyPortProxy()
+        board.connect_system_port(self.ruby_system.sys_port_proxy.in_ports)
+
+    def _create_directory_controllers(self, board):
+        # Adding controllers
+        self.directory_controllers = [Directory(
+                self.ruby_system.network, board.get_cache_line_size(), addr_range, mem_port
+            ) for addr_range, mem_port in board.get_mem_ports()
+        ]
+        for ctrl in self.directory_controllers:
+            ctrl.ruby_system = self.ruby_system
+        # Adding controller routers
+        self.directory_controller_routers = [RubyRouter(self.ruby_system.network) for _ in range(len(self.directory_controllers))]
+        for router in self.directory_controller_routers:
+            self.ruby_system.network._add_router(router)
+        # Adding an external link for each controller and its router
+        self.directory_controller_ext_links = [RubyExtLink(ext_node=dir_ctrl, int_node=dir_router) for dir_ctrl, dir_router in zip(self.directory_controllers, self.directory_controller_routers)]
+        for ext_link in self.directory_controller_ext_links:
+            self.ruby_system.network._add_ext_link(ext_link)
+        _directory_controller_int_links = []
+        for router in self.directory_controller_routers:
+            int_link_1, int_link_2 = RubyIntLink.create_bidirectional_links(router, self.ruby_system.network.cross_ccd_router)
+            _directory_controller_int_links.extend([int_link_1, int_link_2])
+            self.ruby_system.network._add_int_link(int_link_1)
+            self.ruby_system.network._add_int_link(int_link_2)
+        self.directory_controller_int_links = _directory_controller_int_links
+
+    def _create_dma_controllers(self, board, ruby_system):
+        # IOController for full system simulation
+        if self._is_fullsystem:
+            self.io_sequencer = DMASequencer(version=0, ruby_system=self.ruby_system)
+            self.io_sequencer.in_ports = board.get_mem_side_coherent_io_port()
+            self.ruby_system.io_controller = DMAController(
+                dma_sequencer=self.io_sequencer,
+                ruby_system=self.ruby_system
+            )
+            self._io_controllers.append(self.ruby_system.io_controller)
+            self.io_controller_router = RubyRouter(self.ruby_system.network)
+            self.ruby_system.network._add_router(self.io_controller_router)
+            self.io_controller_ext_link = RubyExtLink(ext_node=self._io_controllers[0], int_node=self.io_controller_router)
+            self.ruby_system.network._add_ext_link(self.io_controller_ext_link)
+            self.io_controller_int_links = RubyIntLink.create_bidirectional_links(self.io_controller_router, self.ruby_system.network.cross_ccd_router)
+            self.ruby_system.network._add_int_link(self.io_controller_int_links[0])
+            self.ruby_system.network._add_int_link(self.io_controller_int_links[1])
+
+        self._dma_controllers = []
+        if board.has_dma_ports():
+            for i, port in enumerate(dma_ports):
+                ctrl = DMAController(self.ruby_system.network, cache_line_size)
+                ctrl.dma_sequencer = DMASequencer(version=i+1, in_ports=port)
+                self._dma_controllers.append(ctrl)
+                ctrl.ruby_system = self.ruby_system
+            ruby_system.dma_controllers = self._dma_controllers

Octopi-cache/components/core_complex.py

@@ -0,0 +1,192 @@
+from typing import List, Tuple
+
+from gem5.isas import ISA
+from gem5.components.boards.abstract_board import AbstractBoard
+from gem5.components.processors.abstract_core import AbstractCore
+from gem5.components.cachehierarchies.abstract_three_level_cache_hierarchy import (
+    AbstractThreeLevelCacheHierarchy,
+)
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l1_cache import L1Cache
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l2_cache import L2Cache
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l3_cache import L3Cache
+from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.directory import Directory
+
+from m5.objects import SubSystem, L2Cache_Controller, AddrRange, RubySequencer, Switch, SimpleIntLink, SimpleExtLink, SubSystem, SimObject
+
+from .ruby_network_components import RubyRouter, RubyExtLink, RubyIntLink, RubyNetworkComponent
+
+class CoreComplex(SubSystem, RubyNetworkComponent):
+    _core_id = 0
+    _core_complex_id = 0
+
+    @classmethod
+    def _get_core_id(cls):
+        cls._core_id += 1
+        return cls._core_id - 1
+
+    @classmethod
+    def _get_core_complex_id(cls):
+        cls._core_complex_id += 1
+        return cls._core_complex_id - 1
+
+    def __init__(
+        self,
+        board: AbstractBoard,
+        cores: List[AbstractCore],
+        ruby_system,
+        l1i_size: str,
+        l1i_assoc: int,
+        l1d_size: str,
+        l1d_assoc: int,
+        l2_size: str,
+        l2_assoc: int,
+        l3_size: str,
+        l3_assoc: int,
+    ):
+        SubSystem.__init__(self=self)
+        RubyNetworkComponent.__init__(self=self)
+
+        self._l1i_size = l1i_size
+        self._l1i_assoc = l1i_assoc
+        self._l1d_size = l1d_size
+        self._l1d_assoc = l1d_assoc
+        self._l2_size = l2_size
+        self._l2_assoc = l2_assoc
+        self._l3_size = l3_size
+        self._l3_assoc = l3_assoc
+
+        self._board = board
+        self._cores = cores
+        self._ruby_system = ruby_system
+        self._cache_line_size = 64
+
+        self._directory_controllers = []
+
+        self._core_complex_id = self._get_core_complex_id()
+        self.main_router = RubyRouter(self._ruby_system) # this will be connect to component outside the core complex
+        self._add_router(self.main_router)
+        self._create_core_complex()
+
+    def get_main_router(self):
+        return self.main_router
+
+    def _create_core_complex(self):
+        # Create L1 caches, L2 cache, and corresponding controllers per core
+        self.core_clusters = [self._create_core_cluster(core) for core in self._cores]
+        # Create L3 cache and its corresponding controller
+        self._create_shared_cache()
+        # Setting up one router and one external link per controller
+        self._create_external_links()
+        # Setting up L1/L2 links, L2/main links, L3/main link
+        self._create_internal_links()
+
+    def _create_core_cluster(self, core: AbstractCore):
+        cluster = SubSystem()
+        core_id = self._get_core_id()
+
+        cluster.l1_cache = L1Cache(
+            l1i_size = self._l1i_size,
+            l1i_assoc = self._l1i_assoc,
+            l1d_size = self._l1d_size,
+            l1d_assoc = self._l1d_assoc,
+            network = self._ruby_system.network,
+            core = core,
+            cache_line_size = self._cache_line_size,
+            target_isa = self._board.processor.get_isa(),
+            clk_domain = self._board.get_clock_domain(),
+        )
+        cluster.l1_cache.sequencer = RubySequencer(
+            version = core_id,
+            dcache = cluster.l1_cache.Dcache,
+            clk_domain = cluster.l1_cache.clk_domain
+        )
+
+        if self._board.has_io_bus():
+            cluster.l1_cache.sequencer.connectIOPorts(self._board.get_io_bus())
+        cluster.l1_cache.ruby_system = self._ruby_system
+        core.connect_icache(cluster.l1_cache.sequencer.in_ports)
+        core.connect_dcache(cluster.l1_cache.sequencer.in_ports)
+        core.connect_walker_ports(
+            cluster.l1_cache.sequencer.in_ports, cluster.l1_cache.sequencer.in_ports
+        )
+        if self._board.get_processor().get_isa() == ISA.X86:
+            core.connect_interrupt(
+                cluster.l1_cache.sequencer.interrupt_out_port,
+                cluster.l1_cache.sequencer.in_ports
+            )
+        else:
+            core.connect_interrupt()
+
+        cluster.l2_cache = L2Cache(
+            l2_size=self._l2_size,
+            l2_assoc=self._l2_assoc,
+            network=self._ruby_system.network,
+            core=core,
+            num_l3Caches=1, # each core complex has 1 slice of L3 Cache
+            cache_line_size=self._cache_line_size,
+            cluster_id=self._core_complex_id,
+            target_isa=self._board.processor.get_isa(),
+            clk_domain=self._board.get_clock_domain(),
+        )
+        cluster.l2_cache.ruby_system = self._ruby_system
+        # L0Cache in the ruby backend is l1 cache in stdlib
+        # L1Cache in the ruby backend is l2 cache in stdlib
+        cluster.l2_cache.bufferFromL0 = cluster.l1_cache.bufferToL1
+        cluster.l2_cache.bufferToL0 = cluster.l1_cache.bufferFromL1
+
+        return cluster
+
+    def _create_shared_cache(self):
+        self.l3_cache = L3Cache(
+            l3_size=self._l3_size,
+            l3_assoc=self._l3_assoc,
+            network=self._ruby_system.network,
+            num_l3Caches=1,
+            cache_line_size=self._cache_line_size,
+            cluster_id=self._core_complex_id,
+        )
+        self.l3_cache.ruby_system = self._ruby_system
+
+    # This is where all routers and links are created
+    def _create_external_links(self):
+        # create a router per cache controller
+        #  - there is one L3 per ccd
+        self.l3_router = RubyRouter(self._ruby_system)
+        self._add_router(self.l3_router)
+        #  - there is one L1 and one L2 per cluster
+        for cluster in self.core_clusters:
+            cluster.l1_router = RubyRouter(self._ruby_system)
+            self._add_router(cluster.l1_router)
+            cluster.l2_router = RubyRouter(self._ruby_system)
+            self._add_router(cluster.l2_router)
+
+        # create an ext link from a controller to a router
+        self.l3_router_link = RubyExtLink(ext_node=self.l3_cache, int_node=self.l3_router, bandwidth_factor=64)
+        self._add_ext_link(self.l3_router_link)
+        for cluster in self.core_clusters:
+            cluster.l1_router_link = RubyExtLink(ext_node=cluster.l1_cache, int_node=cluster.l1_router)
+            self._add_ext_link(cluster.l1_router_link)
+            cluster.l2_router_link = RubyExtLink(ext_node=cluster.l2_cache, int_node=cluster.l2_router)
+            self._add_ext_link(cluster.l2_router_link)
+
+    def _create_internal_links(self):
+        # create L1/L2 links
+        for cluster in self.core_clusters:
+            l1_to_l2, l2_to_l1 = RubyIntLink.create_bidirectional_links(cluster.l1_router, cluster.l2_router)
+            cluster.l1_to_l2_link = l1_to_l2
+            cluster.l2_to_l1_link = l2_to_l1
+            self._add_int_link(l1_to_l2)
+            self._add_int_link(l2_to_l1)
+        # create L2/main_router links
+        for cluster in self.core_clusters:
+            l2_to_main, main_to_l2 = RubyIntLink.create_bidirectional_links(cluster.l2_router, self.main_router)
+            cluster.l2_to_main_link = l2_to_main
+            cluster.main_to_l2_link = main_to_l2
+            self._add_int_link(l2_to_main)
+            self._add_int_link(main_to_l2)
+        # create L3/main_router link
+        l3_to_main, main_to_l3 = RubyIntLink.create_bidirectional_links(self.l3_router, self.main_router, bandwidth_factor=64)
+        self.l3_to_main_link = l3_to_main
+        self.main_to_l3_link = main_to_l3
+        self._add_int_link(l3_to_main)
+        self._add_int_link(main_to_l3)