v1 of Alignment chapter

tamingllms/_build/html/_static/alignment/alignment_simplified.d2

@@ -0,0 +1,38 @@
+# Models with consistent styling
+Alignment: {
+  direction: right
+
+  base: Base Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  sft: SFT Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  reward: Reward Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  aligned: Aligned Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  # Main flow connections
+  base -> sft: Human Instructions
+  sft -> aligned: PPO optimization {
+    style.stroke-width: 2
+  }
+
+  # Branch to reward model
+  sft -> reward: Human\npreferences
+
+  # Connection from reward to main flow
+  reward -> aligned: Re-use preference\nprompts
+}
+
+direction: right

tamingllms/_build/html/_static/alignment/instruct.d2

@@ -0,0 +1,67 @@
+# Main container for models and explanations
+models: {
+  direction: right
+
+  base: Base Language Model {
+    shape: rectangle
+    style.fill: "#E8F6F3"
+    style.stroke: "#2ECC71"
+  }
+
+  inst: Instruction-Tuned Model {
+    shape: rectangle
+    style.fill: "#FCF3CF"
+    style.stroke: "#F4D03F"
+  }
+
+  aligned: Aligned Model {
+    shape: rectangle
+    style.fill: "#EBF5FB"
+    style.stroke: "#3498DB"
+  }
+
+  # Main progression
+  base -> inst -> aligned
+
+  # Direct alignment path
+  base -> aligned: {
+    style.stroke-dash: 5
+  }
+}
+
+# Notes under models
+base_note: Next-Token Prediction {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#2ECC71"
+  style.stroke-dash: 3
+}
+
+inst_note: Task-Specific {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#F4D03F"
+  style.stroke-dash: 3
+}
+
+aligned_note: Human Preference-Aligned {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#3498DB"
+  style.stroke-dash: 3
+}
+
+# Connect models to notes
+models.base -> base_note: {
+  direction: down
+}
+
+models.inst -> inst_note: {
+  direction: down
+}
+
+models.aligned -> aligned_note: {
+  direction: down
+}
+
+direction: right

tamingllms/_build/html/genindex.html

@@ -152,7 +152,7 @@
 
           <li class="toctree-l1 ">
 
-              <a href="notebooks/alignment.html" class="reference internal ">Safety and Alignment</a>
+              <a href="notebooks/alignment.html" class="reference internal ">Preference-Based Alignment</a>
 
 
 

tamingllms/_build/html/markdown/intro.html

@@ -170,7 +170,7 @@
 
           <li class="toctree-l1 ">
 
-              <a href="../notebooks/alignment.html" class="reference internal ">Safety and Alignment</a>
+              <a href="../notebooks/alignment.html" class="reference internal ">Preference-Based Alignment</a>
 
 
 

tamingllms/_build/html/markdown/toc.html

@@ -145,7 +145,7 @@
 
           <li class="toctree-l1 ">
 
-              <a href="../notebooks/alignment.html" class="reference internal ">Safety and Alignment</a>
+              <a href="../notebooks/alignment.html" class="reference internal ">Preference-Based Alignment</a>
 
 
 

tamingllms/_build/html/search.html

@@ -159,7 +159,7 @@
 
           <li class="toctree-l1 ">
 
-              <a href="notebooks/alignment.html" class="reference internal ">Safety and Alignment</a>
+              <a href="notebooks/alignment.html" class="reference internal ">Preference-Based Alignment</a>
 
 
 

tamingllms/_build/jupyter_execute/markdown/intro.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "id": "47492779",
+   "id": "83bbc8a2",
    "metadata": {},
    "source": [
     "(intro)=\n",

tamingllms/_config.yml

@@ -41,6 +41,7 @@ sphinx:
   extra_extensions:
     - sphinxcontrib.mermaid
   config:
+    mathjax_path: https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js
     bibtex_reference_style: author_year
     html_theme: 'press' #insipid
     #html_logo: '_static/logo_w.png'
@@ -49,4 +50,5 @@ sphinx:
     #html_sidebars: {'**': ['globaltoc.html']}
     html_theme_options:
       external_links:
-        - ["Github", "https://github.com/souzatharsis/tamingllms"]    
+        - ["Github", "https://github.com/souzatharsis/tamingllms"]    
+

tamingllms/_static/alignment/alignment_simplified.d2

@@ -0,0 +1,38 @@
+# Models with consistent styling
+Alignment: {
+  direction: right
+
+  base: Base Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  sft: SFT Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  reward: Reward Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  aligned: Aligned Model {
+    shape: rectangle
+    style.fill: "#FFA500"
+  }
+
+  # Main flow connections
+  base -> sft: Human Instructions
+  sft -> aligned: PPO optimization {
+    style.stroke-width: 2
+  }
+
+  # Branch to reward model
+  sft -> reward: Human\npreferences
+
+  # Connection from reward to main flow
+  reward -> aligned: Re-use preference\nprompts
+}
+
+direction: right

tamingllms/_static/alignment/instruct.d2

@@ -0,0 +1,67 @@
+# Main container for models and explanations
+models: {
+  direction: right
+
+  base: Base Language Model {
+    shape: rectangle
+    style.fill: "#E8F6F3"
+    style.stroke: "#2ECC71"
+  }
+
+  inst: Instruction-Tuned Model {
+    shape: rectangle
+    style.fill: "#FCF3CF"
+    style.stroke: "#F4D03F"
+  }
+
+  aligned: Aligned Model {
+    shape: rectangle
+    style.fill: "#EBF5FB"
+    style.stroke: "#3498DB"
+  }
+
+  # Main progression
+  base -> inst -> aligned
+
+  # Direct alignment path
+  base -> aligned: {
+    style.stroke-dash: 5
+  }
+}
+
+# Notes under models
+base_note: Next-Token Prediction {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#2ECC71"
+  style.stroke-dash: 3
+}
+
+inst_note: Task-Specific {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#F4D03F"
+  style.stroke-dash: 3
+}
+
+aligned_note: Human Preference-Aligned {
+  shape: rectangle
+  style.fill: "#FFFFFF"
+  style.stroke: "#3498DB"
+  style.stroke-dash: 3
+}
+
+# Connect models to notes
+models.base -> base_note: {
+  direction: down
+}
+
+models.inst -> inst_note: {
+  direction: down
+}
+
+models.aligned -> aligned_note: {
+  direction: down
+}
+
+direction: right