hed_multi_agent_retrieval_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "008a33da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "\n",
    "import chromadb\n",
    "\n",
    "import autogen\n",
    "from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent\n",
    "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n",
    "\n",
    "# Accepted file formats for that can be stored in\n",
    "# a vector database instance\n",
    "from autogen.retrieve_utils import TEXT_FORMATS\n",
    "# When using a single openai endpoint, you can use the following:\n",
    "config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1863e0aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. create an RetrieveAssistantAgent instance named \"assistant\"\n",
    "assistant = RetrieveAssistantAgent(\n",
    "    name=\"assistant\",\n",
    "    system_message=\"You are a helpful assistant.\",\n",
    "    llm_config={\n",
    "        \"timeout\": 600,\n",
    "        \"cache_seed\": 42,\n",
    "        \"config_list\": config_list,\n",
    "    },\n",
    ")\n",
    "\n",
    "# 2. create the RetrieveUserProxyAgent instance named \"ragproxyagent\"\n",
    "# By default, the human_input_mode is \"ALWAYS\", which means the agent will ask for human input at every step. We set it to \"NEVER\" here.\n",
    "# `docs_path` is the path to the docs directory. It can also be the path to a single file, or the url to a single file. By default,\n",
    "# it is set to None, which works only if the collection is already created.\n",
    "# `task` indicates the kind of task we're working on. In this example, it's a `code` task.\n",
    "# `chunk_token_size` is the chunk token size for the retrieve chat. By default, it is set to `max_tokens * 0.6`, here we set it to 2000.\n",
    "# `custom_text_types` is a list of file types to be processed. Default is `autogen.retrieve_utils.TEXT_FORMATS`.\n",
    "# This only applies to files under the directories in `docs_path`. Explicitly included files and urls will be chunked regardless of their types.\n",
    "# In this example, we set it to [\"non-existent-type\"] to only process markdown files. Since no \"non-existent-type\" files are included in the `websit/docs`,\n",
    "# no files there will be processed. However, the explicitly included urls will still be processed.\n",
    "ragproxyagent = RetrieveUserProxyAgent(\n",
    "    name=\"ragproxyagent\",\n",
    "    # human_input_mode=\"NEVER\",\n",
    "    # max_consecutive_auto_reply=3,\n",
    "    retrieve_config={\n",
    "        \"task\": \"qa\",\n",
    "        \"docs_path\": [\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/BidsAnnotationQuickstart.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/CTaggerGuiTaggingTool.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/DocumentationSummary.md\",\n",
    "            # \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/FileRemodelingQuickstart.md\",\n",
    "            # \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/FileRemodelingTools.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedAndEEGLAB.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedAnnotationQuickstart.md\",\n",
    "            # \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedConditionsAndDesignMatrices.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/llm/docs/source/HedConditionsAndDesignMatricesPart1.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/llm/docs/source/HedConditionsAndDesignMatricesPart2.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedGovernance.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedJavascriptTools.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedMatlabTools.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedOnlineTools.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedPythonTools.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedSchemaDevelopersGuide.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedSchemas.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedSearchGuide.md\",\n",
    "            # \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedSummaryGuide.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedTestDatasets.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HedValidationGuide.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/HowCanYouUseHed.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/IntroductionToHed.md\",\n",
    "            \"https://raw.githubusercontent.com/hed-standard/hed-examples/main/docs/source/WhatsNew.md\",\n",
    "        ],\n",
    "        # \"model\": config_list[0][\"model\"],\n",
    "        \"vector_db\": \"chroma\",  # to use the deprecated `client` parameter, set to None and uncomment the line above\n",
    "        \"overwrite\": True,  # set to True if you want to overwrite an existing collection\n",
    "    },\n",
    "    code_execution_config=False,  # set to False if you don't want to execute the code\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0cc5544a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from autogen.agentchat.contrib.capabilities.text_compressors import LLMLingua\n",
    "from autogen.agentchat.contrib.capabilities.transforms import TextMessageCompressor\n",
    "from autogen.agentchat.contrib.capabilities import transform_messages\n",
    "\n",
    "llm_lingua = LLMLingua()\n",
    "text_compressor = TextMessageCompressor(text_compressor=llm_lingua)\n",
    "\n",
    "context_handling = transform_messages.TransformMessages(transforms=[text_compressor])\n",
    "context_handling.add_to_agent(ragproxyagent)\n",
    "context_handling.add_to_agent(assistant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "61c97c36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VectorDB returns doc_ids:  [['f0766f55', '85315238', '22a4f1b0', 'c5088cd3']]\n",
      "\u001b[32mAdding content of doc f0766f55 to context.\u001b[0m\n",
      "\u001b[33mragproxyagent\u001b[0m (to assistant):\n",
      "\n",
      "You're a retrieve augmented chatbot. You answer user's questions based on your own knowledge and the\n",
      "context provided by the user.\n",
      "If you can't answer the question with or without the current context, you should reply exactly `UPDATE CONTEXT`.\n",
      "You must give as short an answer as possible.\n",
      "\n",
      "User's question is: ('Give example of annotating three stimulus condition variables using HED. The stimulus types information is detailed below:\\nsquare_stimulus: A picture of a dark blue square.\\n', 'circle_stimulus: A picture of a green circle.\\n', 'animal_stimulus: A picture of an animal.')\n",
      "\n",
      "Context is: # HED conditions and design matrices\n",
      "\n",
      "This tutorial discusses how information from neuroimaging experiments should be\n",
      "stored and annotated so that the underlying experimental design and experimental conditions\n",
      "for a dataset can be automatically extracted, summarized, and used in analysis.\n",
      "The mechanisms for doing this use HED (Hierarchical Event Descriptors) in conjunction\n",
      "with a [BIDS](https://bids.neuroimaging.io/)\n",
      "(Brain Imaging Data Structure) representation of the dataset.\n",
      "\n",
      "This tutorial introduces tools and strategies for encoding information\n",
      "about the experimental design as part of a dataset metadata\n",
      "without excessive effort on the part of the researcher.\n",
      "The discussion mainly focuses on categorical variables.\n",
      "\n",
      "(hed-annotations-for-conditions-anchor)=\n",
      "## HED annotations for conditions\n",
      "\n",
      "As mentioned above, HED (Hierarchical Event Descriptors) provide several mechanisms for easily\n",
      "annotating the experimental conditions represented by a BIDS dataset so that\n",
      "the information can be automatically extracted, summarized, and used by tools.\n",
      "\n",
      "HED has three ways of annotating experimental conditions: condition variables without definitions,\n",
      "condition variables with definitions but no levels, and condition variables with levels.\n",
      "All three mechanisms use the *Condition-variable* tag as part of the annotation.\n",
      "The three mechanisms can be used in any combination to document the experimental design\n",
      "for a dataset.\n",
      "\n",
      "(direct-condition-variables-anchor)=\n",
      "### Direct condition variables\n",
      "\n",
      "The simplest way to encode experimental conditions is to use named *Condition-variable*\n",
      "tags for each condition value. The following is a sample excerpt from\n",
      "a simplified event file for an experiment to distinguish brain responses\n",
      "for houses and faces. \n",
      "\n",
      "(sample-house-face-example-anchor)=\n",
      "````{admonition} Example 1. Excerpt from a sample event file from a simplified house-face experiment.\n",
      "| onset | duration | event_type | stim_file |\n",
      "| ----- | -------- |----------- | ---------- |\n",
      "| 2.010 |  0.1     | show_house  | ranch1.png |\n",
      "| 3.210 |  0.1     | show_house  | colonial68.png |   \n",
      "| 4.630 |  0.1     | show_face   | female43.png | \n",
      "| 6.012 |  0.1     | show_house  | castle2.png | \n",
      "| 7.440 |  0.1     | show_face  | male81.png  |   \n",
      "````\n",
      "\n",
      "As explained in [**BIDS annotation quickstart**](https://hed-examples.readthedocs.io/en/latest/BidsAnnotationQuickstart.html), \n",
      "the most commonly used strategy for annotating events in a BIDS dataset is\n",
      "to create a single JSON file located in the dataset root containing the annotations\n",
      "for the columns. The following shows a minimal example:\n",
      "\n",
      "````{admonition} Example 2: Minimal JSON sidecar with HED annotations for Example 1.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"event_type\": {\n",
      "      \"HED\": {\n",
      "         \"show_house\": \"Sensory-presentation, Visual-presentation, Experimental-stimulus, (Image, Building/House), Condition-variable/House-cond\",\n",
      "         \"show_face\": \"Sensory-presentation, Visual-presentation, Experimental-stimulus, (Image, Face), Condition-variable/Face-cond\"\n",
      "      }\n",
      "   },\n",
      "   \"stim_file\": {\n",
      "      \"HED\": \"(Image, Pathname/#)\"\n",
      "   }\n",
      "}\n",
      "```\n",
      "````\n",
      "\n",
      "Each row in an `events.tsv` file represents a time marker in the corresponding data recording.\n",
      "At analysis time, HED tools look up each `events.tsv` column value in the JSON file and concatenate the\n",
      "corresponding HED annotation into a single string representing the annotation for that row.\n",
      "Annotations without #'s are used directly, while annotations with # have the corresponding\n",
      "column values substituted when the annotation is assembled. \n",
      "\n",
      "Example 3 shows the HED annotation for the first row in the `events.tsv` file of Example 1.\n",
      "\n",
      "````{admonition} Example 3: HED annotation for first event in Example 1 using JSON sidecar of Example 2.\n",
      ":class: tip\n",
      "\n",
      "> \"*Sensory-presentation*, *Visual-presentation*, *Experimental-stimulus*,  \n",
      "> (*Image*, *Building/House*), *Condition-variable/House-cond*,  \n",
      "> (*Image*, *Pathname/ranch1.png*)\"\n",
      "````\n",
      "\n",
      "Notice that *Building/House* is a partial path rather than a single tag.\n",
      "This is because *House* is currently not part of the base HED vocabulary.\n",
      "However, users are allowed to extend tags at most nodes in the HED schema,\n",
      "but they must use a path that includes at least one ancestor in the HED schema.\n",
      "\n",
      "HED tools have the capability of automatically detecting *Condition-variable*\n",
      "tags in annotated HED datasets to create factor vectors and summaries automatically.\n",
      "Example 4 shows the event file after HED tools have appended one-hot factor vectors\n",
      "for the two condition variables *Condition-variable/House-cond* and\n",
      "*Condition-variable/Face-cond*. \n",
      "The 1's and 0's *house_cond* and *face-cond* columns indicate presence or absence\n",
      "of the corresponding condition variables.\n",
      "\n",
      "\n",
      "````{admonition} Example 4. Event file from Example 2 after one-hot factor vector extraction.\n",
      "\n",
      "| onset | duration | event_type | stim_file | house-cond | face-cond |\n",
      "| ----- | -------- |----------- |-------- | ---------- |----------- |\n",
      "| 2.010 |  0.1     | show_house      | ranch1.png |    1  |   0    |\n",
      "| 3.210 |  0.1     | show_house      | colonial68.png |  1  |  0  | \n",
      "| 4.630 |  0.1     | show_face       | female43.png |  0   |  1 |\n",
      "| 6.012 |  0.1     | show_house      | castle2.png |  1    |  0 |\n",
      "| 7.440 |  0.1     | show_face       | male81.png  |  0    | 1 |\n",
      "````\n",
      "\n",
      "Example 5 shows a JSON summary that HED tools can extract from a single events file\n",
      "once a dataset has been annotated using HED. \n",
      "This very simple example only had two condition variables\n",
      "and only used direct references to these condition variables. \n",
      "Dataset-wide summaries can also be extracted.\n",
      "\n",
      "\n",
      "````{admonition} Example 5: The HED tools summary of condition variables for Example 4.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"house-cond\": {\n",
      "      \"name\": \"house-cond\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 0,\n",
      "      \"direct_references\": 3,\n",
      "      \"total_events\": 5,\n",
      "      \"number_type_events\": 3,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {}\n",
      "   },\n",
      "   \"face-cond\": {\n",
      "      \"name\": \"face-cond\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 0,\n",
      "      \"direct_references\": 2,\n",
      "      \"total_events\": 5,\n",
      "      \"number_type_events\": 2,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {}\n",
      "   }\n",
      "}\n",
      "````\n",
      "The summary shows that of the 5 events in the file: 3 events were under\n",
      "the house condition and 2 events were under the face condition.\n",
      "There were no events in multiple categories of the same condition variables\n",
      "(which would not be possible since these condition variables were referenced\n",
      "directly rather than using assigned levels).\n",
      "All names are translated to lower case as HED is case-insensitive with respect to analysis,\n",
      "and the summary and factorization tools convert to lower case before processing.\n",
      "\n",
      "These HED summaries can be created for other tags besides *Condition-variable*,\n",
      "hence the *variable_type* is given in the summary of Example 5.\n",
      "Other commonly created summaries are for *Task* and *Control-variable*.\n",
      "\n",
      "In this example, the two conditions: *house-cond* and *face-cond* are\n",
      "treated as though they were unrelated. These direct condition variables\n",
      "are very easy to annotate--- just make up a name and stick the tags anywhere\n",
      "you want to create factor variables or summaries.\n",
      "However, a more common situation is for a condition variable to have multiple levels,\n",
      "which direct use condition variables does not support.\n",
      "\n",
      "Another disadvantage of direct condition variables is that there is\n",
      "no information about what the conditions represent beyond the arbitrarily chosen condition names.\n",
      "\n",
      "A third disadvantage is that direct condition variables can not be used to\n",
      "anchor events with temporal extent.\n",
      "\n",
      "The next section introduces defined condition variables,\n",
      "which address both of these disadvantages.\n",
      "\n",
      "(defined-condition-variables-anchor)=\n",
      "### Defined condition variables\n",
      "\n",
      "\n",
      "````{admonition} Example 6: A revised JSON sidecar using defined conditions for Example 1.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"event_type\": {\n",
      "      \"HED\": {\n",
      "         \"show_house\": \"Sensory-presentation, Visual-presentation, Experimental-stimulus, (Image, Building/House), Def/House-cond\",\n",
      "         \"show_face\": \"Sensory-presentation, Visual-presentation, Experimental-stimulus, (Image, Face), Def/Face-cond\"\n",
      "      }\n",
      "   },\n",
      "   \"stim_file\": {\n",
      "      \"HED\": \"(Image, Pathname/#)\"\n",
      "   },\n",
      "   \"my_definitions\": {\n",
      "      \"HED\": {\n",
      "          \"house_cond_def\": \"(Definition/House-cond, (Condition-variable/Presentation-type, (Image, Building/House)))\",\n",
      "          \"face_cond_def\": \"(Definition/Face-cond, (Condition-variable/Presentation-type, (Image, Face)))\"\n",
      "}\n",
      "```\n",
      "````\n",
      "\n",
      "Example 6 defines a condition variable called *Presentation-type* with two levels:\n",
      "*House-cond* and *Face-cond*.\n",
      "The definitions of *House-cond* and *Face-cond* both include the same *Presentation-type*\n",
      "*Condition-variable* so tools recognize these as levels of the same variable and\n",
      "automatically extract the 2-factor experimental design.\n",
      "\n",
      "Notice that the (*Image*, *Building/House*) tags are included both in the definition of\n",
      "the *House-cond* level of the *Presentation-type* condition variable\n",
      "and in the tags for the *event_type* column value *show_house*.\n",
      "Similarly, the (*Image*, *Face*) tags appear in both the definition of the\n",
      "*Face-cond* level of the *Presentation-type* condition variable\n",
      "and in the tags for the *event_type* column value *show_face*.\n",
      "We have included these tags in both places because generally the condition variable definitions\n",
      "are removed prior to searching for HED tags. The tags in the definitions\n",
      "define the meaning of the conditions.\n",
      "\n",
      "````{admonition} Example 7: The summary extracted when the JSON sidecar of Example 6 is used.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"presentation-type\": {\n",
      "      \"name\": \"presentation-type\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 2,\n",
      "      \"direct_references\": 0,\n",
      "      \"total_events\": 5,\n",
      "      \"number_type_events\": 5,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {\n",
      "         \"house-cond\": 3,\n",
      "         \"face-cond\": 2\n",
      "      }\n",
      "  }\n",
      "}\n",
      "```\n",
      "````\n",
      "\n",
      "(direct-vs-defined-approaches-anchor)=\n",
      "### Direct vs defined approaches\n",
      "\n",
      "Table 1 compares the two approaches for encoding experimental conditions and design in HED.\n",
      "Both approaches use the *Condition-variable* tag.\n",
      "While direct condition variables (just using a *Condition-variable* tag without defining it)\n",
      "is very easy, it provides limited information about meaning in downstream summaries.\n",
      "In general defined condition variables, while more work, provide a more complete picture.\n",
      "\n",
      "(direct-vs-defined-anchor)=\n",
      "````{table} **Table 1:** Comparison of direct versus definition conditions.\n",
      "| Approach | Advantages  | Disadvantages |\n",
      "| -------- | ----------  | ------------- |\n",
      "| **Direct**   | Easy to use--just a label.<br/>Can appear in summaries.<br/>Can generate factor vectors. | Give no information about meaning.<br/>No levels for condition variables.<br/>Limited information about experimental design.<br/>Do not support event temporal extent. |\n",
      "| **Defined** | Better information in summaries.<br/>Encode condition variables with levels.<br/>Can give factor vectors for levels.<br/>Better experimental design information.</br>Can anchor events with temporal extent.| Must give definitions. |\n",
      "````\n",
      "\n",
      "It should be noted that other tags, particularly those in the HED `Structural-property` subtree such\n",
      "as `Task` can be summarized and used as factor vectors in a way similar to *Condition-variable*.\n",
      "\n",
      "\n",
      "(column-vs-row-annotations-anchor)=\n",
      "### Column vs row annotations\n",
      "\n",
      "In this section, we look at a more complicated example based on the Wakeman-Henson face-processing dataset. \n",
      "This dataset, which is available on [OpenNeuro](https://openneuro.org) under accession number\n",
      "ds003645, was used in as a case study on HED annotation described in the \n",
      "[Capturing the nature of events paper](https://www.sciencedirect.com/science/article/pii/S1053811921010387).\n",
      "The experiment is based on a 3 x 3 x 2 experimental design: face type x repetition status x key choice.\n",
      "\n",
      "The experimental stimulus in each trial was the visual presentation of one of 3 possible types of images:\n",
      "a well-known face, an unfamiliar face, and a scrambled face image. \n",
      "The type of face was randomized across trials.\n",
      "\n",
      "The repetition status condition variable also had one of three possible values and indicated\n",
      "whether the stimulus image had not been seen before (first show), \n",
      "was just seen in the previous trial (immediate repeat),\n",
      "or had been last seen several trials ago (delayed repeat). \n",
      "The repetition status was randomized across trials.\n",
      "\n",
      "The final condition variable in the experimental design was the key assignment.\n",
      "In the right symmetry condition, participants pressed the right mouse button\n",
      "to indicate that the presented face had above average symmetry, \n",
      "while in the left symmetry condition, participants pressed the left mouse button\n",
      "to indicate that the presented face had above average symmetry.\n",
      "The key assignment was held constant for each recording, but the key choice was \n",
      "counter-balanced across participants.\n",
      "\n",
      "Example 8 shows an excerpt from the event file of sub-002 run-1.\n",
      "(You may find it useful to look at the full event file [sub-002_task-FacePerception_run-1_events.tsv](./_static/data/sub-002_task-FacePerception_run-1_events.tsv) and the dataset's\n",
      "JSON sidecar with full HED annotations:\n",
      "[task-facePerception_events.json](./_static/data/task-FacePerception_events.json)\n",
      "\n",
      "(sample-design-matrix-events-file-anchor)=\n",
      "```{admonition} Example 8: An excerpt from the Wakeman-Henson face-processing dataset.\n",
      "\n",
      "| onset | duration | event_type | face_type | rep_status | trial | rep_lag | value | stim_file |\n",
      "| ----- | -------- | ---------- | --------- | ---------- | ----- | ------- | ----- | --------- |\n",
      "| 0.004 | n/a | setup_right_sym | n/a | n/a | n/a | n/a | 3 | n/a |\n",
      "| 24.2098 | n/a | show_face_initial | unfamiliar_face | first_show | 1 | n/a | 13 | u032.bmp |\n",
      "| 25.0353 | n/a | show_circle | n/a | n/a | 1 | n/a | 0 | circle.bmp |\n",
      "| 25.158 | n/a | left_press | n/a | n/a | 1 | n/a | 256 | n/a |\n",
      "| 26.7353 | n/a | show_cross | n/a | n/a | 2 | n/a | 1 | cross.bmp |\n",
      "| 27.2498 | n/a | show_face | unfamiliar_face | immediate_repeat | 2 | 1 | 14 | u032.bmp |\n",
      "| 27.8971 | n/a | left_press | n/a | n/a | 2 | n/a | 256 | n/a |\n",
      "| 28.0998 | n/a | show_circle | n/a | n/a | 2 | n/a | 0 | circle.bmp |\n",
      "| 29.7998 | n/a | show_cross | n/a | n/a | 3 | n/a | 1 | cross.bmp |\n",
      "| 30.3571 | n/a | show_face | unfamiliar_face | first_show | 3 | n/a | 13 | u088.bmp |\n",
      "```\n",
      "\n",
      "Example 8 illustrates two different ways of using defined conditions for encoding:\n",
      "**inserting an event with temporal extent** or using **column encoding**.\n",
      "\n",
      "The key assignment condition is marked by inserting an event with *event_type* equal to\n",
      "*setup_right_sym* at the beginning of the file. \n",
      "As we shall see below, this event is annotated with having temporal extent,\n",
      "as defined by HED *Onset* and *Offset* tags,\n",
      "so the condition is in effect until the event's extent ends.\n",
      "\n",
      "In the column strategy, an event file column represents the condition variable,\n",
      "and the values in that column represent the levels.\n",
      "With this encoding, the condition variable is only applicable at a particular\n",
      "level when that level name appears in the column.\n",
      "An n/a value in that column indicates the condition does not apply to that event.\n",
      "\n",
      "Example 9 shows the portion of the\n",
      "[**task-facePerception_events.json**](./_static/data/task-FacePerception_events.json)\n",
      "that encodes information about the *setup_right_sym* event found as the first event\n",
      "in the event file excerpt of Example 8.\n",
      "This excerpt only contains the relevant definition and the relevant annotation.\n",
      "\n",
      "\n",
      "````{admonition} Example 9: Excerpt of the JSON sidecar relevant to the *setup_right_sym* event.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"event_type\": {\n",
      "      \"HED\": {\n",
      "         \"setup_right_sym\": \"Experiment-structure, (Def/Right-sym-cond, Onset), (Def/Initialize-recording, Onset)\"\n",
      "      }\n",
      "   },\n",
      "   \"hed_def_conds\": {\n",
      "      \"HED\": {\n",
      "        \"right_sym_cond_def\": \"(Definition/Right-sym-cond, (Condition-variable/Key-assignment, ((Index-finger, (Right-side-of, Experiment-participant)), (Behavioral-evidence, Symmetrical)), ((Index-finger, (Left-side-of, Experiment-participant)), (Behavioral-evidence, Asymmetrical)), Description/Right index finger key press indicates a face with above average symmetry.))\"\n",
      "      }\n",
      "   }\n",
      "}\n",
      "```\n",
      "````\n",
      "\n",
      "Only the *event_type* column is relevant for assembling the annotations for the first row\n",
      "of Example 8, since the other annotated columns have n/a values.\n",
      "The assembled HED annotation for the first row of Example 8 is shown in Example 10.\n",
      "\n",
      "````{admonition} Example 10: The HED annotation of the first row of Example 8.\n",
      ":class: tip\n",
      "> \"*Experiment-structure*, \n",
      "> (*Def/Right-sym-cond*, *Onset*), \n",
      "> (*Def/Initialize-recording*, *Onset*)\"\n",
      "\n",
      "````\n",
      "\n",
      "HED represents events of temporal extent using HED definitions with the *Onset* \n",
      "and *Offset* tags grouped with a user-defined term.\n",
      "The (*Def/Right-sym-cond*, *Onset*) specifies that an event defined by\n",
      "*Right-sym-cond* begins at the time-marker represented by this row in the event file.\n",
      "This event continues until the end of the file or until an event marker with\n",
      "(*Def/Right-sym-cond*, *Offset*) occurs.\n",
      "In this case, no *Offset* marker for *Right-sym-cond* appears in the file,\n",
      "so the event represented by *Right-sym-cond* occurs over the entire recording.\n",
      "\n",
      "The user-defined term is prefixed with *Def/* and indicates what the event\n",
      "of temporal extent represents.\n",
      "If the definition includes a *Condition-variable*,\n",
      "then the event represents the occurrence of that experimental condition.\n",
      "The user-defined terms are usually defined in the `events.json` file\n",
      "located at the top-level of the BIDS dataset.\n",
      "\n",
      "\n",
      "Example 11 shows a more readable form for the definition of *Right-sym-cond*.\n",
      "\n",
      "````{admonition} Example 11: The contents of the definition for *Right-sym-cond*.\n",
      ":class: tip\n",
      "\n",
      "```text\n",
      "(  \n",
      "   Definition/Right-sym-cond, (  \n",
      "      Condition-variable/Key-assignment,   \n",
      "      ((Index-finger, (Right-side-of, Experiment-participant)), (Behavioral-evidence, Symmetrical)),  \n",
      "      ((Index-finger, (Left-side-of, Experiment-participant)), (Behavioral-evidence, Asymmetrical)),  \n",
      "      Description/Right index finger key press indicates a face with above average symmetry.  \n",
      "   )  \n",
      ")\n",
      "```\n",
      "````\n",
      "The primary use of the definitions for condition variables is to encode\n",
      "the experimental design in an actionable format.\n",
      "Thus, as a general practice, *Defs* representing condition variables are\n",
      "removed prior to searching for other tags to avoid repeats.\n",
      "Notice that both *Right-side-of* and *Left-side-of* appear in the definition.\n",
      "Thus, if these *Defs* were included, every event would have both left and right tags.\n",
      "\n",
      "Once a dataset includes the appropriate annotations,\n",
      "HED tools can automatically extract the experimental design.\n",
      "Example 12 shows the result of extraction of categorical factor vectors for\n",
      "the event file of Example 8.\n",
      "\n",
      "```{admonition} Example 12: HED tools categorical form extraction of the design matrix for Example 8.\n",
      "\n",
      "| onset | key-assignment | face-type | repetition-type | \n",
      "| ----- | -------------- | --------- | --------------- |\n",
      "| 0.004 | right-sym-cond | n/a | n/a |\n",
      "| 24.2098 | right-sym-cond | unfamiliar-face-cond | first-show-cond |\n",
      "| 25.0353 | right-sym-cond | n/a | n/a |\n",
      "| 25.158 | right-sym-cond | n/a | n/a |\n",
      "| 26.7353 | right-sym-cond | n/a | n/a | \n",
      "| 27.2498 | right-sym-cond | unfamiliar-face-cond | immediate-repeat-cond |\n",
      "| 27.8971 | right-sym-cond | n/a | n/a |\n",
      "| 28.0998 | right-sym-cond | n/a | n/a |\n",
      "| 29.7998 | right-sym-cond | n/a | n/a | \n",
      "| 30.3571 | right-sym-cond | unfamiliar-face-cond | first-show-cond | \n",
      "\n",
      "````\n",
      "\n",
      "In the categorical representation,\n",
      "HED uses the condition variable name as the column name.\n",
      "The level values appear in the columns for event markers at which \n",
      "the condition variable at that level applies.\n",
      "Notice that *right-sym-cond* appears in every row because it was used in an event\n",
      "that extended to the end of the file. \n",
      "On the other hand, the other condition variables were encoded using\n",
      "columns and only appear when present in the column.\n",
      "\n",
      "Note that if an event has multiple levels of the same condition,\n",
      "categorical and ordinal encoding cannot be used.\n",
      "Only one-hot encoding supports multiple levels in the same event.\n",
      "\n",
      "Example 13 below shows the condition variable summary that HED produces for the\n",
      "full [sub-002_task-FacePerception_run-1_events.tsv](./_static/data/sub-002_task-FacePerception_run-1_events.tsv) \n",
      "and JSON sidecar \n",
      "[task-facePerception_events.json](./_static/data/task-FacePerception_events.json).\n",
      "\n",
      "````{admonition} Example 13: The condition variable summary extracted from the full event file.\n",
      ":class: tip\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"key-assignment\": {\n",
      "      \"name\": \"key-assignment\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 1,\n",
      "      \"direct_references\": 0,\n",
      "      \"total_events\": 552,\n",
      "      \"number_type_events\": 552,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {\n",
      "         \"right-sym-cond\": 552\n",
      "      }\n",
      "   },\n",
      "   \"face-type\": {\n",
      "      \"name\": \"face-type\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 3,\n",
      "      \"direct_references\": 0,\n",
      "      \"total_events\": 552,\n",
      "      \"number_type_events\": 146,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {\n",
      "         \"unfamiliar-face-cond\": 47,\n",
      "         \"famous-face-cond\": 49,\n",
      "         \"scrambled-face-cond\": 50\n",
      "      }\n",
      "   },\n",
      "   \"repetition-type\": {\n",
      "      \"name\": \"repetition-type\",\n",
      "      \"variable_type\": \"condition-variable\",\n",
      "      \"levels\": 3,\n",
      "      \"direct_references\": 0,\n",
      "      \"total_events\": 552,\n",
      "      \"number_type_events\": 146,\n",
      "      \"number_multiple_events\": 0,\n",
      "      \"multiple_event_maximum\": 1,\n",
      "      \"level_counts\": {\n",
      "         \"first-show-cond\": 75,\n",
      "         \"immediate-repeat-cond\": 36,\n",
      "         \"delayed-repeat-cond\": 35\n",
      "      }\n",
      "    }\n",
      "}\n",
      "```\n",
      "````\n",
      "\n",
      "The file has 552 events.\n",
      "Since the *key-assignment* condition variable with level *right-sym-cond*\n",
      "applies to every event in this file, the *number_type_events* is also 552.\n",
      "On the other hand, the *face-type* condition variable is only applicable in 146 events.\n",
      "\n",
      "All the condition variables have *number_multiple_events* equal to 0,\n",
      "so any of the three possible encodings: categorical, ordinal, or one-hot can be used.\n",
      "\n",
      "\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\u001b[33massistant\u001b[0m (to ragproxyagent):\n",
      "\n",
      "Annotating three stimulus condition variables using HED can be carried out as shown below:\n",
      "\n",
      "1. Square stimulus: `Square-Stimulus: { \"HED\": \"Object/2D-Shape/Square, Attribute/Visual/Color/Dark-blue, Experimental-stimulus, Image-type/Still-image, Sensory-presentation, Visual-presentation, Condition-variable/Square-stimulus\" }`\n",
      "2. Circle stimulus: `Circle-Stimulus: { \"HED\": \"Object/2D-Shape/Circle, Attribute/Visual/Color/Green, Experimental-stimulus, Image-type/Still-image, Sensory-presentation, Visual-presentation, Condition-variable/Circle-stimulus\" }`\n",
      "3. Animal stimulus: `Animal-Stimulus: { \"HED\": \"Object-category/Animal, Experimental-stimulus, Image-type/Still-image, Sensory-presentation, Visual-presentation, Condition-variable/Animal-stimulus\" }`\n",
      "\n",
      "Each annotation consists of a defined condition variable (e.g., \"Condition-variable/Square-stimulus\") and associated tags describing the stimulus properties (e.g., \"Object/2D-Shape/Square\", \"Attribute/Visual/Color/Dark-blue\", etc.).\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\u001b[33mragproxyagent\u001b[0m (to assistant):\n",
      "\n",
      "Use definitions and also show example of event file for annotating the temporal scope of the conditions\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\u001b[33massistant\u001b[0m (to ragproxyagent):\n",
      "\n",
      "To define event conditions and also illustrate temporal annotation of the conditions, let's assume the following conditions in an experimental context:\n",
      "\n",
      "1. Square stimulus: This stimulus is presented to the participants for the first 10 minutes of the experiment.\n",
      "2. Circle stimulus: This is presented for the next 10 minutes of the experiment.\n",
      "3. Animal stimulus: This is presented for the last 10 minutes of the experiment.\n",
      "\n",
      "First, define the variables in your JSON sidecar file:\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"definitions\": {\n",
      "      \"HED\": {\n",
      "         \"square_stimulus_def\": \"(Definition/Square-stim-cond, (Condition-variable/Visual-stimulus, (Object/2D-Shape/Square, Attribute/Color/Dark-blue)))\",\n",
      "         \"circle_stimulus_def\": \"(Definition/Circle-stim-cond, (Condition-variable/Visual-stimulus, (Object/2D-Shape/Circle, Attribute/Color/Green)))\",\n",
      "         \"animal_stimulus_def\": \"(Definition/Animal-stim-cond, (Condition-variable/Visual-stimulus, Object-category/Animal))\" \n",
      "      }\n",
      "   }\n",
      "}\n",
      "```\n",
      "Now, your event file could look like this:\n",
      "\n",
      "```\n",
      "| onset | duration | event_type |\n",
      "| ----- | -------- | ---------- |\n",
      "| 0     | 600      | Square-stim-cond-start |\n",
      "| 600   | 600      | Square-stim-cond-end, Circle-stim-cond-start |\n",
      "| 1200  | 600      | Circle-stim-cond-end, Animal-stim-cond-start |\n",
      "| 1800  | --       | Animal-stim-cond-end |\n",
      "```\n",
      "In this event.tsv file...\n",
      "\n",
      "- Each row signifies the start or end of a condition. \"Square-stim-cond-start\" and \"Square-stim-cond-end\" represent the start and end times of the square stimulus condition.\n",
      "- The \"onset\" column specifies when the event starts, and \"duration\" indicates how long the event lasts.\n",
      "- Both the start and end of the conditions are marked. For example, at the time stamp 600, the Square condition ends and the Circle condition begins.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\u001b[33mragproxyagent\u001b[0m (to assistant):\n",
      "\n",
      "Can you use Onset and Offset tags?\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\u001b[33massistant\u001b[0m (to ragproxyagent):\n",
      "\n",
      "Yes, the HED (Hierarchical Event Descriptor) does use `Onset` and `Offset` tags to define the beginning and the end of an event. The temporal scope can be defined in JSON (`events.json`) file as shown below:\n",
      "\n",
      "```json\n",
      "{\n",
      "   \"event_type\": {\n",
      "      \"HED\": {\n",
      "         \"Square-stim-cond-start\": \"Event/Category/Stimulus, Event/Label/Square-stimulus, (Def/Square-stim-cond, Onset)\",\n",
      "         \"Square-stim-cond-end\": \"(Def/Square-stim-cond, Offset)\",\n",
      "         \"Circle-stim-cond-start\": \"Event/Category/Stimulus, Event/Label/Circle-stimulus, (Def/Circle-stim-cond, Onset)\",\n",
      "         \"Circle-stim-cond-end\": \"(Def/Circle-stim-cond, Offset)\",\n",
      "         \"Animal-stim-cond-start\": \"Event/Category/Stimulus, Event/Label/Animal-stimulus, (Def/Animal-stim-cond, Onset)\",\n",
      "         \"Animal-stim-cond-end\": \"(Def/Animal-stim-cond, Offset)\"\n",
      "      }\n",
      "   }\n",
      "}\n",
      "``` \n",
      "This uses `Onset` and `Offset` tags to mark the beginning and the end of the defined stimuli conditions respectively. This event information can be used by HED tools to extract the timing of different experimental conditions and stimuli presentations.\n",
      "\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# reset the assistant. Always reset the assistant before starting a new conversation.\n",
    "assistant.reset()\n",
    "\n",
    "# given a problem, we use the ragproxyagent to generate a prompt to be sent to the assistant as the initial message.\n",
    "# the assistant receives the message and generates a response. The response will be sent back to the ragproxyagent for processing.\n",
    "# The conversation continues until the termination condition is met, in RetrieveChat, the termination condition when no human-in-loop is no code block detected.\n",
    "# With human-in-loop, the conversation will continue until the user says \"exit\".\n",
    "\n",
    "# Wakeman Henson example\n",
    "# code_problem = \"Give example of annotating three face types condition variables using HED. The face types information is detailed below:\\n\" \\\n",
    "#     \"famous_face: A face that should be recognized by the participants.\\n\", \\\n",
    "#     \"unfamiliar_face: A face that should not be recognized by the participants.\\n\", \\\n",
    "#     \"scrambled_face: A scrambled face image generated by taking face 2D FFT.\"\n",
    "\n",
    "# Generic example\n",
    "# code_problem = \"Give example of annotating three stimulus condition variables using HED. The stimulus types information is detailed below:\\n\" \\\n",
    "#     \"square_stimulus: A picture of a dark blue square.\\n\", \\\n",
    "#     \"circle_stimulus: A picture of a green circle.\\n\", \\\n",
    "#     \"animal_stimulus: A picture of an animal.\"\n",
    "\n",
    "# Generic example with Onset and Offset suggestion\n",
    "code_problem = \"Give example of annotating three stimulus condition variables using HED. The stimulus types information is detailed below:\\n\" \\\n",
    "    \"square_stimulus: A picture of a dark blue square.\\n\", \\\n",
    "    \"circle_stimulus: A picture of a green circle.\\n\", \\\n",
    "    \"animal_stimulus: A picture of an animal.\"\n",
    "chat_result = ragproxyagent.initiate_chat(\n",
    "    assistant, \n",
    "    message=ragproxyagent.message_generator, \n",
    "    problem=code_problem, \n",
    "    search_string=\"conditions and design matrices\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c901be69",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/c2/k7vlgt7j1y306mj9yk_51jvr0000gn/T/ipykernel_3145/3844544330.py:20: XMLParsedAsHTMLWarning: It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features=\"xml\"` into the BeautifulSoup constructor.\n",
      "  soup = BeautifulSoup(xml_content, \"lxml\")\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from io import StringIO\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def get_hed_vocab():\n",
    "    if os.path.exists('HEDLatest_terms'):\n",
    "        with open('HEDLatest_terms', 'r') as fin:\n",
    "            return fin.read()\n",
    "    else:\n",
    "        # URL of the XML file\n",
    "        url = \"https://raw.githubusercontent.com/hed-standard/hed-schemas/main/standard_schema/hedxml/HEDLatest.xml\"\n",
    "        \n",
    "        # Send a GET request to the URL\n",
    "        response = requests.get(url)\n",
    "        \n",
    "        # Check if the request was successful (status code 200)\n",
    "        if response.status_code == 200:\n",
    "            # Parse the XML content\n",
    "            xml_content = response.text\n",
    "            soup = BeautifulSoup(xml_content, \"lxml\")\n",
    "        \n",
    "            # Find all nodes and extract their names\n",
    "            all_nodes = soup.find_all('node')\n",
    "            node_names = [node.find('name', recursive=False).string for node in all_nodes]\n",
    "        \n",
    "            return node_names\n",
    "        else:\n",
    "            print(f\"Failed to retrieve data from the URL. Status code: {response.status_code}\") \n",
    "hed_vocab = \",\".join(get_hed_vocab())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "378ab45c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}