Image prompt template

libs/core/langchain_core/prompt_values.py

@@ -3,6 +3,8 @@
 from abc import ABC, abstractmethod
 from typing import List, Literal, Sequence
 
+from typing_extensions import TypedDict
+
 from langchain_core.load.serializable import Serializable
 from langchain_core.messages import (
     AnyMessage,
@@ -67,6 +69,30 @@ def to_messages(self) -> List[BaseMessage]:
         return list(self.messages)
 
 
+class ImageURL(TypedDict, total=False):
+    detail: Literal["auto", "low", "high"]
+    """Specifies the detail level of the image."""
+
+    url: str
+    """Either a URL of the image or the base64 encoded image data."""
+
+
+class ImagePromptValue(PromptValue):
+    """Image prompt value."""
+
+    image_url: ImageURL
+    """Prompt image."""
+    type: Literal["ImagePromptValue"] = "ImagePromptValue"
+
+    def to_string(self) -> str:
+        """Return prompt as string."""
+        return self.image_url["url"]
+
+    def to_messages(self) -> List[BaseMessage]:
+        """Return prompt as messages."""
+        return [HumanMessage(content=[self.image_url])]
+
+
 class ChatPromptValueConcrete(ChatPromptValue):
     """Chat prompt value which explicitly lists out the message types it accepts.
     For use in external schemas."""

libs/core/langchain_core/prompts/base.py

@@ -8,10 +8,12 @@
     Any,
     Callable,
     Dict,
+    Generic,
     List,
     Mapping,
     Optional,
     Type,
+    TypeVar,
     Union,
 )
 
@@ -30,7 +32,12 @@
     from langchain_core.documents import Document
 
 
-class BasePromptTemplate(RunnableSerializable[Dict, PromptValue], ABC):
+FormatOutputType = TypeVar("FormatOutputType")
+
+
+class BasePromptTemplate(
+    RunnableSerializable[Dict, PromptValue], Generic[FormatOutputType], ABC
+):
     """Base class for all prompt templates, returning a prompt."""
 
     input_variables: List[str]
@@ -138,7 +145,7 @@ def _merge_partial_and_user_variables(self, **kwargs: Any) -> Dict[str, Any]:
         return {**partial_kwargs, **kwargs}
 
     @abstractmethod
-    def format(self, **kwargs: Any) -> str:
+    def format(self, **kwargs: Any) -> FormatOutputType:
         """Format the prompt with the inputs.
 
         Args:
@@ -206,7 +213,7 @@ def save(self, file_path: Union[Path, str]) -> None:
             raise ValueError(f"{save_path} must be json or yaml")
 
 
-def format_document(doc: Document, prompt: BasePromptTemplate) -> str:
+def format_document(doc: Document, prompt: BasePromptTemplate[str]) -> str:
     """Format a document into a string based on a prompt template.
 
     First, this pulls information from the document from two sources:
@@ -232,7 +239,7 @@ def format_document(doc: Document, prompt: BasePromptTemplate) -> str:
     Example:
         .. code-block:: python
 
-            from langchain_core import Document
+            from langchain_core.documents import Document
             from langchain_core.prompts import PromptTemplate
 
             doc = Document(page_content="This is a joke", metadata={"page": "1"})