merge

.github/workflows/_all_ci.yml

@@ -48,7 +48,6 @@ jobs:
 
   compile-integration-tests:
     uses: ./.github/workflows/_compile_integration_test.yml
-    if: ${{ inputs.working-directory != 'libs/core' }}
     with:
       working-directory: ${{ inputs.working-directory }}
     secrets: inherit
@@ -68,7 +67,6 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
-    if: ${{ inputs.working-directory == 'libs/langchain' }}
     name: Python ${{ matrix.python-version }} extended tests
     defaults:
       run:
@@ -88,12 +86,7 @@ jobs:
         shell: bash
         run: |
           echo "Running extended tests, installing dependencies with poetry..."
-          poetry install -E extended_testing
-
-      - name: Install langchain core editable
-        shell: bash
-        run: |
-          poetry run pip install -e ../core
+          poetry install -E extended_testing --with test
 
       - name: Run extended tests
         run: make extended_tests

.github/workflows/_compile_integration_test.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Install integration dependencies
         shell: bash
-        run: poetry install --with=test_integration
+        run: poetry install --with=test_integration,test
 
       - name: Check integration tests compile
         shell: bash

.github/workflows/_lint.yml

@@ -90,4 +90,31 @@ jobs:
       - name: Analysing the code with our lint
         working-directory: ${{ inputs.working-directory }}
         run: |
-          make lint
+          make lint_package
+
+      - name: Install test dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry install --with test
+
+      - name: Get .mypy_cache to speed up mypy
+        uses: actions/cache@v3
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
+        with:
+          path: |
+            ${{ env.WORKDIR }}/.mypy_cache
+          key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
+
+      - name: Analysing the code with our lint
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          make lint_tests

.github/workflows/_pydantic_compatibility.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Install dependencies
         shell: bash
-        run: poetry install
+        run: poetry install  --with test
 
       - name: Install langchain editable
         working-directory: ${{ inputs.working-directory }}

.github/workflows/templates_ci.yml

@@ -33,5 +33,4 @@ jobs:
       ./.github/workflows/_lint.yml
     with:
       working-directory: templates
-      langchain-location: ../libs/langchain
     secrets: inherit

.gitignore

@@ -167,8 +167,7 @@ docs/node_modules/
 docs/.docusaurus/
 docs/.cache-loader/
 docs/_dist
-docs/api_reference/api_reference.rst
-docs/api_reference/experimental_api_reference.rst
+docs/api_reference/*api_reference.rst
 docs/api_reference/_build
 docs/api_reference/*/
 !docs/api_reference/_static/

.scripts/community_split/script_integrations.sh

@@ -267,10 +267,6 @@ git checkout master -- langchain/tests/unit_tests/schema
 touch langchain/tests/unit_tests/{llms,chat_models,tools,callbacks,runnables,document_loaders,docstore,document_transformers,embeddings,graphs,storage,utilities,vectorstores}/__init__.py
 touch langchain/tests/unit_tests/document_loaders/{blob_loaders,parsers}/__init__.py
 
-cp core/Makefile community
-cp core/Makefile partners/openai
-sed -i '' 's/libs\/core/libs\/community/g' community/Makefile
-sed -i '' 's/libs\/core/libs\/partners\/openai/g' partners/openai/Makefile
 cp -r core/scripts community
 cp -r core/scripts partners/openai
 

Makefile

@@ -41,7 +41,7 @@ spell_fix:
 # LINTING AND FORMATTING
 ######################
 
-lint:
+lint lint_package lint_tests:
 	poetry run ruff docs templates cookbook
 	poetry run ruff format docs templates cookbook --diff
 	poetry run ruff --select I docs templates cookbook

docs/api_reference/create_api_rst.py

@@ -196,11 +196,13 @@ def _load_package_modules(
     return modules_by_namespace
 
 
-def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) -> str:
+def _construct_doc(
+    package_namespace: str, members_by_namespace: Dict[str, ModuleMembers]
+) -> str:
     """Construct the contents of the reference.rst file for the given package.
 
     Args:
-        pkg: The package name
+        package_namespace: The package top level namespace
         members_by_namespace: The members of the package, dict organized by top level
                               module contains a list of classes and functions
                               inside of the top level namespace.
@@ -210,7 +212,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
     """
     full_doc = f"""\
 =======================
-``{pkg}`` API Reference
+``{package_namespace}`` API Reference
 =======================
 
 """
@@ -222,13 +224,13 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
         functions = _members["functions"]
         if not (classes or functions):
             continue
-        section = f":mod:`{pkg}.{module}`"
+        section = f":mod:`{package_namespace}.{module}`"
         underline = "=" * (len(section) + 1)
         full_doc += f"""\
 {section}
 {underline}
 
-.. automodule:: {pkg}.{module}
+.. automodule:: {package_namespace}.{module}
     :no-members:
     :no-inherited-members:
 
@@ -238,7 +240,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
             full_doc += f"""\
 Classes
 --------------
-.. currentmodule:: {pkg}
+.. currentmodule:: {package_namespace}
 
 .. autosummary::
     :toctree: {module}
@@ -270,7 +272,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
             full_doc += f"""\
 Functions
 --------------
-.. currentmodule:: {pkg}
+.. currentmodule:: {package_namespace}
 
 .. autosummary::
     :toctree: {module}
@@ -282,57 +284,57 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
     return full_doc
 
 
-def _document_langchain_experimental() -> None:
-    """Document the langchain_experimental package."""
-    # Generate experimental_api_reference.rst
-    exp_members = _load_package_modules(EXP_DIR)
-    exp_doc = ".. _experimental_api_reference:\n\n" + _construct_doc(
-        "langchain_experimental", exp_members
-    )
-    with open(EXP_WRITE_FILE, "w") as f:
-        f.write(exp_doc)
+def _build_rst_file(package_name: str = "langchain") -> None:
+    """Create a rst file for building of documentation.
 
+    Args:
+        package_name: Can be either "langchain" or "core" or "experimental".
+    """
+    package_members = _load_package_modules(_package_dir(package_name))
+    with open(_out_file_path(package_name), "w") as f:
+        f.write(
+            _doc_first_line(package_name)
+            + _construct_doc(package_namespace[package_name], package_members)
+        )
 
-def _document_langchain_core() -> None:
-    """Document the langchain_core package."""
-    # Generate core_api_reference.rst
-    core_members = _load_package_modules(CORE_DIR)
-    core_doc = ".. _core_api_reference:\n\n" + _construct_doc(
-        "langchain_core", core_members
-    )
-    with open(CORE_WRITE_FILE, "w") as f:
-        f.write(core_doc)
-
-
-def _document_langchain() -> None:
-    """Document the main langchain package."""
-    # load top level module members
-    lc_members = _load_package_modules(PKG_DIR)
-
-    # Add additional packages
-    tools = _load_package_modules(PKG_DIR, "tools")
-    agents = _load_package_modules(PKG_DIR, "agents")
-    schema = _load_package_modules(PKG_DIR, "schema")
-
-    lc_members.update(
-        {
-            "agents.output_parsers": agents["output_parsers"],
-            "agents.format_scratchpad": agents["format_scratchpad"],
-            "tools.render": tools["render"],
-        }
-    )
 
-    lc_doc = ".. _api_reference:\n\n" + _construct_doc("langchain", lc_members)
+package_namespace = {
+    "langchain": "langchain",
+    "experimental": "langchain_experimental",
+    "core": "langchain_core",
+}
+
+
+def _package_dir(package_name: str = "langchain") -> Path:
+    """Return the path to the directory containing the documentation."""
+    return ROOT_DIR / "libs" / package_name / package_namespace[package_name]
+
+
+def _out_file_path(package_name: str = "langchain") -> Path:
+    """Return the path to the file containing the documentation."""
+    name_prefix = {
+        "langchain": "",
+        "experimental": "experimental_",
+        "core": "core_",
+    }
+    return HERE / f"{name_prefix[package_name]}api_reference.rst"
+
 
-    with open(WRITE_FILE, "w") as f:
-        f.write(lc_doc)
+def _doc_first_line(package_name: str = "langchain") -> str:
+    """Return the path to the file containing the documentation."""
+    prefix = {
+        "langchain": "",
+        "experimental": "experimental",
+        "core": "core",
+    }
+    return f".. {prefix[package_name]}_api_reference:\n\n"
 
 
 def main() -> None:
-    """Generate the reference.rst file for each package."""
-    _document_langchain()
-    _document_langchain_experimental()
-    _document_langchain_core()
+    """Generate the api_reference.rst file for each package."""
+    _build_rst_file(package_name="core")
+    _build_rst_file(package_name="langchain")
+    _build_rst_file(package_name="experimental")
 
 
 if __name__ == "__main__":

libs/community/Makefile

@@ -24,8 +24,10 @@ test_watch:
 PYTHON_FILES=.
 lint format: PYTHON_FILES=.
 lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/community -name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=langchain_community
+lint_tests: PYTHON_FILES=tests
 
-lint lint_diff:
+lint lint_diff lint_package lint_tests:
 	./scripts/check_pydantic.sh .
 	./scripts/check_imports.sh
 	poetry run ruff .

libs/community/langchain_community/vectorstores/clarifai.py

@@ -8,7 +8,6 @@
 from typing import Any, Iterable, List, Optional, Tuple
 
 import requests
-from google.protobuf.struct_pb2 import Struct
 from langchain_core.documents import Document
 from langchain_core.embeddings import Embeddings
 from langchain_core.vectorstores import VectorStore
@@ -87,6 +86,7 @@ def add_texts(
         """
         try:
             from clarifai.client.input import Inputs
+            from google.protobuf.struct_pb2 import Struct
         except ImportError as e:
             raise ImportError(
                 "Could not import clarifai python package. "