diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py new file mode 100644 index 0000000000000..3a93baa588e0f --- /dev/null +++ b/.github/scripts/check_diff.py @@ -0,0 +1,45 @@ +import json +import sys + +ALL_DIRS = { + "libs/core", + "libs/langchain", + "libs/experimental", +} + +if __name__ == "__main__": + files = sys.argv[1:] + dirs_to_run = set() + + for file in files: + if any( + file.startswith(dir_) + for dir_ in ( + ".github/workflows", + ".github/tools", + ".github/actions", + "libs/core", + ".github/scripts/check_diff.py", + ) + ): + dirs_to_run = ALL_DIRS + break + elif "libs/community" in file: + dirs_to_run.update( + ("libs/community", "libs/langchain", "libs/experimental") + ) + elif "libs/partners" in file: + partner_dir = file.split("/")[2] + dirs_to_run.update( + (f"libs/partners/{partner_dir}", "libs/langchain", "libs/experimental") + ) + elif "libs/langchain" in file: + dirs_to_run.update(("libs/langchain", "libs/experimental")) + elif "libs/experimental" in file: + dirs_to_run.add("libs/experimental") + elif file.startswith("libs/"): + dirs_to_run = ALL_DIRS + break + else: + pass + print(json.dumps(list(dirs_to_run))) diff --git a/.github/workflows/langchain_ci.yml b/.github/workflows/_all_ci.yml similarity index 70% rename from .github/workflows/langchain_ci.yml rename to .github/workflows/_all_ci.yml index cd145e0612f30..e72bf101ba1a5 100644 --- a/.github/workflows/langchain_ci.yml +++ b/.github/workflows/_all_ci.yml @@ -1,21 +1,24 @@ --- -name: libs/langchain CI +name: langchain CI on: - push: - branches: [master] - pull_request: - paths: - - ".github/actions/poetry_setup/action.yml" - - ".github/tools/**" - - ".github/workflows/_lint.yml" - - ".github/workflows/_test.yml" - - ".github/workflows/_pydantic_compatibility.yml" - - ".github/workflows/langchain_ci.yml" - - "libs/*" - - "libs/langchain/**" - - "libs/core/**" - workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + workflow_dispatch: + inputs: + working-directory: + required: true + type: choice + default: 'libs/langchain' + options: + - libs/langchain + - libs/core + - libs/experimental + # If another push to the same PR or branch happens while this workflow is still running, # cancel the earlier run in favor of the next run. @@ -24,43 +27,40 @@ on: # a limited number of job runners to be active at the same time, so it's better to cancel # pointless jobs early so that more useful jobs can run sooner. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.working-directory }} cancel-in-progress: true env: POETRY_VERSION: "1.6.1" - WORKDIR: "libs/langchain" jobs: lint: uses: ./.github/workflows/_lint.yml with: - working-directory: libs/langchain + working-directory: ${{ inputs.working-directory }} secrets: inherit test: uses: ./.github/workflows/_test.yml with: - working-directory: libs/langchain + working-directory: ${{ inputs.working-directory }} secrets: inherit compile-integration-tests: uses: ./.github/workflows/_compile_integration_test.yml + if: ${{ inputs.working-directory != 'libs/core' }} with: - working-directory: libs/langchain + working-directory: ${{ inputs.working-directory }} secrets: inherit pydantic-compatibility: uses: ./.github/workflows/_pydantic_compatibility.yml with: - working-directory: libs/langchain + working-directory: ${{ inputs.working-directory }} secrets: inherit extended-tests: runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKDIR }} strategy: matrix: python-version: @@ -68,7 +68,11 @@ jobs: - "3.9" - "3.10" - "3.11" + if: ${{ inputs.working-directory == 'libs/langchain' }} name: Python ${{ matrix.python-version }} extended tests + defaults: + run: + working-directory: ${{ inputs.working-directory }} steps: - uses: actions/checkout@v4 @@ -77,7 +81,7 @@ jobs: with: python-version: ${{ matrix.python-version }} poetry-version: ${{ env.POETRY_VERSION }} - working-directory: libs/langchain + working-directory: ${{ inputs.working-directory }} cache-key: extended - name: Install dependencies diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml new file mode 100644 index 0000000000000..c09b6cc54635b --- /dev/null +++ b/.github/workflows/check_diffs.yml @@ -0,0 +1,47 @@ +--- +name: Check library diffs + +on: + push: + branches: [master] + pull_request: + paths: + - ".github/actions/**" + - ".github/tools/**" + - ".github/workflows/**" + - "libs/**" + +# If another push to the same PR or branch happens while this workflow is still running, +# cancel the earlier run in favor of the next run. +# +# There's no point in testing an outdated version of the code. GitHub only allows +# a limited number of job runners to be active at the same time, so it's better to cancel +# pointless jobs early so that more useful jobs can run sooner. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - id: files + uses: Ana06/get-changed-files@v2.2.0 + - id: set-matrix + run: echo "dirs-to-run=$(python .github/scripts/check_diff.py ${{ steps.files.outputs.all }})" >> $GITHUB_OUTPUT + outputs: + dirs-to-run: ${{ steps.set-matrix.outputs.dirs-to-run }} + ci: + needs: [ build ] + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + uses: ./.github/workflows/_all_ci.yml + with: + working-directory: ${{ matrix.working-directory }} + + diff --git a/.github/workflows/langchain_cli_ci.yml b/.github/workflows/langchain_cli_ci.yml deleted file mode 100644 index c9330c1c18ec4..0000000000000 --- a/.github/workflows/langchain_cli_ci.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: libs/cli CI - -on: - push: - branches: [ master ] - pull_request: - paths: - - '.github/actions/poetry_setup/action.yml' - - '.github/tools/**' - - '.github/workflows/_lint.yml' - - '.github/workflows/_test.yml' - - '.github/workflows/_pydantic_compatibility.yml' - - '.github/workflows/langchain_cli_ci.yml' - - 'libs/cli/**' - - 'libs/*' - workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI - -# If another push to the same PR or branch happens while this workflow is still running, -# cancel the earlier run in favor of the next run. -# -# There's no point in testing an outdated version of the code. GitHub only allows -# a limited number of job runners to be active at the same time, so it's better to cancel -# pointless jobs early so that more useful jobs can run sooner. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POETRY_VERSION: "1.6.1" - WORKDIR: "libs/cli" - -jobs: - lint: - uses: - ./.github/workflows/_lint.yml - with: - working-directory: libs/cli - langchain-location: ../langchain - secrets: inherit - - test: - uses: - ./.github/workflows/_test.yml - with: - working-directory: libs/cli - secrets: inherit diff --git a/.github/workflows/langchain_core_ci.yml b/.github/workflows/langchain_core_ci.yml deleted file mode 100644 index dc035ebfeb61a..0000000000000 --- a/.github/workflows/langchain_core_ci.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: libs/langchain core CI - -on: - push: - branches: [ master ] - pull_request: - paths: - - '.github/actions/poetry_setup/action.yml' - - '.github/tools/**' - - '.github/workflows/_lint.yml' - - '.github/workflows/_test.yml' - - '.github/workflows/_pydantic_compatibility.yml' - - '.github/workflows/langchain_core_ci.yml' - - 'libs/core/**' - workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI - -# If another push to the same PR or branch happens while this workflow is still running, -# cancel the earlier run in favor of the next run. -# -# There's no point in testing an outdated version of the code. GitHub only allows -# a limited number of job runners to be active at the same time, so it's better to cancel -# pointless jobs early so that more useful jobs can run sooner. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POETRY_VERSION: "1.6.1" - WORKDIR: "libs/core" - -jobs: - lint: - uses: - ./.github/workflows/_lint.yml - with: - working-directory: libs/core - secrets: inherit - - test: - uses: - ./.github/workflows/_test.yml - with: - working-directory: libs/core - secrets: inherit - - pydantic-compatibility: - uses: - ./.github/workflows/_pydantic_compatibility.yml - with: - working-directory: libs/core - secrets: inherit diff --git a/.github/workflows/langchain_experimental_ci.yml b/.github/workflows/langchain_experimental_ci.yml deleted file mode 100644 index 228fb66d1f1c2..0000000000000 --- a/.github/workflows/langchain_experimental_ci.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: libs/experimental CI - -on: - push: - branches: [master] - pull_request: - paths: - - ".github/actions/poetry_setup/action.yml" - - ".github/tools/**" - - ".github/workflows/_lint.yml" - - ".github/workflows/_test.yml" - - ".github/workflows/langchain_experimental_ci.yml" - - "libs/*" - - "libs/experimental/**" - - "libs/langchain/**" - - "libs/core/**" - workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI - -# If another push to the same PR or branch happens while this workflow is still running, -# cancel the earlier run in favor of the next run. -# -# There's no point in testing an outdated version of the code. GitHub only allows -# a limited number of job runners to be active at the same time, so it's better to cancel -# pointless jobs early so that more useful jobs can run sooner. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POETRY_VERSION: "1.6.1" - WORKDIR: "libs/experimental" - -jobs: - lint: - uses: ./.github/workflows/_lint.yml - with: - working-directory: libs/experimental - secrets: inherit - - test: - uses: ./.github/workflows/_test.yml - with: - working-directory: libs/experimental - secrets: inherit - - compile-integration-tests: - uses: ./.github/workflows/_compile_integration_test.yml - with: - working-directory: libs/experimental - secrets: inherit - - # It's possible that langchain-experimental works fine with the latest *published* langchain, - # but is broken with the langchain on `master`. - # - # We want to catch situations like that *before* releasing a new langchain, hence this test. - test-with-latest-langchain: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKDIR }} - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - name: test with unpublished langchain - Python ${{ matrix.python-version }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} - uses: "./.github/actions/poetry_setup" - with: - python-version: ${{ matrix.python-version }} - poetry-version: ${{ env.POETRY_VERSION }} - working-directory: ${{ env.WORKDIR }} - cache-key: unpublished-langchain - - - name: Install dependencies - shell: bash - run: | - echo "Running tests with unpublished langchain, installing dependencies with poetry..." - poetry install - - echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..." - poetry run pip install -e ../langchain - poetry run pip install -e ../core - - - name: Run tests - run: make test - extended-tests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ${{ env.WORKDIR }} - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - name: Python ${{ matrix.python-version }} extended tests - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} - uses: "./.github/actions/poetry_setup" - with: - python-version: ${{ matrix.python-version }} - poetry-version: ${{ env.POETRY_VERSION }} - working-directory: libs/experimental - cache-key: extended - - - name: Install dependencies - shell: bash - run: | - echo "Running extended tests, installing dependencies with poetry..." - poetry install -E extended_testing - - - name: Run extended tests - run: make extended_tests - - - name: Ensure the tests did not create any additional files - shell: bash - run: | - set -eu - - STATUS="$(git status)" - echo "$STATUS" - - # grep will exit non-zero if the target message isn't found, - # and `set -e` above will cause the step to fail. - echo "$STATUS" | grep 'nothing to commit, working tree clean'