From 4b8ac7991b6c63c737d056275fb436662e1798c1 Mon Sep 17 00:00:00 2001 From: Cms Build Date: Wed, 1 Nov 2023 13:20:55 +0100 Subject: [PATCH] [CMS] Changes to build TF cms externals --- tensorflow/core/kernels/matmul_op_impl.h | 5 + tensorflow/workspace0.bzl | 6 +- tensorflow/workspace1.bzl | 2 - tensorflow/workspace2.bzl | 138 +----------- third_party/cms/BUILD | 0 third_party/cms/repo.bzl | 43 ++++ third_party/cms/workspace.bzl | 211 ++++++++++++++++++ third_party/flatbuffers/BUILD.system | 9 +- third_party/googleapis/build_rules.bzl | 2 +- third_party/hwloc/BUILD.system | 3 +- third_party/jpeg/BUILD.system | 3 +- .../systemlibs/backports_weakref.BUILD | 9 + third_party/systemlibs/curl.BUILD | 3 +- third_party/systemlibs/eigen.BUILD | 41 ++++ third_party/systemlibs/gif.BUILD | 3 +- third_party/systemlibs/grpc.BUILD | 10 + .../systemlibs/grpc.bazel.cc_grpc_library.bzl | 2 +- .../systemlibs/grpc.bazel.protobuf.bzl | 1 + third_party/systemlibs/png.BUILD | 3 +- third_party/systemlibs/protobuf.BUILD | 36 ++- third_party/systemlibs/pybind11.BUILD | 7 + third_party/systemlibs/sqlite.BUILD | 3 +- third_party/systemlibs/syslibs_configure.bzl | 2 + third_party/systemlibs/zlib.BUILD | 3 +- .../tsl/tsl/platform/default/build_config.bzl | 2 +- 25 files changed, 389 insertions(+), 158 deletions(-) create mode 100644 third_party/cms/BUILD create mode 100644 third_party/cms/repo.bzl create mode 100644 third_party/cms/workspace.bzl create mode 100644 third_party/systemlibs/backports_weakref.BUILD create mode 100644 third_party/systemlibs/eigen.BUILD diff --git a/tensorflow/core/kernels/matmul_op_impl.h b/tensorflow/core/kernels/matmul_op_impl.h index 71f338f266bf55a..8da13c91e305ea8 100644 --- a/tensorflow/core/kernels/matmul_op_impl.h +++ b/tensorflow/core/kernels/matmul_op_impl.h @@ -93,6 +93,7 @@ struct ParallelMatMulKernel { z.device(d) = z.conjugate(); } + __attribute__((used)) static void Run(const OpKernelContext* context, const Tensor& in_x, const Tensor& in_y, bool adj_x, bool adj_y, bool trans_x, bool trans_y, const MatMulBCast& bcast, Tensor* out, @@ -138,6 +139,7 @@ template struct ParallelMatMulKernel { static void Conjugate(const OpKernelContext* context, Tensor* out) {} + __attribute__((used)) static void Run(const OpKernelContext* context, const Tensor& in_x, const Tensor& in_y, bool adj_x, bool adj_y, bool trans_x, bool trans_y, const MatMulBCast& bcast, Tensor* out, @@ -209,6 +211,7 @@ struct SequentialMatMulKernel { t->dim_size(1), t->dim_size(2)); } + __attribute__((used)) static void Run(const Tensor& in_x, const Tensor& in_y, bool adj_x, bool adj_y, bool trans_x, bool trans_y, const MatMulBCast& bcast, Tensor* out, int start, int limit) { @@ -270,6 +273,8 @@ struct SingleBatchParallelMatMulKernel { return MatrixMap(t->flat().data(), t->dim_size(1), t->dim_size(2)); } + + __attribute__((used)) static void Run(const CPUDevice& device, const Tensor& in_x, const Tensor& in_y, bool adj_x, bool adj_y, bool trans_x, bool trans_y, Tensor* out) { diff --git a/tensorflow/workspace0.bzl b/tensorflow/workspace0.bzl index d8b53e1ef8b3be5..c8eb459b87ae6f4 100644 --- a/tensorflow/workspace0.bzl +++ b/tensorflow/workspace0.bzl @@ -6,7 +6,6 @@ load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_reposi load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies") load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") -load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") load("@local_config_android//:android.bzl", "android_workspace") load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") @@ -22,11 +21,11 @@ def _tf_bind(): # Needed by Protobuf native.bind( name = "grpc_cpp_plugin", - actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", + actual = "@com_github_grpc_grpc//:grpc_cpp_plugin", ) native.bind( name = "grpc_python_plugin", - actual = "@com_github_grpc_grpc//src/compiler:grpc_python_plugin", + actual = "@com_github_grpc_grpc//:grpc_python_plugin", ) native.bind( @@ -132,7 +131,6 @@ def workspace(): # at the end of the WORKSPACE file. _tf_bind() - grpc_extra_deps() rules_foreign_cc_dependencies() config_googleapis() diff --git a/tensorflow/workspace1.bzl b/tensorflow/workspace1.bzl index 9b092a10bf33109..a3f1456d39e4224 100644 --- a/tensorflow/workspace1.bzl +++ b/tensorflow/workspace1.bzl @@ -2,7 +2,6 @@ load("//third_party/android:android_configure.bzl", "android_configure") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps") load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories") load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") @@ -31,7 +30,6 @@ def workspace(with_rules_cc = True): android_configure(name = "local_config_android") - grpc_deps() benchmark_deps() # Alias so it can be loaded without assigning to a different symbol to prevent diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl index 65074788800b787..60eabaad9a3f84c 100644 --- a/tensorflow/workspace2.bzl +++ b/tensorflow/workspace2.bzl @@ -24,16 +24,13 @@ load("//third_party/absl:workspace.bzl", absl = "repo") load("//third_party/benchmark:workspace.bzl", benchmark = "repo") load("//third_party/dlpack:workspace.bzl", dlpack = "repo") load("//third_party/ducc:workspace.bzl", ducc = "repo") -load("//third_party/eigen3:workspace.bzl", eigen3 = "repo") load("//third_party/farmhash:workspace.bzl", farmhash = "repo") -load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo") load("//third_party/gemmlowp:workspace.bzl", gemmlowp = "repo") load("//third_party/hexagon:workspace.bzl", hexagon_nn = "repo") load("//third_party/highwayhash:workspace.bzl", highwayhash = "repo") load("//third_party/hwloc:workspace.bzl", hwloc = "repo") load("//third_party/implib_so:workspace.bzl", implib_so = "repo") load("//third_party/icu:workspace.bzl", icu = "repo") -load("//third_party/jpeg:workspace.bzl", jpeg = "repo") load("//third_party/libprotobuf_mutator:workspace.bzl", libprotobuf_mutator = "repo") load("//third_party/nasm:workspace.bzl", nasm = "repo") load("//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") @@ -57,6 +54,9 @@ load("//tensorflow/tools/toolchains/remote:configure.bzl", "remote_execution_con load("//tensorflow/tools/toolchains/clang6:repo.bzl", "clang6_configure") load("@rules_jvm_external//:defs.bzl", "maven_install") +#import CMS specific repos +load("//third_party/cms:workspace.bzl", cms= "repos") + def _initialize_third_party(): """ Load third party repositories. See above load() statements. """ FP16() @@ -65,16 +65,12 @@ def _initialize_third_party(): benchmark() ducc() dlpack() - eigen3() farmhash() - flatbuffers() gemmlowp() hexagon_nn() highwayhash() - hwloc() icu() implib_so() - jpeg() kissfft() libprotobuf_mutator() ml_dtypes() @@ -89,6 +85,7 @@ def _initialize_third_party(): vulkan_headers() tensorrt() triton() + cms() # copybara: tsl vendor @@ -309,74 +306,6 @@ def _tf_repositories(): urls = tf_mirror_urls("https://github.com/googleapis/googleapis/archive/6b3fdcea8bc5398be4e7e9930c693f0ea09316a0.tar.gz"), ) - tf_http_archive( - name = "png", - build_file = "//third_party:png.BUILD", - patch_file = ["//third_party:png_fix_rpi.patch"], - sha256 = "a00e9d2f2f664186e4202db9299397f851aea71b36a35e74910b8820e380d441", - strip_prefix = "libpng-1.6.39", - system_build_file = "//third_party/systemlibs:png.BUILD", - urls = tf_mirror_urls("https://github.com/glennrp/libpng/archive/v1.6.39.tar.gz"), - ) - - tf_http_archive( - name = "org_sqlite", - build_file = "//third_party:sqlite.BUILD", - sha256 = "bb5849ae4d7129c09d20596379a0b3f7b1ac59cf9998eba5ef283ea9b6c000a5", - strip_prefix = "sqlite-amalgamation-3430000", - system_build_file = "//third_party/systemlibs:sqlite.BUILD", - urls = tf_mirror_urls("https://www.sqlite.org/2023/sqlite-amalgamation-3430000.zip"), - ) - - tf_http_archive( - name = "gif", - build_file = "//third_party:gif.BUILD", - patch_file = [ - "//third_party:gif_fix_strtok_r.patch", - "//third_party:gif_fix_image_counter.patch", - ], - sha256 = "31da5562f44c5f15d63340a09a4fd62b48c45620cd302f77a6d9acf0077879bd", - strip_prefix = "giflib-5.2.1", - system_build_file = "//third_party/systemlibs:gif.BUILD", - urls = tf_mirror_urls("https://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.2.1.tar.gz"), - ) - - tf_http_archive( - name = "six_archive", - build_file = "//third_party:six.BUILD", - sha256 = "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - strip_prefix = "six-1.16.0", - system_build_file = "//third_party/systemlibs:six.BUILD", - urls = tf_mirror_urls("https://pypi.python.org/packages/source/s/six/six-1.16.0.tar.gz"), - ) - - tf_http_archive( - name = "absl_py", - sha256 = "a7c51b2a0aa6357a9cbb2d9437e8cd787200531867dc02565218930b6a32166e", - strip_prefix = "abseil-py-1.0.0", - system_build_file = "//third_party/systemlibs:absl_py.BUILD", - system_link_files = { - "//third_party/systemlibs:absl_py.absl.BUILD": "absl/BUILD", - "//third_party/systemlibs:absl_py.absl.flags.BUILD": "absl/flags/BUILD", - "//third_party/systemlibs:absl_py.absl.testing.BUILD": "absl/testing/BUILD", - "//third_party/systemlibs:absl_py.absl.logging.BUILD": "absl/logging/BUILD", - }, - urls = tf_mirror_urls("https://github.com/abseil/abseil-py/archive/refs/tags/v1.0.0.tar.gz"), - ) - - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - tf_http_archive( name = "nsync", patch_file = ["//third_party:nsync.patch"], @@ -407,37 +336,6 @@ def _tf_repositories(): urls = tf_mirror_urls("https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"), ) - tf_http_archive( - name = "curl", - build_file = "//third_party:curl.BUILD", - sha256 = "816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427", - strip_prefix = "curl-8.4.0", - system_build_file = "//third_party/systemlibs:curl.BUILD", - urls = tf_mirror_urls("https://curl.se/download/curl-8.4.0.tar.gz"), - ) - - # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD": "bazel/BUILD", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( name = "linenoise", build_file = "//third_party:linenoise.BUILD", @@ -474,16 +372,6 @@ def _tf_repositories(): urls = tf_mirror_urls("https://github.com/google/boringssl/archive/c00d7ca810e93780bd0c8ee4eea28f4f2ea4bcdc.tar.gz"), ) - # Note: if you update this, you have to update libpng too. See cl/437813808 - tf_http_archive( - name = "zlib", - build_file = "//third_party:zlib.BUILD", - sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", - strip_prefix = "zlib-1.2.13", - system_build_file = "//third_party/systemlibs:zlib.BUILD", - urls = tf_mirror_urls("https://zlib.net/fossils/zlib-1.2.13.tar.gz"), - ) - # LINT.IfChange tf_http_archive( name = "fft2d", @@ -597,15 +485,6 @@ def _tf_repositories(): urls = tf_mirror_urls("https://github.com/NVIDIA/NVTX/archive/v3.0.1.tar.gz"), ) - tf_http_archive( - name = "cython", - build_file = "//third_party:cython.BUILD", - sha256 = "0c2eae8a4ceab7955be1e11a4ddc5dcc3aa06ce22ad594262f1555b9d10667f0", - strip_prefix = "cython-3.0.3", - system_build_file = "//third_party/systemlibs:cython.BUILD", - urls = tf_mirror_urls("https://github.com/cython/cython/archive/3.0.3.tar.gz"), - ) - # LINT.IfChange tf_http_archive( name = "arm_neon_2_x86_sse", @@ -755,15 +634,6 @@ def _tf_repositories(): urls = tf_mirror_urls("https://github.com/nlohmann/json/archive/v3.10.5.tar.gz"), ) - tf_http_archive( - name = "pybind11", - urls = tf_mirror_urls("https://github.com/pybind/pybind11/archive/v2.10.4.tar.gz"), - sha256 = "832e2f309c57da9c1e6d4542dedd34b24e4192ecb4d62f6f4866a737454c9970", - strip_prefix = "pybind11-2.10.4", - build_file = "//third_party:pybind11.BUILD", - system_build_file = "//third_party/systemlibs:pybind11.BUILD", - ) - tf_http_archive( name = "pybind11_protobuf", urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.zip"), diff --git a/third_party/cms/BUILD b/third_party/cms/BUILD new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/third_party/cms/repo.bzl b/third_party/cms/repo.bzl new file mode 100644 index 000000000000000..a4760e5a3ba80e0 --- /dev/null +++ b/third_party/cms/repo.bzl @@ -0,0 +1,43 @@ +def _cms_new_local_repository(ctx): + cmd = ["mkdir", "-p", ctx.path(".")] + ctx.execute(cmd) + + cms_root_file = ctx.os.environ["TF_CMS_EXTERNALS"] + cmd = ["cat", cms_root_file] + res = ctx.execute(cmd) + cms_path = [ + line.split(":", 1)[1] + for line in res.stdout.split("\n") + if line.startswith(ctx.attr.name + ":") + ][-1].strip() + + if cms_path: + cmd = ["ls", cms_path + "/"] + res = ctx.execute(cmd) + for rfile in res.stdout.split("\n"): + if rfile: + ctx.symlink(cms_path + "/" + rfile, ctx.path(rfile)) + + ctx.template("BUILD.bazel", ctx.attr.build_file, { + "%prefix%": "external", + }, False) + + for xfile in ctx.attr.create_files: + ctx.file(xfile, content="") + + for src, dest in ctx.attr.symlinks.items(): + if src.startswith("//") or src.startswith("@"): + src = Label(src) + ctx.symlink(src, dest) + +cms_new_local_repository = repository_rule( + implementation = _cms_new_local_repository, + attrs = { + "build_file": attr.label(), + "symlinks": attr.string_dict(), + "create_files": attr.string_list(), + }, + environ = [ + "TF_CMS_EXTERNALS", + ], +) diff --git a/third_party/cms/workspace.bzl b/third_party/cms/workspace.bzl new file mode 100644 index 000000000000000..edf50aee7d3c2f4 --- /dev/null +++ b/third_party/cms/workspace.bzl @@ -0,0 +1,211 @@ +load("//third_party/cms:repo.bzl", "cms_new_local_repository") + +def png(): + cms_new_local_repository( + name = "png", + build_file = "//third_party/systemlibs:png.BUILD", + create_files = ["COPYING"], + ) + +def jpeg(): + cms_new_local_repository( + name = "libjpeg_turbo", + build_file = "//third_party/jpeg:BUILD.system", + create_files = ["LICENSE.md"], + ) + +def gif(): + cms_new_local_repository( + name = "gif", + build_file = "//third_party/systemlibs:gif.BUILD", + create_files = ["COPYING"], + ) + +def hwloc(): + cms_new_local_repository( + name = "hwloc", + build_file = "//third_party/hwloc:BUILD.system", + create_files = ["COPYING"], + ) + +def grpc(): + cms_new_local_repository( + name = "com_github_grpc_grpc", + build_file = "//third_party/systemlibs:grpc.BUILD", + create_files = ["LICENSE"], + symlinks = { + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + "//third_party/systemlibs:BUILD": "bazel/BUILD", + }, + ) + +def eigen(): + cms_new_local_repository( + name = "eigen_archive", + build_file = "//third_party/systemlibs:eigen.BUILD", + create_files = ["COPYING.MPL2"], + ) + +def protobuf(): + cms_new_local_repository( + name = "com_google_protobuf", + build_file = "//third_party/systemlibs:protobuf.BUILD", + create_files = ["LICENSE"], + symlinks = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + }, + ) + +def pcre(): + cms_new_local_repository( + name = "pcre", + build_file = "//third_party/systemlibs:pcre.BUILD", + create_files = ["LICENCE"], + ) + +def pybind11(): + cms_new_local_repository( + name = "pybind11", + build_file = "//third_party/systemlibs:pybind11.BUILD", + create_files = ["LICENCE"], + ) + +def flatbuffers(): + cms_new_local_repository( + name = "flatbuffers", + build_file = "//third_party/flatbuffers:BUILD.system", + create_files = ["LICENSE"], + symlinks = { + "//third_party/flatbuffers:build_defs.bzl": "build_defs.bzl", + }, + ) + +def curl(): + cms_new_local_repository( + name = "curl", + build_file = "//third_party/systemlibs:curl.BUILD", + create_files = ["COPYING"], + ) + +def zlib(): + cms_new_local_repository( + name = "zlib", + build_file = "//third_party/systemlibs:zlib.BUILD", + symlinks = { + "include/zlib.h": "zlib.h", + }, + ) + +def cython(): + cms_new_local_repository( + name = "cython", + build_file = "//third_party/systemlibs:cython.BUILD", + ) + +def functools32(): + cms_new_local_repository( + name = "functools32_archive", + build_file = "//third_party/systemlibs:functools32.BUILD", + ) + +def astor(): + cms_new_local_repository( + name = "astor_archive", + build_file = "//third_party/systemlibs:astor.BUILD", + ) + +def six(): + cms_new_local_repository( + name = "six_archive", + build_file = "//third_party/systemlibs:six.BUILD", + ) + +def termcolor(): + cms_new_local_repository( + name = "termcolor_archive", + build_file = "//third_party/systemlibs:termcolor.BUILD", + ) + +def typing_extensions(): + cms_new_local_repository( + name = "typing_extensions_archive", + build_file = "//third_party/systemlibs:typing_extensions.BUILD", + ) + +def sqlite(): + cms_new_local_repository( + name = "org_sqlite", + build_file = "//third_party/systemlibs:sqlite.BUILD", + ) + +def absl_py(): + cms_new_local_repository( + name = "absl_py", + build_file = "//third_party/systemlibs:absl_py.BUILD", + symlinks = { + "//third_party/systemlibs:absl_py.absl.BUILD": "absl/BUILD", + "//third_party/systemlibs:absl_py.absl.flags.BUILD": "absl/flags/BUILD", + "//third_party/systemlibs:absl_py.absl.logging.BUILD": "absl/logging/BUILD", + "//third_party/systemlibs:absl_py.absl.testing.BUILD": "absl/testing/BUILD", + }, + ) + +def pasta(): + cms_new_local_repository( + name = "pasta", + build_file = "//third_party/pasta:BUILD.system", + ) + +def wrapt(): + cms_new_local_repository( + name = "wrapt", + build_file = "//third_party/systemlibs:wrapt.BUILD", + ) + +def gast(): + cms_new_local_repository( + name = "gast_archive", + build_file = "//third_party/systemlibs:gast.BUILD", + ) + +def backports_weakref(): + cms_new_local_repository( + name = "org_python_pypi_backports_weakref", + build_file = "//third_party/systemlibs:backports_weakref.BUILD", + create_files = ["LICENSE"], + ) + +def opt_einsum(): + cms_new_local_repository( + name = "opt_einsum_archive", + build_file = "//third_party/systemlibs:opt_einsum.BUILD", + ) + +def repos(): + png() + jpeg() + gif() + hwloc() + grpc() + eigen() + protobuf() + pybind11() + curl() + zlib() + cython() + flatbuffers() + functools32() + astor() + six() + termcolor() + typing_extensions() + sqlite() + absl_py() + pasta() + wrapt() + gast() + backports_weakref() diff --git a/third_party/flatbuffers/BUILD.system b/third_party/flatbuffers/BUILD.system index 8fe4d7a590719f3..60da82c79e5c467 100644 --- a/third_party/flatbuffers/BUILD.system +++ b/third_party/flatbuffers/BUILD.system @@ -1,21 +1,23 @@ licenses(["notice"]) # Apache 2.0 filegroup( - name = "LICENSE.txt", + name = "LICENSE", visibility = ["//visibility:public"], ) # Public flatc library to compile flatbuffer files at runtime. cc_library( name = "flatbuffers", - linkopts = ["-lflatbuffers"], + linkopts = ["-Lexternal/flatbuffers/lib64 -lflatbuffers"], + includes = ["include"], visibility = ["//visibility:public"], ) # Public flatc compiler library. cc_library( name = "flatc_library", - linkopts = ["-lflatbuffers"], + linkopts = ["-Lexternal/flatbuffers/lib64 -lflatbuffers"], + includes = ["include"], visibility = ["//visibility:public"], ) @@ -34,6 +36,7 @@ sh_binary( cc_library( name = "runtime_cc", + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/googleapis/build_rules.bzl b/third_party/googleapis/build_rules.bzl index 377d74be1adaf20..f91d75fabbae4b5 100644 --- a/third_party/googleapis/build_rules.bzl +++ b/third_party/googleapis/build_rules.bzl @@ -75,7 +75,7 @@ def cc_grpc_library(name, srcs, deps, service_namespace = "grpc", **kwargs): flags = [ "services_namespace=" + service_namespace, ], - plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", + plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin", well_known_protos = True, generate_mocks = True, ) diff --git a/third_party/hwloc/BUILD.system b/third_party/hwloc/BUILD.system index 2989102a1d96c9d..c089d6e82e305bc 100644 --- a/third_party/hwloc/BUILD.system +++ b/third_party/hwloc/BUILD.system @@ -15,8 +15,9 @@ filegroup( cc_library( name = "hwloc", linkopts = select({ - ":with_numa_support": ["-lhwloc"], + ":with_numa_support": ["-Lexternal/hwloc/lib -lhwloc"], "//conditions:default": [], }), + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/jpeg/BUILD.system b/third_party/jpeg/BUILD.system index f4f52da9bdae1be..15efd19e8fd42aa 100644 --- a/third_party/jpeg/BUILD.system +++ b/third_party/jpeg/BUILD.system @@ -7,6 +7,7 @@ filegroup( cc_library( name = "jpeg", - linkopts = ["-ljpeg"], + linkopts = ["-Lexternal/libjpeg_turbo/lib64 -ljpeg"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/backports_weakref.BUILD b/third_party/systemlibs/backports_weakref.BUILD new file mode 100644 index 000000000000000..e878b73ef11196a --- /dev/null +++ b/third_party/systemlibs/backports_weakref.BUILD @@ -0,0 +1,9 @@ +py_library( + name = "org_python_pypi_backports_weakref", + visibility = ["//visibility:public"], +) + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/curl.BUILD b/third_party/systemlibs/curl.BUILD index c5f125caa9eb46d..9e2dd123c92f00f 100644 --- a/third_party/systemlibs/curl.BUILD +++ b/third_party/systemlibs/curl.BUILD @@ -7,6 +7,7 @@ filegroup( cc_library( name = "curl", - linkopts = ["-lcurl"], + linkopts = ["-Lexternal/curl/lib -lcurl"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/eigen.BUILD b/third_party/systemlibs/eigen.BUILD new file mode 100644 index 000000000000000..443a8f89caab181 --- /dev/null +++ b/third_party/systemlibs/eigen.BUILD @@ -0,0 +1,41 @@ +# Description: +# Eigen is a C++ template library for linear algebra: vectors, +# matrices, and related algorithms. + +licenses([ + # Note: Eigen is an MPL2 library that includes GPL v3 and LGPL v2.1+ code. + # We've taken special care to not reference any restricted code. + "reciprocal", # MPL2 + "notice", # Portions BSD +]) + +filegroup( + name = "COPYING.MPL2", + visibility = ["//visibility:public"], +) + +cc_library( + name = "eigen", + includes = ["include/eigen3"], + defines = [ + # This define (mostly) guarantees we don't link any problematic + # code. We use it, but we do not rely on it, as evidenced above. + "EIGEN_MPL2_ONLY", + "EIGEN_MAX_ALIGN_BYTES=64", + "EIGEN_HAS_TYPE_TRAITS=0", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "eigen3", + hdrs = [], + defines = [ + # This define (mostly) guarantees we don't link any problematic + # code. We use it, but we do not rely on it, as evidenced above. + "EIGEN_MPL2_ONLY", + "EIGEN_MAX_ALIGN_BYTES=64", + ], + includes = ["include/eigen3"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/gif.BUILD b/third_party/systemlibs/gif.BUILD index 5eb2c918ba443fd..61f1befe542a491 100644 --- a/third_party/systemlibs/gif.BUILD +++ b/third_party/systemlibs/gif.BUILD @@ -7,6 +7,7 @@ filegroup( cc_library( name = "gif", - linkopts = ["-lgif"], + linkopts = ["-Lexternal/gif/lib -lgif"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD index 8b703f11556daa2..01a89db0ad8ecea 100644 --- a/third_party/systemlibs/grpc.BUILD +++ b/third_party/systemlibs/grpc.BUILD @@ -7,7 +7,9 @@ filegroup( cc_library( name = "grpc", + includes = ["include"], linkopts = [ + "-Lexternal/com_github_grpc_grpc/lib", "-lgrpc", "-lgpr", ], @@ -16,7 +18,9 @@ cc_library( cc_library( name = "grpc++", + includes = ["include"], linkopts = [ + "-Lexternal/com_github_grpc_grpc/lib", "-lgrpc++", "-lgpr", ], @@ -25,17 +29,21 @@ cc_library( cc_library( name = "grpc++_public_hdrs", + includes = ["include"], visibility = ["//visibility:public"], ) cc_library( name = "grpc++_codegen_proto", + includes = ["include"], visibility = ["//visibility:public"], ) cc_library( name = "grpc_unsecure", + includes = ["include"], linkopts = [ + "-Lexternal/com_github_grpc_grpc/lib", "-lgrpc_unsecure", "-lgpr", ], @@ -44,7 +52,9 @@ cc_library( cc_library( name = "grpc++_unsecure", + includes = ["include"], linkopts = [ + "-Lexternal/com_github_grpc_grpc/lib", "-lgrpc++_unsecure", "-lgpr", ], diff --git a/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl index e427328c39be802..87f99c22601f4b1 100644 --- a/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl +++ b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl @@ -88,7 +88,7 @@ def cc_grpc_library( generate_cc( name = codegen_grpc_target, srcs = proto_targets, - plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", + plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin", well_known_protos = well_known_protos, generate_mocks = generate_mocks, **kwargs diff --git a/third_party/systemlibs/grpc.bazel.protobuf.bzl b/third_party/systemlibs/grpc.bazel.protobuf.bzl index 3eca97dc2311fb1..214ff2cea1a73f6 100644 --- a/third_party/systemlibs/grpc.bazel.protobuf.bzl +++ b/third_party/systemlibs/grpc.bazel.protobuf.bzl @@ -4,6 +4,7 @@ _PROTO_EXTENSION = ".proto" _VIRTUAL_IMPORTS = "/_virtual_imports/" def well_known_proto_libs(): + return [] return [ "@com_google_protobuf//:any_proto", "@com_google_protobuf//:api_proto", diff --git a/third_party/systemlibs/png.BUILD b/third_party/systemlibs/png.BUILD index fc6b6f2d8bb0f87..33ca2968368dc5e 100644 --- a/third_party/systemlibs/png.BUILD +++ b/third_party/systemlibs/png.BUILD @@ -7,6 +7,7 @@ filegroup( cc_library( name = "png", - linkopts = ["-lpng"], + linkopts = ["-Lexternal/png/lib -lpng"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/protobuf.BUILD b/third_party/systemlibs/protobuf.BUILD index 4d05ab28d12e995..e356db64bcf02a8 100644 --- a/third_party/systemlibs/protobuf.BUILD +++ b/third_party/systemlibs/protobuf.BUILD @@ -51,29 +51,55 @@ genrule( name = "link_proto_files", outs = RELATIVE_WELL_KNOWN_PROTOS, cmd = """ + pdir=$$(which protoc | sed 's|/bin/protoc||') for i in $(OUTS); do f=$${i#$(@D)/} mkdir -p $(@D)/$${f%/*} - ln -sf $(PROTOBUF_INCLUDE_PATH)/$$f $(@D)/$$f + ln -sf $$pdir/include/$$f $(@D)/$$f done """, ) cc_library( name = "protobuf", - linkopts = ["-lprotobuf"], + linkopts = ["-Lexternal/com_google_protobuf/lib -lprotobuf"], + includes = ["include"], visibility = ["//visibility:public"], ) cc_library( name = "protobuf_headers", - linkopts = ["-lprotobuf"], + linkopts = ["-Lexternal/com_google_protobuf/lib -lprotobuf"], + includes = ["include"], visibility = ["//visibility:public"], ) cc_library( name = "protoc_lib", - linkopts = ["-lprotoc"], + linkopts = ["-Lexternal/com_google_protobuf/lib -lprotoc"], + includes = ["include"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "protobuf_lite", + linkopts = ["-Lexternal/com_google_protobuf/lib -lprotobuf"], + includes = ["include"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "proto_api", + linkopts = ["-Lexternal/com_google_protobuf/lib -lprotoc"], + includes = ["include"], + visibility = ["//visibility:public"], +) + +proto_lang_toolchain( + name = "cc_toolchain", + blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], + command_line = "--cpp_out=$(OUT)", + runtime = ":protobuf", visibility = ["//visibility:public"], ) @@ -94,7 +120,7 @@ cc_proto_library( proto_gen( name = "protobuf_python_genproto", - includes = ["."], + includes = ["include"], protoc = "@com_google_protobuf//:protoc", visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/pybind11.BUILD b/third_party/systemlibs/pybind11.BUILD index 79a483d7b5d75c3..dce31e0cbee459d 100644 --- a/third_party/systemlibs/pybind11.BUILD +++ b/third_party/systemlibs/pybind11.BUILD @@ -2,7 +2,14 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "pybind11", + includes = ['lib/python3.8/site-packages/pybind11/include'], deps = [ "@org_tensorflow//third_party/python_runtime:headers", ], ) + +# Needed by pybind11_bazel. +config_setting( + name = "osx", + constraint_values = ["@platforms//os:osx"], +) diff --git a/third_party/systemlibs/sqlite.BUILD b/third_party/systemlibs/sqlite.BUILD index 88a84a961376b7c..70eee789fd29651 100644 --- a/third_party/systemlibs/sqlite.BUILD +++ b/third_party/systemlibs/sqlite.BUILD @@ -3,7 +3,8 @@ licenses(["unencumbered"]) # Public Domain # Production build of SQLite library that's baked into TensorFlow. cc_library( name = "org_sqlite", - linkopts = ["-lsqlite3"], + linkopts = ["-Lexternal/org_sqlite/lib -lsqlite3"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl index 822ecda6d699797..3e31f616a1aecbf 100644 --- a/third_party/systemlibs/syslibs_configure.bzl +++ b/third_party/systemlibs/syslibs_configure.bzl @@ -22,6 +22,7 @@ VALID_LIBS = [ "cython", "dill_archive", "double_conversion", + "eigen_archive", "flatbuffers", "functools32_archive", "gast_archive", @@ -33,6 +34,7 @@ VALID_LIBS = [ "nasm", "nsync", "org_sqlite", + "org_python_pypi_backports_weakref", "pasta", "png", "pybind11", diff --git a/third_party/systemlibs/zlib.BUILD b/third_party/systemlibs/zlib.BUILD index 69462ae6cbc2fa7..d65e3838c4dd9b6 100644 --- a/third_party/systemlibs/zlib.BUILD +++ b/third_party/systemlibs/zlib.BUILD @@ -7,6 +7,7 @@ filegroup( cc_library( name = "zlib", - linkopts = ["-lz"], + linkopts = ["-Lexternal/zlib/lib -lz"], + includes = ["include"], visibility = ["//visibility:public"], ) diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl b/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl index 3fe7b2069a1bdf9..5bed6e721cc41e4 100644 --- a/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl +++ b/third_party/xla/third_party/tsl/tsl/platform/default/build_config.bzl @@ -404,7 +404,7 @@ def py_proto_library( if dep != "@com_google_protobuf//:protobuf_python": genproto_deps.append(dep + "_genproto") else: - genproto_deps.append("@com_google_protobuf//:well_known_types_py_pb2_genproto") + pass proto_gen( name = name + "_genproto",