diff --git a/WORKSPACE b/WORKSPACE index 870c7ab0c..e50186399 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,7 +1,7 @@ """Workspace file for lingvo.""" load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//lingvo:repo.bzl", "cc_tf_configure", "icu", "lingvo_protoc_deps", "lingvo_testonly_deps") +load("//lingvo:repo.bzl", "cc_tf_configure", "icu", "lingvo_testonly_deps") http_archive( name = "org_tensorflow", @@ -14,10 +14,45 @@ load("@org_tensorflow//tensorflow/tools/toolchains/remote_config:configs.bzl", " initialize_rbe_configs() +http_archive( + name = "rules_python", + sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036", + strip_prefix = "rules_python-0.25.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +http_archive( + name = "com_google_protobuf", + sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", + strip_prefix = "protobuf-21.7", + urls = [ + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + ], +) +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + +http_archive( + name = "rules_proto", + sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", + strip_prefix = "rules_proto-5.3.0-21.7", + urls = [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", + ], +) +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + + cc_tf_configure() lingvo_testonly_deps() -lingvo_protoc_deps() - icu() diff --git a/lingvo/BUILD b/lingvo/BUILD index f0cd49f65..83d56d6ae 100644 --- a/lingvo/BUILD +++ b/lingvo/BUILD @@ -5,7 +5,8 @@ load( # Placeholder: load py_library # Placeholder: load py_test -load("//lingvo:lingvo.bzl", "pytype_library", "pytype_strict_library") +load("//lingvo:lingvo.bzl", "pytype_library", "pytype_strict_library", + "WELL_KNOWN_PROTO_LIBS") package(default_visibility = ["//visibility:public"]) @@ -327,10 +328,38 @@ lingvo_py_binary( ], ) +TF_PROTOS = [ + "tensorflow/core/framework/tensor.proto", + "tensorflow/core/framework/versions.proto", + "tensorflow/core/framework/op_def.proto", + "tensorflow/core/framework/resource_handle.proto", + "tensorflow/core/framework/function.proto", + "tensorflow/core/framework/graph_debug_info.proto", + "tensorflow/core/framework/node_def.proto", + "tensorflow/core/framework/graph.proto", + "tensorflow/core/framework/attr_value.proto", + "tensorflow/core/framework/variable.proto", + "tensorflow/core/framework/full_type.proto", + "tensorflow/core/framework/tensor_shape.proto", + "tensorflow/core/framework/types.proto", + "tensorflow/core/protobuf/trackable_object_graph.proto", + "tensorflow/core/protobuf/saver.proto", + "tensorflow/core/protobuf/struct.proto", + "tensorflow/core/protobuf/saved_object_graph.proto", + "tensorflow/core/protobuf/meta_graph.proto", +] + +proto_library( + name = "tf_protos", + srcs = TF_PROTOS, + strip_import_prefix = "/lingvo", + deps = WELL_KNOWN_PROTO_LIBS, +) + genrule( name = "tf_dot_protos", srcs = [], - outs = ["tf_protos.tar"], + outs = TF_PROTOS, cmd = "$(location //lingvo/tools:" + "generate_tf_dot_protos) $(location " + diff --git a/lingvo/lingvo.bzl b/lingvo/lingvo.bzl index 251e25f88..2f502c294 100644 --- a/lingvo/lingvo.bzl +++ b/lingvo/lingvo.bzl @@ -1,5 +1,7 @@ """Implements custom rules for Lingvo.""" +load("@rules_python//python:proto.bzl", "py_proto_library") + def tf_copts(): # "-Wno-sign-compare", "-mavx" removed for compat with aarch64 return ["-std=c++17"] + select({ @@ -107,65 +109,52 @@ def lingvo_cuda_py_test(name, tags = [], deps = [], **kwargs): **kwargs ) -def _proto_gen_cc_src(name, basename): - native.genrule( - name = name, - srcs = [basename + ".proto"], - outs = [basename + ".pb.cc", basename + ".pb.h"], - tools = [ - "@protobuf_protoc//:protoc_bin", - "//lingvo:tf_dot_protos", - ], - # TODO(drpng): only unpack if tf_proto dependency is requested. - cmd = """ - mkdir -p $(@D)/tf_proto.$$$$; - tar -C $(@D)/tf_proto.$$$$ -xf $(location //lingvo:tf_dot_protos); - $(location @protobuf_protoc//:protoc_bin) --proto_path=$(@D)/tf_proto.$$$$ --proto_path=. --cpp_out=$(GENDIR) $(<); - rm -rf $(@D)/tf_proto.$$$$ - """, - ) - -def _proto_gen_py_src(name, basename): - native.genrule( - name = name, - srcs = [basename + ".proto"], - outs = [basename + "_pb2.py"], - tools = [ - "@protobuf_protoc//:protoc_bin", - "//lingvo:tf_dot_protos", - ], - # TODO(drpng): only unpack if tf_proto dependency is requested. - cmd = """ - mkdir -p $(@D)/tf_proto.$$$$; - tar -C $(@D)/tf_proto.$$$$ -xf $(location //lingvo:tf_dot_protos); - $(location @protobuf_protoc//:protoc_bin) --proto_path=$(@D)/tf_proto.$$$$ --proto_path=. --python_out=$(GENDIR) $(<); - rm -rf $(@D)/tf_proto.$$$$ - """, - ) +WELL_KNOWN_PROTO_LIBS = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", +] def lingvo_proto_cc(name, src, deps = []): # TODO(drpng): only works with proto with no deps within lingvo. _unused = [deps] basename = src.replace(".proto", "") - _proto_gen_cc_src(name + "_gencc", basename) - lingvo_cc_library( + native.proto_library( name = name, - srcs = [basename + ".pb.cc"], - hdrs = [basename + ".pb.h"], + srcs = [src], + deps = [ + "//lingvo:tf_protos", + ] + WELL_KNOWN_PROTO_LIBS, ) - lingvo_cc_library( - name = "%s_cc" % name, - deps = [":%s" % name], + native.cc_proto_library( + name = name + "_cc", + deps = [":" + name] ) + def lingvo_proto_py(name, src, deps = []): # TODO(drpng): only works with proto with no deps within lingvo. _unused = [deps] basename = src.replace(".proto", "") - _proto_gen_py_src(name + "_genpy", basename) - native.py_library( + native.proto_library( + name = name + "_pyproto", + srcs = [src], + deps = [ + "//lingvo:tf_protos", + ] + WELL_KNOWN_PROTO_LIBS, + ) + py_proto_library( name = name, - srcs = [basename + "_pb2.py"], + deps = [name + "_pyproto"], ) # Placeholders to use until bazel supports pytype_{,strict_}{library,test,binary}. diff --git a/lingvo/repo.bzl b/lingvo/repo.bzl index ce65822d2..672be1dc8 100644 --- a/lingvo/repo.bzl +++ b/lingvo/repo.bzl @@ -221,22 +221,6 @@ cc_library( strip_prefix = "googletest-release-1.10.0", ) -def lingvo_protoc_deps(): - http_archive( - name = "protobuf_protoc", - build_file_content = """ -filegroup( - name = "protoc_bin", - srcs = ["bin/protoc"], - visibility = ["//visibility:public"], -) -""", - urls = [ - "https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip", - ], - sha256 = "3cd951aff8ce713b94cde55e12378f505f2b89d47bf080508cf77e3934f680b6", - ) - def icu(): third_party_http_archive( name = "icu", diff --git a/lingvo/tools/generate_tf_dot_protos.sh b/lingvo/tools/generate_tf_dot_protos.sh index 333154b28..ecf8925ba 100755 --- a/lingvo/tools/generate_tf_dot_protos.sh +++ b/lingvo/tools/generate_tf_dot_protos.sh @@ -31,7 +31,3 @@ mkdir -p ${dest}/tensorflow/core/framework mkdir -p ${dest}/tensorflow/core/protobuf ${binary} ${dest} - -# genrule requires statically determined outputs, so we package all -# into a single file. -tar -C ${dest} -cf ${dest}/tf_protos.tar tensorflow/core/{framework,protobuf}