Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support customed cpp_outs and options for plugins in proto_library #1047

Merged
merged 2 commits into from
Oct 29, 2024
Merged

Support customed cpp_outs and options for plugins in proto_library #1047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aefe0b0
support cpp_outs and plugin_opts in proto_library
Oct 24, 2024
80bab48
improve documentation
Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions doc/en/build_rules/idl.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,26 @@ proto_library(

The `cpp` target code is always generated.

If you want to generate some cpp code with other suffix names, you can use `cpp_outs` attributes.
The default value is `['.pb']`. For example, if you want to use grpc plugin to generate `rpc_meta_info.pb.cc/h` and `rpc_meta_info.grpc.pb.cc/h`, set `cpp_outs = [".pb', '.grpc.pb']`

Besides, use `plugin_opts` to pass some options to plugins, like `plugin_opts={"plugin_name":["option1", "option2.."]},`.
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two attributes are actually belonged to the plugin. Can they be defined as plugin-level attributes ? By this way, they don't need to be written repeatedly for each proto_library call.

Copy link
Contributor Author

@noanswer noanswer Oct 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cpp_outs and plugin_opts would be differnent in differnent targets. The plugin will fragment the original file into multiple pieces based on the size of the proto generated, which the user will specify in different targets, the default is divided into 3 pieces.

Here is the code we will add in BLADE_ROOT:

if get_env("USE_ADTABLE_PLUGIN_FAST_MODE"):
    def _register_proto_library_wrapper():
        from blade import proto_library_target
        from blade import build_rules
        def proto_library(*args, **kwargs):
            default_opts = ["fast_mode", "sharding_size=3"]
            default_cpp_outs = [".pb", ".core.pb", ".controller.pb", ".axu.pb", ".core.1.pb", ".core.2.pb", ".core.3.pb"]
            default_applied = False
            if "plugins" in kwargs and "adtable_codegen" in kwargs["plugins"]:
                if "plugin_opts" not in kwargs:
                    kwargs["plugin_opts"] = {"adtable_codegen":default_opts}
                    default_applied = True
                elif "adtable_codegen" not in kwargs["plugin_opts"]:
                    kwargs["plugin_opts"]["adtable_codegen"] = default_opts
                    default_applied = True
            if default_applied:
                if "cpp_outs" not in kwargs:
                    kwargs["cpp_outs"] = []
                kwargs["cpp_outs"].extend(default_cpp_outs)
                kwargs["cpp_outs"] = list(set(kwargs["cpp_outs"])) # unique
            return proto_library_target.proto_library(*args, **kwargs)
        build_rules.register_function(proto_library)
    _register_proto_library_wrapper()

Blade will pass `--plugin_name_opt=option1 --plugin_name_opt=option2` to grpc plugin.

```python
proto_library(
name = 'rpc_meta_info_proto',
srcs = 'rpc_meta_info.proto',
deps = ':rpc_option_proto',
target_languages = ['java', 'python'],
plugins = ['grpc'],
cpp_outs = ['.pb', '.grpc.pb'],
plugin_opts = {
'grpc': ["option1", "option2"],
},
)
```

## thrift_library ##

Can be used to generate thrift C++ library
Expand Down
20 changes: 20 additions & 0 deletions doc/zh_CN/build_rules/idl.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,26 @@ proto_library(

C++ 代码总是会生成。

如果你想生成C++的其他后缀名代码，可以使用 `cpp_outs` 参数。
默认值是 `['.pb']`。例如，如果你想使用grpc插件生成 `rpc_meta_info.pb.cc/h` 和 `rpc_meta_info.grpc.pb.cc/h`, 那么设置 `cpp_outs = [".pb', '.grpc.pb']`

另外，使用 `plugin_opts` 传递给插件所需参数, 例如 `plugin_opts={"plugin_name":["option1", "option2.."]},`.
Blade 会传递 `--plugin_name_opt=option1 --plugin_name_opt=option2` 给 grpc 插件.

```python
proto_library(
name = 'rpc_meta_info_proto',
srcs = 'rpc_meta_info.proto',
deps = ':rpc_option_proto',
target_languages = ['java', 'python'],
plugins = ['grpc'],
cpp_outs = ['.pb', '.grpc.pb'],
plugin_opts = {
'grpc': ["option1", "option2"],
},
)
```

## thrift_library ##

用于定义thrift库目标
Expand Down
51 changes: 36 additions & 15 deletions src/blade/proto_library_target.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,12 @@ def __init__(self,
deps.append(dep)
self.code_generation[language]['deps'] = deps

def protoc_plugin_flag(self, out):
return '--plugin=protoc-gen-%s=%s --%s_out=%s' % (
def protoc_plugin_flag(self, out, plugin_opts):
flag_value = '--plugin=protoc-gen-%s=%s --%s_out=%s' % (
self.name, self.path, self.name, out)
for opt in plugin_opts:
flag_value += ' --%s_opt=%s' % (self.name, opt)
return flag_value

def __repr__(self):
# This object is a member of proto target's data, provide a textual repr here to make
Expand All @@ -87,6 +90,8 @@ def __init__(self,
target_languages,
plugins,
source_encoding,
cpp_outs,
plugin_opts,
kwargs):
"""Init method.

Expand Down Expand Up @@ -137,6 +142,8 @@ def __init__(self,
self.attr['exported_deps'] += self._unify_deps(protobuf_java_libs)

self._set_protoc_plugins(plugins)
self.attr['cpp_outs'] = cpp_outs
self.attr['plugin_opts'] = plugin_opts

# Link all the symbols by default
self.attr['link_all_symbols'] = True
Expand All @@ -156,10 +163,10 @@ def __init__(self,
full_cpp_headers = []
cpp_headers = []
for src in self.srcs:
full_source, full_header = self._proto_gen_cpp_files(src)
full_cpp_headers.append(full_header)
source, header = self._proto_gen_cpp_file_names(src)
cpp_headers.append(header)
full_sources, full_headers = self._proto_gen_cpp_files(src)
full_cpp_headers.extend(full_headers)
sources, headers = self._proto_gen_cpp_file_names(src)
cpp_headers.extend(headers)
self.attr['generated_hdrs'] = full_cpp_headers
self._set_hdrs(cpp_headers)

Expand Down Expand Up @@ -219,8 +226,12 @@ def _expand_deps_generation(self):
def _proto_gen_cpp_files(self, src):
"""_proto_gen_cpp_files."""
proto_name = src[:-6]
return (self._target_file_path('%s.pb.cc' % proto_name),
self._target_file_path('%s.pb.h' % proto_name))
sources = []
headers = []
for cpp_out in self.attr['cpp_outs']:
sources.append(self._target_file_path('%s%s.cc' % (proto_name, cpp_out)))
headers.append(self._target_file_path('%s%s.h' % (proto_name, cpp_out)))
return (sources, headers)

def _proto_gen_php_file(self, src):
"""Generate the php file name."""
Expand Down Expand Up @@ -320,10 +331,11 @@ def _protoc_plugin_parameters(self, language):
# For Java, we need to be consistent with native java output directory
# which is the build target directory for insertion points to be effected.
# See `protojava` rules for details
plugin_opts = self.attr["plugin_opts"].get(p.name, [])
if language == 'java':
flag_value = p.protoc_plugin_flag(self._target_dir())
flag_value = p.protoc_plugin_flag(self._target_dir(), plugin_opts)
else:
flag_value = p.protoc_plugin_flag(self.build_dir)
flag_value = p.protoc_plugin_flag(self.build_dir, plugin_opts)
vars[flag_key] = vars[flag_key] + ' ' + flag_value if flag_key in vars else flag_value
return paths, vars

Expand All @@ -345,12 +357,12 @@ def _proto_cpp_rules(self):
implicit_deps.extend(plugin_paths)
cpp_sources = []
for src in self.srcs:
full_source, full_header = self._proto_gen_cpp_files(src)
self.generate_build('proto', [full_source, full_header],
full_sources, full_headers = self._proto_gen_cpp_files(src)
self.generate_build('proto', full_sources + full_headers,
inputs=self._source_file_path(src),
implicit_deps=implicit_deps, variables=vars)
source, header = self._proto_gen_cpp_file_names(src)
cpp_sources.append(source)
sources, headers = self._proto_gen_cpp_file_names(src)
cpp_sources.extend(sources)
objs = self._generated_cc_objects(cpp_sources, generated_headers=self.attr['generated_hdrs'])
self._cc_library(objs)

Expand Down Expand Up @@ -421,7 +433,12 @@ def _proto_rules(self):
def _proto_gen_cpp_file_names(self, source):
"""Return just file names"""
base = source[:-6]
return ['%s.pb.cc' % base, '%s.pb.h' % base]
sources = []
headers = []
for cpp_out in self.attr['cpp_outs']:
sources.append('%s%s.cc' % (base, cpp_out))
headers.append('%s%s.h' % (base, cpp_out))
return [sources, headers]

def generate(self):
"""Generate build code for proto files."""
Expand All @@ -445,6 +462,8 @@ def proto_library(
target_languages=None,
plugins=[],
source_encoding='iso-8859-1',
cpp_outs=[".pb"],
plugin_opts={},
**kwargs):
"""proto_library target.
Args:
Expand All @@ -465,6 +484,8 @@ def proto_library(
target_languages=target_languages,
plugins=plugins,
source_encoding=source_encoding,
cpp_outs=cpp_outs,
plugin_opts=plugin_opts,
kwargs=kwargs)
build_manager.instance.register_target(proto_library_target)

Expand Down
Loading