Skip to content

Commit 18ad14f

Browse files
btkostnerv0idpwn
btkostner
and
v0idpwn
authored
Add protobuf code comments as Elixir module documentation (#352)
* Add protobuf code comments as moduledoc * remove debug docs * Remove grpc dep * Refactor test to avoid dependency on grpc library * Ensure moduledoc is set even if modules don't have comments * Fix warning during code generation with multi-line comments --------- Co-authored-by: v0idpwn <[email protected]>
1 parent 4f8fec0 commit 18ad14f

File tree

20 files changed

+443
-85
lines changed

20 files changed

+443
-85
lines changed

‎lib/protobuf/protoc/context.ex

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ defmodule Protobuf.Protoc.Context do
88

99
### All files scope
1010

11+
# All parsed comments from the source file (mapping from path to comment)
12+
# %{"1.4.2" => "this is a comment", "1.5.2.4.2" => "more comment\ndetails"}
13+
comments: %{},
14+
1115
# Mapping from file name to (mapping from type name to metadata, like elixir type name)
1216
# %{"example.proto" => %{".example.FooMsg" => %{type_name: "Example.FooMsg"}}}
1317
global_type_mapping: %{},
@@ -42,7 +46,11 @@ defmodule Protobuf.Protoc.Context do
4246
include_docs?: false,
4347

4448
# Elixirpb.FileOptions
45-
custom_file_options: %{}
49+
custom_file_options: %{},
50+
51+
# Current comment path. The locations are encoded into with a joining "."
52+
# character. E.g. "4.2.3.0"
53+
current_comment_path: ""
4654

4755
@spec custom_file_options_from_file_desc(t(), Google.Protobuf.FileDescriptorProto.t()) :: t()
4856
def custom_file_options_from_file_desc(ctx, desc)
@@ -68,4 +76,17 @@ defmodule Protobuf.Protoc.Context do
6876
module_prefix: Map.get(custom_file_opts, :module_prefix)
6977
}
7078
end
79+
80+
@doc """
81+
Appends a comment path to the current comment path.
82+
83+
## Examples
84+
85+
iex> append_comment_path(%{current_comment_path: "4"}, "2.4")
86+
%{current_comment_path: "4.2.4"}
87+
88+
"""
89+
def append_comment_path(ctx, path) do
90+
%{ctx | current_comment_path: String.trim(ctx.current_comment_path <> "." <> path, ".")}
91+
end
7192
end

‎lib/protobuf/protoc/generator.ex

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,20 @@ defmodule Protobuf.Protoc.Generator do
3636
ctx =
3737
%Context{
3838
ctx
39-
| syntax: syntax(desc.syntax),
39+
| comments: Protobuf.Protoc.Generator.Comment.parse(desc),
40+
syntax: syntax(desc.syntax),
4041
package: desc.package,
4142
dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name)
4243
}
4344
|> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc)
4445

45-
enum_defmodules = Enum.map(desc.enum_type, &Generator.Enum.generate(ctx, &1))
46+
enum_defmodules =
47+
desc.enum_type
48+
|> Enum.with_index()
49+
|> Enum.map(fn {enum, index} ->
50+
{Context.append_comment_path(ctx, "5.#{index}"), enum}
51+
end)
52+
|> Enum.map(fn {ctx, enum} -> Generator.Enum.generate(ctx, enum) end)
4653

4754
{nested_enum_defmodules, message_defmodules} =
4855
Generator.Message.generate_list(ctx, desc.message_type)
@@ -51,7 +58,14 @@ defmodule Protobuf.Protoc.Generator do
5158

5259
service_defmodules =
5360
if "grpc" in ctx.plugins do
54-
Enum.map(desc.service, &Generator.Service.generate(ctx, &1))
61+
desc.service
62+
|> Enum.with_index()
63+
|> Enum.map(fn {service, index} ->
64+
Generator.Service.generate(
65+
Context.append_comment_path(ctx, "6.#{index}"),
66+
service
67+
)
68+
end)
5569
else
5670
[]
5771
end

‎lib/protobuf/protoc/generator/comment.ex

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
defmodule Protobuf.Protoc.Generator.Comment do
2+
@moduledoc false
3+
4+
alias Protobuf.Protoc.Context
5+
6+
@doc """
7+
Parses comment information from `Google.Protobuf.FileDescriptorProto`
8+
into a map with path keys.
9+
"""
10+
@spec parse(Google.Protobuf.FileDescriptorProto.t()) :: %{optional(String.t()) => String.t()}
11+
def parse(file_descriptor_proto) do
12+
file_descriptor_proto
13+
|> get_locations()
14+
|> Enum.reject(&empty_comment?/1)
15+
|> Map.new(fn location ->
16+
{Enum.join(location.path, "."), format_comment(location)}
17+
end)
18+
end
19+
20+
defp get_locations(%{source_code_info: %{location: value}}) when is_list(value),
21+
do: value
22+
23+
defp get_locations(_value), do: []
24+
25+
defp empty_comment?(%{leading_comments: value}) when not is_nil(value) and value != "",
26+
do: false
27+
28+
defp empty_comment?(%{trailing_comments: value}) when not is_nil(value) and value != "",
29+
do: false
30+
31+
defp empty_comment?(%{leading_detached_comments: value}), do: Enum.empty?(value)
32+
33+
defp format_comment(location) do
34+
[location.leading_comments, location.trailing_comments | location.leading_detached_comments]
35+
|> Enum.reject(&is_nil/1)
36+
|> Enum.map(&String.replace(&1, ~r/^\s*\*/, "", global: true))
37+
|> Enum.join("\n\n")
38+
|> String.replace(~r/\n{3,}/, "\n")
39+
|> String.trim()
40+
end
41+
42+
@doc """
43+
Finds a comment via the context. Returns an empty string if the
44+
comment is not found or if `include_docs?` is set to false.
45+
"""
46+
@spec get(Context.t()) :: String.t()
47+
def get(%{include_docs?: false}), do: ""
48+
49+
def get(%{comments: comments, current_comment_path: path}),
50+
do: get(comments, path)
51+
52+
@doc """
53+
Finds a comment via a map of comments and a path. Returns an
54+
empty string if the comment is not found
55+
"""
56+
@spec get(%{optional(String.t()) => String.t()}, String.t()) :: String.t()
57+
def get(comments, path), do: Map.get(comments, path, "")
58+
end

‎lib/protobuf/protoc/generator/enum.ex

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Enum do
22
@moduledoc false
33

44
alias Protobuf.Protoc.Context
5+
alias Protobuf.Protoc.Generator.Comment
56
alias Protobuf.Protoc.Generator.Util
67

78
require EEx
@@ -34,6 +35,7 @@ defmodule Protobuf.Protoc.Generator.Enum do
3435

3536
content =
3637
enum_template(
38+
comment: Comment.get(ctx),
3739
module: msg_name,
3840
use_options: use_options,
3941
fields: desc.value,

‎lib/protobuf/protoc/generator/extension.ex

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ defmodule Protobuf.Protoc.Generator.Extension do
33

44
alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto, FileDescriptorProto}
55
alias Protobuf.Protoc.Context
6+
alias Protobuf.Protoc.Generator.Comment
67
alias Protobuf.Protoc.Generator.Util
78

89
require EEx
@@ -29,7 +30,13 @@ defmodule Protobuf.Protoc.Generator.Extension do
2930

3031
module_contents =
3132
Util.format(
32-
extension_template(use_options: use_options, module: mod_name, extends: extensions)
33+
extension_template(
34+
comment: Comment.get(ctx),
35+
use_options: use_options,
36+
module: mod_name,
37+
extends: extensions,
38+
module_doc?: ctx.include_docs?
39+
)
3340
)
3441

3542
{mod_name, module_contents}
@@ -75,10 +82,15 @@ defmodule Protobuf.Protoc.Generator.Extension do
7582
end
7683

7784
defp get_extensions_from_messages(%Context{} = ctx, use_options, descs) do
78-
Enum.flat_map(descs, fn %DescriptorProto{} = desc ->
79-
generate_module(ctx, use_options, desc) ++
85+
descs
86+
|> Enum.with_index()
87+
|> Enum.flat_map(fn {desc, index} ->
88+
generate_module(Context.append_comment_path(ctx, "7.#{index}"), use_options, desc) ++
8089
get_extensions_from_messages(
81-
%Context{ctx | namespace: ctx.namespace ++ [Macro.camelize(desc.name)]},
90+
%Context{
91+
Context.append_comment_path(ctx, "6.#{index}")
92+
| namespace: ctx.namespace ++ [Macro.camelize(desc.name)]
93+
},
8294
use_options,
8395
desc.nested_type
8496
)
@@ -96,9 +108,11 @@ defmodule Protobuf.Protoc.Generator.Extension do
96108
module_contents =
97109
Util.format(
98110
extension_template(
111+
comment: Comment.get(ctx),
99112
module: module_name,
100113
use_options: use_options,
101-
extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = ""))
114+
extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = "")),
115+
module_doc?: ctx.include_docs?
102116
)
103117
)
104118

‎lib/protobuf/protoc/generator/message.ex

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ defmodule Protobuf.Protoc.Generator.Message do
44
alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto}
55

66
alias Protobuf.Protoc.Context
7+
alias Protobuf.Protoc.Generator.Comment
78
alias Protobuf.Protoc.Generator.Util
89
alias Protobuf.Protoc.Generator.Enum, as: EnumGenerator
910

@@ -21,7 +22,10 @@ defmodule Protobuf.Protoc.Generator.Message do
2122
messages :: [{mod_name :: String.t(), contents :: String.t()}]}
2223
def generate_list(%Context{} = ctx, descs) when is_list(descs) do
2324
descs
24-
|> Enum.map(fn desc -> generate(ctx, desc) end)
25+
|> Enum.with_index()
26+
|> Enum.map(fn {desc, index} ->
27+
generate(Context.append_comment_path(ctx, "4.#{index}"), desc)
28+
end)
2529
|> Enum.unzip()
2630
end
2731

@@ -46,6 +50,7 @@ defmodule Protobuf.Protoc.Generator.Message do
4650
{msg_name,
4751
Util.format(
4852
message_template(
53+
comment: Comment.get(ctx),
4954
module: msg_name,
5055
use_options: msg_opts_str(ctx, desc.options),
5156
oneofs: desc.oneof_decl,
@@ -61,11 +66,19 @@ defmodule Protobuf.Protoc.Generator.Message do
6166
end
6267

6368
defp gen_nested_msgs(ctx, desc) do
64-
Enum.map(desc.nested_type, fn msg_desc -> generate(ctx, msg_desc) end)
69+
desc.nested_type
70+
|> Enum.with_index()
71+
|> Enum.map(fn {msg_desc, index} ->
72+
generate(Context.append_comment_path(ctx, "3.#{index}"), msg_desc)
73+
end)
6574
end
6675

6776
defp gen_nested_enums(ctx, desc) do
68-
Enum.map(desc.enum_type, fn enum_desc -> EnumGenerator.generate(ctx, enum_desc) end)
77+
desc.enum_type
78+
|> Enum.with_index()
79+
|> Enum.map(fn {enum_desc, index} ->
80+
EnumGenerator.generate(Context.append_comment_path(ctx, "4.#{index}"), enum_desc)
81+
end)
6982
end
7083

7184
defp gen_fields(syntax, fields) do
@@ -103,7 +116,15 @@ defmodule Protobuf.Protoc.Generator.Message do
103116
oneofs = get_real_oneofs(desc.oneof_decl)
104117

105118
nested_maps = nested_maps(ctx, desc)
106-
for field <- desc.field, do: get_field(ctx, field, nested_maps, oneofs)
119+
120+
for {field, index} <- Enum.with_index(desc.field) do
121+
get_field(
122+
Context.append_comment_path(ctx, "2.#{index}"),
123+
field,
124+
nested_maps,
125+
oneofs
126+
)
127+
end
107128
end
108129

109130
# Public and used by extensions.
@@ -137,6 +158,7 @@ defmodule Protobuf.Protoc.Generator.Message do
137158

138159
%{
139160
name: field_desc.name,
161+
comment: Comment.get(ctx),
140162
number: field_desc.number,
141163
label: label_name(field_desc.label),
142164
type: type,

‎lib/protobuf/protoc/generator/service.ex

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Service do
22
@moduledoc false
33

44
alias Protobuf.Protoc.Context
5+
alias Protobuf.Protoc.Generator.Comment
56
alias Protobuf.Protoc.Generator.Util
67

78
require EEx
@@ -31,6 +32,7 @@ defmodule Protobuf.Protoc.Generator.Service do
3132
{mod_name,
3233
Util.format(
3334
service_template(
35+
comment: Comment.get(ctx),
3436
module: mod_name,
3537
service_name: name,
3638
package: ctx.package,

‎lib/protobuf/protoc/generator/util.ex

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,19 @@ defmodule Protobuf.Protoc.Generator.Util do
8686
|> IO.iodata_to_binary()
8787
end
8888

89+
@spec pad_comment(String.t(), non_neg_integer()) :: String.t()
90+
def pad_comment(comment, size) do
91+
padding = String.duplicate(" ", size)
92+
93+
comment
94+
|> String.split("\n")
95+
|> Enum.map(fn line ->
96+
trimmed = String.trim_leading(line, " ")
97+
padding <> trimmed
98+
end)
99+
|> Enum.join("\n")
100+
end
101+
89102
@spec version() :: String.t()
90103
def version do
91104
{:ok, value} = :application.get_key(:protobuf, :vsn)

‎mix.exs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -149,15 +149,15 @@ defmodule Protobuf.Mixfile do
149149
proto_src = path_in_protobuf_source(["src"])
150150

151151
protoc!(
152-
"-I #{proto_src} -I src -I test/protobuf/protoc/proto",
152+
"-I #{proto_src} -I src -I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
153153
"./generated",
154154
["test/protobuf/protoc/proto/extension.proto"]
155155
)
156156

157157
protoc!(
158-
"-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my",
158+
"-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my,include_docs=true",
159159
"./generated",
160-
["test/protobuf/protoc/proto/test.proto"]
160+
["test/protobuf/protoc/proto/test.proto", "test/protobuf/protoc/proto/service.proto"]
161161
)
162162

163163
protoc!(
@@ -168,7 +168,7 @@ defmodule Protobuf.Mixfile do
168168
)
169169

170170
protoc!(
171-
"-I test/protobuf/protoc/proto",
171+
"-I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
172172
"./generated",
173173
["test/protobuf/protoc/proto/no_package.proto"]
174174
)
@@ -194,7 +194,7 @@ defmodule Protobuf.Mixfile do
194194
google/protobuf/test_messages_proto3.proto
195195
)
196196

197-
protoc!("-I \"#{proto_root}\"", "./generated", files)
197+
protoc!("-I \"#{proto_root}\" --elixir_opt=include_docs=true", "./generated", files)
198198
end
199199

200200
defp gen_conformance_protos(_args) do

‎priv/templates/enum.ex.eex

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
defmodule <%= @module %> do
2-
<%= unless @module_doc? do %>
2+
<%= if @module_doc? do %>
3+
<%= if @comment != "" do %>
4+
@moduledoc """
5+
<%= Protobuf.Protoc.Generator.Util.pad_comment(@comment, 2) %>
6+
"""
7+
<% end %>
8+
<% else %>
39
@moduledoc false
410
<% end %>
11+
512
use Protobuf, <%= @use_options %>
613

714
<%= if @descriptor_fun_body do %>

0 commit comments

Comments
 (0)