Automate schema-next generation in release workflow

Author: Liudmila Molkova

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,4 +11,4 @@ Note: if the PR is touching an area that is not listed in the [existing areas](h
 * [ ] [CONTRIBUTING.md](https://github.com/open-telemetry/semantic-conventions/blob/main/CONTRIBUTING.md) guidelines followed.
 * [ ] Change log entry added, according to the guidelines in [When to add a changelog entry](https://github.com/open-telemetry/semantic-conventions/blob/main/CONTRIBUTING.md#when-to-add-a-changelog-entry).
   * If your PR does not need a change log, start the PR title with `[chore]`
-* [ ] [schema-next.yaml](https://github.com/open-telemetry/semantic-conventions/blob/main/schema-next.yaml) updated with changes to existing conventions.
+

.github/workflows/prepare-release.yml

@@ -31,12 +31,7 @@ jobs:
           fi
 
           version=${{ inputs.version }}
-          cp schema-next.yaml "schemas/$version"
-
-          sed -i "s/  next:$/  next:\n  $version:/" schema-next.yaml
-
-          sed -i "s/^schema_url: .*/schema_url: https:\/\/opentelemetry.io\/schemas\/$version/" "schemas/$version"
-          sed -i "s/  next:$/  $version:/" "schemas/$version"
+          make generate-schema-next SCHEMA_NEXT_VERSION=$version
           git add "schemas/$version"
 
       - name: Update change log

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
     "editor.rulers": [80],
     "yaml.schemas": {
-        "https://raw.githubusercontent.com/open-telemetry/weaver/v0.15.0/schemas/semconv.schema.json": [
+        "https://raw.githubusercontent.com/open-telemetry/weaver/v0.15.3/schemas/semconv.schema.json": [
             "model/**/*.yaml"
         ]
     },

CONTRIBUTING.md

@@ -18,7 +18,6 @@ requirements and recommendations.
   - [Prerequisites](#prerequisites)
   - [1. Modify the YAML model](#1-modify-the-yaml-model)
     - [Code structure](#code-structure)
-    - [Schema files](#schema-files)
   - [2. Update the markdown files](#2-update-the-markdown-files)
     - [Hugo frontmatter](#hugo-frontmatter)
   - [3. Check new convention](#3-check-new-convention)
@@ -36,6 +35,7 @@ requirements and recommendations.
   - [Update the tables of content](#update-the-tables-of-content)
   - [Markdown link check](#markdown-link-check)
   - [Yamllint check](#yamllint-check)
+- [Schema files](#schema-files)
 - [Merging existing ECS conventions](#merging-existing-ecs-conventions)
 
 <!-- tocstop -->
@@ -57,8 +57,7 @@ key, but non-obvious, aspects:
 - All descriptions, normative language are defined in the `docs/` directory.
 - All changes to existing attributes, metrics, etc. MUST be allowed as
   per our [stability guarantees][stability guarantees] and
-  defined in a schema file. As part of any contribution, you should
-  include attribute changes defined in the `schema-next.yaml` file.
+  defined in a schema file.
 - Links to the specification repository MUST point to a tag and **not** to the `main` branch.
   The tag version MUST match with the one defined in [README](README.md).
 
@@ -164,21 +163,6 @@ are defined in `/model/aws/lambda-spans.yaml` and `/model/aws/sdk-spans.yaml` fi
 Deprecated conventions should be placed under `/model/{root-namespace}/deprecated`
 folder.
 
-#### Schema files
-
-When making changes to existing semantic conventions (attributes, metrics, etc)
-you MUST also update the `schema-next.yaml` file with the changes.
-
-For details, please read
-[the schema specification](https://opentelemetry.io/docs/specs/otel/schemas/).
-
-You can also take examples from past changes inside the `schemas` folder.
-
-> [!WARNING]
->
-> DO NOT add your changes to files inside the `schemas` folder. Always add your
-> changes to the `schema-next.yaml` file.
-
 ### 2. Update the markdown files
 
 After updating the YAML file(s), you need to update
@@ -434,6 +418,23 @@ If it is the first time to run this command, install `yamllint` first:
 make install-yamllint
 ```
 
+## Schema files
+
+> [!WARNING]
+>
+> DO NOT add your changes to files inside the `schemas` folder. These files are
+> generated automatically by the release scripts and can't be updated after
+> the corresponding version is released.
+
+Release script uses the following command to generate new schema file:
+
+```bash
+make generate-schema-next NEXT_SEMCONV_VERSION={next version}
+```
+
+For details, please read
+[the schema specification](https://opentelemetry.io/docs/specs/otel/schemas/).
+
 ## Merging existing ECS conventions
 
 The Elastic Common Schema (ECS) is being merged into OpenTelemetry Semantic

Makefile

@@ -348,4 +348,22 @@ check-dead-yaml:
 		--config=/home/weaver/templates/registry/signal-groups-weaver.yaml \
 		. \
 		/home/weaver/target
-	$(TOOLS_DIR)/scripts/find-dead-yaml.sh $(PWD)/internal/tools/bin/signal-groups.txt $(PWD)/docs
+	$(TOOLS_DIR)/scripts/find-dead-yaml.sh $(PWD)/internal/tools/bin/signal-groups.txt $(PWD)/docs
+
+NEXT_SEMCONV_VERSION ?= next
+.PHONY: generate-schema-next
+generate-schema-next:
+	mkdir -p $(TOOLS_DIR)/bin
+	$(DOCKER_RUN) --rm \
+	$(DOCKER_USER_IS_HOST_USER_ARG) \
+	--mount 'type=bind,source=$(PWD)/internal/tools/scripts,target=/home/weaver/templates,readonly' \
+	--mount 'type=bind,source=$(PWD)/model,target=/home/weaver/source,readonly' \
+	--mount 'type=bind,source=$(TOOLS_DIR)/bin,target=/home/weaver/target' \
+	$(WEAVER_CONTAINER) registry diff \
+		--registry=/home/weaver/source \
+		--baseline-registry=https://github.com/open-telemetry/semantic-conventions/archive/refs/tags/v$(LATEST_RELEASED_SEMCONV_VERSION).zip[model] \
+		--diff-format yaml \
+		--diff-template /home/weaver/templates/schema-diff \
+		--output /home/weaver/target
+		# --param next_version=$(NEXT_SEMCONV_VERSION)		
+	$(TOOLS_DIR)/scripts/generate-schema-next.sh $(NEXT_SEMCONV_VERSION) $(LATEST_RELEASED_SEMCONV_VERSION) $(TOOLS_DIR)/bin/schema-diff.yaml

dependencies.Dockerfile

@@ -3,7 +3,7 @@
 # Dependabot can keep this file up to date with latest containers.
 
 # Weaver is used to generate markdown docs, and enforce policies on the model.
-FROM otel/weaver:v0.15.2@sha256:b13acea09f721774daba36344861f689ac4bb8d6ecd94c4600b4d590c8fb34b9 AS weaver
+FROM otel/weaver:v0.15.3@sha256:a84032d6eb95b81972d19de61f6ddc394a26976c1c1697cf9318bef4b4106976 AS weaver
 
 # OPA is used to test policies enforced by weaver.
 FROM openpolicyagent/opa:1.5.1@sha256:7d30d984125161b7f30599c6bdf80a6f2301dbbd526725714c231aad8179e4b9 AS opa

internal/tools/scripts/generate-schema-next.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Generate header and `{next_version}:` section in next schema yaml based on weaver
+# diff output.
+
+set -euo pipefail
+
+cur_version="$1"
+prev_version="$2"
+diff_file="$3"
+cur_version_schema_file="schemas/$cur_version"
+prev_version_schema_file="schemas/$prev_version"
+
+# Check if the previous version schema file exists
+if [[ ! -f "$prev_version_schema_file" ]]; then
+  echo "Previous version schema file not found: $prev_version_schema_file"
+  exit 1
+fi
+
+# check if the current version schema file exists
+if [[ -f "$cur_version_schema_file" ]]; then
+  echo "Current version schema file already exists: $cur_version_schema_file"
+  exit 1
+fi
+
+if ! grep -q "^versions:$" "$prev_version_schema_file"; then
+  echo "String 'versions:' not found in the file $prev_version_schema_file."
+  exit 1
+fi
+
+prev_header_lines=$(awk '/^versions:/{print NR; exit}' "$prev_version_schema_file")
+
+# need to replace the next_version_placeholder in the header file due to https://github.com/open-telemetry/weaver/issues/775
+# TODO: remove it after the issue is fixed
+{
+  sed "s/next_version_placeholder/$cur_version/g" "$diff_file"
+  tail -n +"$((prev_header_lines + 1))" "$prev_version_schema_file"
+} > "$cur_version_schema_file"

internal/tools/scripts/schema-diff/yaml/schema-diff.j2

@@ -0,0 +1,23 @@
+{{- template.set_file_name("schema-diff.yaml") }}
+
+file_format: 1.1.0
+schema_url: https://opentelemetry.io/schemas/{{ctx.next_version}}
+versions:
+  {{ctx.next_version}}:
+{% if ctx.renames.registry_attributes | length > 0 %}
+    all:
+      changes:
+        - rename_attributes:
+            attribute_map:
+{% for attr in ctx.renames.registry_attributes %}
+              {{attr.old_name}}: {{attr.new_name}}
+{% endfor %}
+{% endif %}
+{% if ctx.renames.metrics | length > 0 %}
+    metrics:
+      changes:
+        - rename_metrics:
+{% for metric in ctx.renames.metrics %}
+            {{metric.old_name}}: {{metric.new_name}}
+{% endfor %}
+{% endif %}

internal/tools/scripts/schema-diff/yaml/weaver.yaml

@@ -0,0 +1,16 @@
+params:
+  next_version: "next_version_placeholder" # https://github.com/open-telemetry/weaver/issues/775
+templates:
+  - pattern: schema-diff.j2
+    filter: >
+      .
+      | {
+          renames: .changes | with_entries(.value |= (map(select(.type == "renamed")) | sort_by(.old_name))),
+          next_version: $next_version,
+          latest_released_version: .baseline.semconv_version,
+        }
+    application_mode: single
+whitespace_control:
+  trim_blocks: true
+  lstrip_blocks: true
+  keep_trailing_newline: true