Weaver (#5898)

Builds off of #5793 Resolve #5668 This migrates the generation of our semconv packages from using the `semconvgen` tooling to the new [`weaver`](https://github.com/open-telemetry/weaver) project. The configuration and templating has been added in a way to generate as close as we can to what the `semconvgen` tooling already generated. There are notable differences: ### Acronym/Initialism Fixes As metioned [here](#5898 (comment)), the evaluated exported output of regenerating the `semconv/v1.27.0` package resulted in the following changes: | `weaver` | `semconvgen` | | --- | --- | | ASPNETCoreDiagnosticsExceptionsDescription | AspnetcoreDiagnosticsExceptionsDescription| | ASPNETCoreDiagnosticsExceptionsName | AspnetcoreDiagnosticsExceptionsName| | ASPNETCoreDiagnosticsExceptionsUnit | AspnetcoreDiagnosticsExceptionsUnit| | ASPNETCoreRateLimitingActiveRequestLeasesDescription | AspnetcoreRateLimitingActiveRequestLeasesDescription| | ASPNETCoreRateLimitingActiveRequestLeasesName | AspnetcoreRateLimitingActiveRequestLeasesName| | ASPNETCoreRateLimitingActiveRequestLeasesUnit | AspnetcoreRateLimitingActiveRequestLeasesUnit| | ASPNETCoreRateLimitingQueuedRequestsDescription | AspnetcoreRateLimitingQueuedRequestsDescription| | ASPNETCoreRateLimitingQueuedRequestsName | AspnetcoreRateLimitingQueuedRequestsName| | ASPNETCoreRateLimitingQueuedRequestsUnit | AspnetcoreRateLimitingQueuedRequestsUnit| | ASPNETCoreRateLimitingRequestLeaseDurationDescription | AspnetcoreRateLimitingRequestLeaseDurationDescription| | ASPNETCoreRateLimitingRequestLeaseDurationName | AspnetcoreRateLimitingRequestLeaseDurationName| | ASPNETCoreRateLimitingRequestLeaseDurationUnit | AspnetcoreRateLimitingRequestLeaseDurationUnit| | ASPNETCoreRateLimitingRequestTimeInQueueDescription | AspnetcoreRateLimitingRequestTimeInQueueDescription| | ASPNETCoreRateLimitingRequestTimeInQueueName | AspnetcoreRateLimitingRequestTimeInQueueName| | ASPNETCoreRateLimitingRequestTimeInQueueUnit | AspnetcoreRateLimitingRequestTimeInQueueUnit| | ASPNETCoreRateLimitingRequestsDescription | AspnetcoreRateLimitingRequestsDescription| | ASPNETCoreRateLimitingRequestsName | AspnetcoreRateLimitingRequestsName| | ASPNETCoreRateLimitingRequestsUnit | AspnetcoreRateLimitingRequestsUnit| | ASPNETCoreRoutingMatchAttemptsDescription | AspnetcoreRoutingMatchAttemptsDescription| | ASPNETCoreRoutingMatchAttemptsName | AspnetcoreRoutingMatchAttemptsName| | ASPNETCoreRoutingMatchAttemptsUnit | AspnetcoreRoutingMatchAttemptsUnit| | DBSystemDb2 | DBSystemDB2| | GenAIClientOperationDurationDescription | GenAiClientOperationDurationDescription| | GenAIClientOperationDurationName | GenAiClientOperationDurationName| | GenAIClientOperationDurationUnit | GenAiClientOperationDurationUnit| | GenAIClientTokenUsageDescription | GenAiClientTokenUsageDescription| | GenAIClientTokenUsageName | GenAiClientTokenUsageName| | GenAIClientTokenUsageUnit | GenAiClientTokenUsageUnit| | GenAIServerRequestDurationDescription | GenAiServerRequestDurationDescription| | GenAIServerRequestDurationName | GenAiServerRequestDurationName| | GenAIServerRequestDurationUnit | GenAiServerRequestDurationUnit| | GenAIServerTimePerOutputTokenDescription | GenAiServerTimePerOutputTokenDescription| | GenAIServerTimePerOutputTokenName | GenAiServerTimePerOutputTokenName| | GenAIServerTimePerOutputTokenUnit | GenAiServerTimePerOutputTokenUnit| | GenAIServerTimeToFirstTokenDescription | GenAiServerTimeToFirstTokenDescription| | GenAIServerTimeToFirstTokenName | GenAiServerTimeToFirstTokenName| | GenAIServerTimeToFirstTokenUnit | GenAiServerTimeToFirstTokenUnit| | GoMemoryGCGoalDescription | GoMemoryGcGoalDescription| | GoMemoryGCGoalName | GoMemoryGcGoalName| | GoMemoryGCGoalUnit | GoMemoryGcGoalUnit| | JVMBufferCountDescription | JvmBufferCountDescription| | JVMBufferCountName | JvmBufferCountName| | JVMBufferCountUnit | JvmBufferCountUnit| | JVMBufferMemoryLimitDescription | JvmBufferMemoryLimitDescription| | JVMBufferMemoryLimitName | JvmBufferMemoryLimitName| | JVMBufferMemoryLimitUnit | JvmBufferMemoryLimitUnit| | JVMBufferMemoryUsageDescription | JvmBufferMemoryUsageDescription| | JVMBufferMemoryUsageName | JvmBufferMemoryUsageName| | JVMBufferMemoryUsageUnit | JvmBufferMemoryUsageUnit| | JVMBufferMemoryUsedDescription | JvmBufferMemoryUsedDescription| | JVMBufferMemoryUsedName | JvmBufferMemoryUsedName| | JVMBufferMemoryUsedUnit | JvmBufferMemoryUsedUnit| | JVMCPUCountDescription | JvmCPUCountDescription| | JVMCPUCountName | JvmCPUCountName| | JVMCPUCountUnit | JvmCPUCountUnit| | JVMCPURecentUtilizationDescription | JvmCPURecentUtilizationDescription| | JVMCPURecentUtilizationName | JvmCPURecentUtilizationName| | JVMCPURecentUtilizationUnit | JvmCPURecentUtilizationUnit| | JVMCPUTimeDescription | JvmCPUTimeDescription| | JVMCPUTimeName | JvmCPUTimeName| | JVMCPUTimeUnit | JvmCPUTimeUnit| | JVMClassCountDescription | JvmClassCountDescription| | JVMClassCountName | JvmClassCountName| | JVMClassCountUnit | JvmClassCountUnit| | JVMClassLoadedDescription | JvmClassLoadedDescription| | JVMClassLoadedName | JvmClassLoadedName| | JVMClassLoadedUnit | JvmClassLoadedUnit| | JVMClassUnloadedDescription | JvmClassUnloadedDescription| | JVMClassUnloadedName | JvmClassUnloadedName| | JVMClassUnloadedUnit | JvmClassUnloadedUnit| | JVMGCDurationDescription | JvmGcDurationDescription| | JVMGCDurationName | JvmGcDurationName| | JVMGCDurationUnit | JvmGcDurationUnit| | JVMMemoryCommittedDescription | JvmMemoryCommittedDescription| | JVMMemoryCommittedName | JvmMemoryCommittedName| | JVMMemoryCommittedUnit | JvmMemoryCommittedUnit| | JVMMemoryInitDescription | JvmMemoryInitDescription| | JVMMemoryInitName | JvmMemoryInitName| | JVMMemoryInitUnit | JvmMemoryInitUnit| | JVMMemoryLimitDescription | JvmMemoryLimitDescription| | JVMMemoryLimitName | JvmMemoryLimitName| | JVMMemoryLimitUnit | JvmMemoryLimitUnit| | JVMMemoryUsedAfterLastGCDescription | JvmMemoryUsedAfterLastGcDescription| | JVMMemoryUsedAfterLastGCName | JvmMemoryUsedAfterLastGcName| | JVMMemoryUsedAfterLastGCUnit | JvmMemoryUsedAfterLastGcUnit| | JVMMemoryUsedDescription | JvmMemoryUsedDescription| | JVMMemoryUsedName | JvmMemoryUsedName| | JVMMemoryUsedUnit | JvmMemoryUsedUnit| | JVMSystemCPULoad1mDescription | JvmSystemCPULoad1mDescription| | JVMSystemCPULoad1mName | JvmSystemCPULoad1mName| | JVMSystemCPULoad1mUnit | JvmSystemCPULoad1mUnit| | JVMSystemCPUUtilizationDescription | JvmSystemCPUUtilizationDescription| | JVMSystemCPUUtilizationName | JvmSystemCPUUtilizationName| | JVMSystemCPUUtilizationUnit | JvmSystemCPUUtilizationUnit| | JVMThreadCountDescription | JvmThreadCountDescription| | JVMThreadCountName | JvmThreadCountName| | JVMThreadCountUnit | JvmThreadCountUnit| | V8JSGCDurationDescription | V8jsGcDurationDescription| | V8JSGCDurationName | V8jsGcDurationName| | V8JSGCDurationUnit | V8jsGcDurationUnit| | V8JSHeapSpaceAvailableSizeDescription | V8jsHeapSpaceAvailableSizeDescription| | V8JSHeapSpaceAvailableSizeName | V8jsHeapSpaceAvailableSizeName| | V8JSHeapSpaceAvailableSizeUnit | V8jsHeapSpaceAvailableSizeUnit| | V8JSHeapSpacePhysicalSizeDescription | V8jsHeapSpacePhysicalSizeDescription| | V8JSHeapSpacePhysicalSizeName | V8jsHeapSpacePhysicalSizeName| | V8JSHeapSpacePhysicalSizeUnit | V8jsHeapSpacePhysicalSizeUnit| | V8JSMemoryHeapLimitDescription | V8jsMemoryHeapLimitDescription| | V8JSMemoryHeapLimitName | V8jsMemoryHeapLimitName| | V8JSMemoryHeapLimitUnit | V8jsMemoryHeapLimitUnit| | V8JSMemoryHeapUsedDescription | V8jsMemoryHeapUsedDescription| | V8JSMemoryHeapUsedName | V8jsMemoryHeapUsedName| | V8JSMemoryHeapUsedUnit | V8jsMemoryHeapUsedUnit| An audit of these changes leads to the conclusion that they are appropriate fixes to things that were mis-named. ### Doc changes Also mentioned [here](#5898 (comment)), there documentation changes that are included. Mostly this is unavoidable based on the new format of the semconv models, and effort has spent ensuring nothing substantive is lost. See the reverted commit 66523cb for the details of how the `semconv/v1.27.0` changes. ## Next Steps This PR has been paired down to migrate tooling. The next steps are to generate `semconv/v1.28.0` with desired changes (i.e. maybe we don't generate `ASPCoreNet` attrs(?)). From there the missing semconv packages will be generated. - #6226 - #6227
open-telemetry · Jan 31, 2025 · 8324155 · 8324155
1 parent a81250c
commit 8324155
Show file tree

Hide file tree

Showing 10 changed files with 342 additions and 208 deletions.
diff --git a/Makefile b/Makefile
@@ -269,13 +269,32 @@ check-clean-work-tree:
 	  exit 1; \
 	fi
 
+# The weaver docker image to use for semconv-generate.
+WEAVER_IMAGE := $(shell awk '$$4=="weaver" {print $$2}' $(DEPENDENCIES_DOCKERFILE))
+
 SEMCONVPKG ?= "semconv/"
 .PHONY: semconv-generate
-semconv-generate: $(SEMCONVGEN) $(SEMCONVKIT)
+semconv-generate: $(SEMCONVKIT)
 	[ "$(TAG)" ] || ( echo "TAG unset: missing opentelemetry semantic-conventions tag"; exit 1 )
-	[ "$(OTEL_SEMCONV_REPO)" ] || ( echo "OTEL_SEMCONV_REPO unset: missing path to opentelemetry semantic-conventions repo"; exit 1 )
-	$(SEMCONVGEN) -i "$(OTEL_SEMCONV_REPO)/model/." --only=attribute_group -p conventionType=trace -f attribute_group.go -z "$(SEMCONVPKG)/capitalizations.txt" -t "$(SEMCONVPKG)/template.j2" -s "$(TAG)"
-	$(SEMCONVGEN) -i "$(OTEL_SEMCONV_REPO)/model/." --only=metric  -f metric.go -t "$(SEMCONVPKG)/metric_template.j2" -s "$(TAG)"
+	[ "$(SEMCONV_REPO)" ] || ( echo "SEMCONV_REPO unset: missing path to opentelemetry semantic-conventions repo"; exit 1 )
+	# Ensure the target directory for source code is available.
+	mkdir -p $(PWD)/$(SEMCONVPKG)/${TAG}
+	# Note: We mount a home directory for downloading/storing the semconv repository.
+	# Weaver will automatically clean the cache when finished, but the directories will remain.
+	mkdir -p ~/.weaver
+	docker run --rm \
+		-u $(DOCKER_USER) \
+		--env HOME=/tmp/weaver \
+		--mount 'type=bind,source=$(SEMCONV_REPO),target=/source,readonly' \
+		--mount 'type=bind,source=$(PWD)/semconv,target=/home/weaver/templates/registry/go,readonly' \
+		--mount 'type=bind,source=$(PWD)/semconv/${TAG},target=/home/weaver/target' \
+		--mount 'type=bind,source=$(HOME)/.weaver,target=/tmp/weaver/.weaver' \
+		$(WEAVER_IMAGE) registry generate \
+		--registry=/source/model \
+		--templates=/home/weaver/templates \
+		--param tag=$(TAG) \
+		go \
+		/home/weaver/target
 	$(SEMCONVKIT) -output "$(SEMCONVPKG)/$(TAG)" -tag "$(TAG)"
 
 .PHONY: gorelease

diff --git a/RELEASING.md b/RELEASING.md
@@ -6,15 +6,13 @@ New versions of the [OpenTelemetry Semantic Conventions] mean new versions of th
 The `semconv-generate` make target is used for this.
 
 1. Checkout a local copy of the [OpenTelemetry Semantic Conventions] to the desired release tag.
-2. Pull the latest `otel/semconvgen` image: `docker pull otel/semconvgen:latest`
-3. Run the `make semconv-generate ...` target from this repository.
+2. Run the `make semconv-generate ...` target from this repository.
 
 For example,
 
 ```sh
 export TAG="v1.21.0" # Change to the release version you are generating.
 export OTEL_SEMCONV_REPO="/absolute/path/to/opentelemetry/semantic-conventions"
-docker pull otel/semconvgen:latest
 make semconv-generate # Uses the exported TAG and OTEL_SEMCONV_REPO.
 ```
 

diff --git a/dependencies.Dockerfile b/dependencies.Dockerfile
@@ -1,2 +1,3 @@
 # This is a renovate-friendly source of Docker images.
 FROM python:3.13.1-slim-bullseye AS python
+FROM otel/weaver:v0.12.0 AS weaver
diff --git a/semconv/attribute_group.go.j2 b/semconv/attribute_group.go.j2
@@ -0,0 +1,14 @@
+{% import 'helpers.j2' as h -%}
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Code generated from semantic convention specification. DO NOT EDIT.
+
+package semconv // import "go.opentelemetry.io/otel/semconv/{{params.tag}}"
+
+import "go.opentelemetry.io/otel/attribute"
+{%- for group in ctx %}
+{{- h.generate_consts(group) }}
+{{- h.generate_funcs(group) }}
+{{- h.generate_vars(group) }}
+{%- endfor %}
diff --git a/semconv/capitalizations.txt b/semconv/capitalizations.txt
diff --git a/semconv/helpers.j2 b/semconv/helpers.j2
@@ -0,0 +1,112 @@
+{%- macro repl(text) -%}
+{#- Copied from semconvgen: https://github.com/open-telemetry/opentelemetry-go-build-tools/blob/3e69152c51c56213b65c0fc6e5954293b522103c/semconvgen/generator.go#L419-L426 -#}
+{{ text | replace("RedisDatabase", "RedisDB") | replace("IPTCP", "TCP") | replace("IPUDP", "UDP") | replace("Lineno", "LineNumber") }}
+{%- endmacro -%}
+
+{%- macro smart_title_case(text) -%}
+{%- for i in range(0, text | length) -%}
+    {%- if i == 0 or text[i-1] in ['.', '_'] -%}
+        {{ text[i] | upper }}
+    {%- elif not text[i] in ['.', '_'] -%}
+        {{ text[i] }}
+    {%- endif -%}
+{%- endfor -%}
+{%- endmacro -%}
+
+{%- macro to_go_name(fqn) -%}
+{{ repl(smart_title_case(fqn | replace(" ", "") | replace("_", ".") | acronym)) }}
+{%- endmacro -%}
+
+{%- macro deprecated_doc(attr) -%}
+{% if attr is deprecated %}Deprecated: {{ attr.deprecated }}{% endif %}
+{%- endmacro -%}
+
+{%- macro notes_doc(attr) -%}
+{% if attr.note %}Note: {{ attr.note }}{% endif %}
+{%- endmacro -%}
+
+{%- macro examples_doc(attr) -%}
+{%- if attr.examples is iterable %}
+Examples: {{ attr.examples | trim("[]") }}
+{%- endif %}
+{%- endmacro -%}
+
+{%- macro it_reps(brief) -%}
+{%- set brief = brief | trim() -%}
+It represents {% if brief[:2] == "A " or brief[:3] == "An " or brief[:4] == "The " -%}
+  {{ brief[0]|lower }}{{ brief[1:] | trim(".") }}.
+{%- else -%}
+  the {{ brief[0]|lower }}{{ brief[1:] | trim(".") }}.
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro keydoc(attr) -%}
+{{ to_go_name(attr.name) }}Key is the attribute Key conforming to the "{{ attr.name }}" semantic conventions. {{ it_reps(attr.brief) }}
+
+{% if attr is enum -%}
+Type: Enum
+{%- else -%}
+Type: {{ attr.type }}
+{%- endif %}
+RequirementLevel: {{ attr.requirement_level | title }}
+Stability: {{ attr.stability | title }}
+{{ examples_doc(attr) }}
+{{ notes_doc(attr) }}
+{{ deprecated_doc(attr) }}
+{%- endmacro -%}
+
+{%- macro generate_consts(group) -%}
+{#- TODO: generate with group docs (i.e group by registry namespace) #}
+
+{{ ["Namespace: " ~ group.root_namespace] | comment(format="go") }}
+const (
+{%- for attribute in group.attributes if not attribute.deprecated %}
+{#- TODO: Handle template attributes. #}
+{%- if not attribute.type is template_type %}
+{{ keydoc(attribute) | comment(format="go_1tab") }}
+	{{to_go_name(attribute.name)}}Key = attribute.Key("{{attribute.name}}")
+{% endif -%}
+{%- endfor -%}
+)
+{%- endmacro -%}
+
+{%- macro generate_funcs(group) -%}
+{%- for attribute in group.attributes if not attribute is enum %}
+{#- TODO: Handle template attributes. #}
+{%- if not attribute.type is template_type %}
+
+{{ [to_go_name(attribute.name) ~ " returns an attribute KeyValue conforming to the \"" ~ attribute.name ~ "\" semantic conventions. " ~ it_reps(attribute.brief) ] | comment(format="go") }}
+func {{to_go_name(attribute.name)}}(val {{attribute.type | instantiated_type | map_text("attribute_type_value")}}) attribute.KeyValue {
+	return {{to_go_name(attribute.name)}}Key.{{attribute.type | instantiated_type | map_text("attribute_type_method")}}(val)
+}
+{%- endif %}
+{%- endfor %}
+{%- endmacro -%}
+
+{%- macro generate_vars(group) -%}
+{#- Render values for enums #}
+{%- for attribute in group.attributes %}
+{%- if attribute is enum %}
+
+{{ ["Enum values for " ~ attribute.name] | comment(format="go") }}
+var (
+{%- for value in attribute.type.members %}
+{%- if value.deprecated %}
+{{ ["Deprecated: " ~ value.deprecated | trim(".") ~ "." ] | comment(format="go_1tab") }}
+{%- else %}
+{{ [value.brief or value.id, "Stability: " ~ value.stability] | comment(format="go_1tab") }}
+{%- endif %}
+	{{to_go_name(attribute.name ~ "." ~ value.id)}} = {{ to_go_name(attribute.name) }}Key.{{attribute.type | instantiated_type | map_text("attribute_type_method")}}({{ value.value | print_member_value }})
+{%- endfor %}
+)
+{%- endif %}
+{%- endfor %}
+{%- endmacro -%}
+
+{%- macro metric_keydoc(metric) -%}
+{%- if not metric.brief -%}
+{{ to_go_name(metric.metric_name) }} is the metric conforming to the "{{ metric.metric_name}}" semantic conventions.
+{%- else -%}
+{{ to_go_name(metric.metric_name) }} is the metric conforming to the "{{ metric.metric_name}}" semantic conventions. {{ it_reps(metric.brief)|trim(".") }}.
+{%- endif %}
+{%- endmacro -%}
diff --git a/semconv/metric.go.j2 b/semconv/metric.go.j2
@@ -0,0 +1,27 @@
+{% import 'helpers.j2' as h -%}
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Code generated from semantic convention specification. DO NOT EDIT.
+
+package semconv // import "go.opentelemetry.io/otel/semconv/{{params.tag}}"
+
+const (
+{%- for metric in ctx %}
+  {{ h.metric_keydoc(metric) | comment(indent=2) }}
+  // Instrument: {{ metric.instrument }}
+  // Unit: {{ metric.unit }}
+  // Stability: {{ metric.stability }}
+{%- if metric is deprecated %}
+  // Deprecated: {{ metric.deprecated }}
+{%- endif %}
+{%- if not metric.brief %}
+  // NOTE: The description (brief) for this metric is not defined in the semantic-conventions repository.
+{%- endif %}
+  {{ h.to_go_name(metric.metric_name) }}Name = "{{ metric.metric_name }}"
+  {{ h.to_go_name(metric.metric_name) }}Unit = "{{ metric.unit }}"
+{%- if metric.brief %}
+  {{ h.to_go_name(metric.metric_name) }}Description = "{{ metric.brief | trim }}"
+{%- endif %}
+{%- endfor %}
+)
diff --git a/semconv/metric_template.j2 b/semconv/metric_template.j2