From a2ae14f3429869e4892d165516a9743b4daf7b72 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Wed, 24 Jul 2024 09:40:39 -0700 Subject: [PATCH 01/11] Update version to 0.6.0.dev (#36) --- anaconda/meta.yaml | 2 +- src/snowflake/telemetry/version.py | 2 +- tests/snowflake-telemetry-test-utils/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/anaconda/meta.yaml b/anaconda/meta.yaml index 8bfdb31..47d12c8 100644 --- a/anaconda/meta.yaml +++ b/anaconda/meta.yaml @@ -1,6 +1,6 @@ package: name: snowflake_telemetry_python - version: "0.5.0" + version: "0.6.0.dev" source: path: {{ environ.get('SNOWFLAKE_TELEMETRY_DIR') }} diff --git a/src/snowflake/telemetry/version.py b/src/snowflake/telemetry/version.py index 8eeb137..2c851d2 100644 --- a/src/snowflake/telemetry/version.py +++ b/src/snowflake/telemetry/version.py @@ -4,4 +4,4 @@ # """Update this for the versions.""" -VERSION = "0.5.0" +VERSION = "0.6.0.dev" diff --git a/tests/snowflake-telemetry-test-utils/setup.py b/tests/snowflake-telemetry-test-utils/setup.py index a1a6f65..4f78b56 100644 --- a/tests/snowflake-telemetry-test-utils/setup.py +++ b/tests/snowflake-telemetry-test-utils/setup.py @@ -16,7 +16,7 @@ long_description=LONG_DESCRIPTION, install_requires=[ "pytest >= 7.0.0", - "snowflake-telemetry-python == 0.5.0", + "snowflake-telemetry-python == 0.6.0.dev", ], packages=find_namespace_packages( where='src' From 2736fadc2dd5fe81f6f2d210b37106f25791d6bc Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Mon, 19 Aug 2024 13:52:51 -0700 Subject: [PATCH 02/11] Bump opentelemetry to latest version (#33) --- CHANGELOG.md | 4 +++ anaconda/meta.yaml | 6 ++-- setup.py | 6 ++-- .../exporter/otlp/proto/logs/__init__.py | 8 ++--- tests/test_log_encoder.py | 29 +++++++++------- tests/test_metrics_encoder.py | 6 ++-- tests/test_trace_encoder.py | 33 +++++++++---------- 7 files changed, 49 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1a3c6..81b0510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## Unreleased + +* Upgrade OpenTelemetry Python dependencies to version 1.26.0 + ## 0.5.0 (2024-07-23) * Set empty resource for Python OpenTelemetry config. diff --git a/anaconda/meta.yaml b/anaconda/meta.yaml index 47d12c8..cd27e14 100644 --- a/anaconda/meta.yaml +++ b/anaconda/meta.yaml @@ -11,9 +11,9 @@ requirements: - setuptools >=40.0.0 run: - python - - opentelemetry-api ==1.23.0 - - opentelemetry-exporter-otlp-proto-common ==1.23.0 - - opentelemetry-sdk ==1.23.0 + - opentelemetry-api ==1.26.0 + - opentelemetry-exporter-otlp-proto-common ==1.26.0 + - opentelemetry-sdk ==1.26.0 about: home: https://www.snowflake.com/ diff --git a/setup.py b/setup.py index 92c3682..06b813c 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,9 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, install_requires=[ - "opentelemetry-api == 1.23.0", - "opentelemetry-exporter-otlp-proto-common == 1.23.0", - "opentelemetry-sdk == 1.23.0", + "opentelemetry-api == 1.26.0", + "opentelemetry-exporter-otlp-proto-common == 1.26.0", + "opentelemetry-sdk == 1.26.0", ], packages=find_namespace_packages( where='src' diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py index a08f74c..3214e3e 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py @@ -183,10 +183,10 @@ class _SnowflakeTelemetryLoggerProvider(_logs.LoggerProvider): """ def get_logger( - self, - name: str, - version: types.Optional[str] = None, - schema_url: types.Optional[str] = None, + self, name: str, + version: types.Optional[str] = None, + schema_url: types.Optional[str] = None, + attributes: types.Optional[types.Attributes] = None, ) -> _logs.Logger: return _SnowflakeTelemetryLogger( Resource.get_empty(), diff --git a/tests/test_log_encoder.py b/tests/test_log_encoder.py index cf189d6..a4cfb95 100644 --- a/tests/test_log_encoder.py +++ b/tests/test_log_encoder.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest -from typing import List, Tuple +from typing import Sequence, Tuple from opentelemetry._logs import SeverityNumber from opentelemetry.exporter.otlp.proto.common._internal import ( @@ -40,8 +40,7 @@ from opentelemetry.proto.resource.v1.resource_pb2 import ( Resource as PB2Resource, ) -from opentelemetry.sdk._logs import LogData, LogLimits -from opentelemetry.sdk._logs import LogRecord as SDKLogRecord +from opentelemetry.sdk._logs import LogData, LogLimits, LogRecord as SDKLogRecord from opentelemetry.sdk.resources import Resource as SDKResource from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.trace import TraceFlags @@ -82,10 +81,11 @@ def test_dropped_attributes_count(self): ) @staticmethod - def _get_sdk_log_data() -> List[LogData]: + def _get_sdk_log_data() -> Sequence[LogData]: log1 = LogData( log_record=SDKLogRecord( timestamp=1644650195189786880, + observed_timestamp=1644660000000000000, trace_id=89564621134313219400156819398935297684, span_id=1312458408527513268, trace_flags=TraceFlags(0x01), @@ -106,6 +106,7 @@ def _get_sdk_log_data() -> List[LogData]: log2 = LogData( log_record=SDKLogRecord( timestamp=1644650249738562048, + observed_timestamp=1644660000000000000, trace_id=0, span_id=0, trace_flags=TraceFlags.DEFAULT, @@ -123,6 +124,7 @@ def _get_sdk_log_data() -> List[LogData]: log3 = LogData( log_record=SDKLogRecord( timestamp=1644650427658989056, + observed_timestamp=1644660000000000000, trace_id=271615924622795969659406376515024083555, span_id=4242561578944770265, trace_flags=TraceFlags(0x01), @@ -138,6 +140,7 @@ def _get_sdk_log_data() -> List[LogData]: log4 = LogData( log_record=SDKLogRecord( timestamp=1644650584292683008, + observed_timestamp=1644660000000000000, trace_id=212592107417388365804938480559624925555, span_id=6077757853989569223, trace_flags=TraceFlags(0x01), @@ -159,7 +162,7 @@ def _get_sdk_log_data() -> List[LogData]: def get_test_logs( self, - ) -> Tuple[List[SDKLogRecord], ExportLogsServiceRequest]: + ) -> Tuple[Sequence[LogData], ExportLogsServiceRequest]: sdk_logs = self._get_sdk_log_data() pb2_service_request = ExportLogsServiceRequest( @@ -181,13 +184,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650195189786880, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 89564621134313219400156819398935297684 ), span_id=_encode_span_id( 1312458408527513268 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="WARN", severity_number=SeverityNumber.WARN.value, body=_encode_value( @@ -207,13 +211,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650584292683008, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 212592107417388365804938480559624925555 ), span_id=_encode_span_id( 6077757853989569223 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="INFO", severity_number=SeverityNumber.INFO.value, body=_encode_value( @@ -249,9 +254,8 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650249738562048, - trace_id=_encode_trace_id(0), - span_id=_encode_span_id(0), - flags=int(TraceFlags.DEFAULT), + observed_time_unix_nano=1644660000000000000, + flags=int(0x00), severity_text="WARN", severity_number=SeverityNumber.WARN.value, body=_encode_value( @@ -266,13 +270,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650427658989056, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 271615924622795969659406376515024083555 ), span_id=_encode_span_id( 4242561578944770265 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="DEBUG", severity_number=SeverityNumber.DEBUG.value, body=_encode_value("To our galaxy"), @@ -290,7 +295,7 @@ def get_test_logs( return sdk_logs, pb2_service_request @staticmethod - def _get_test_logs_dropped_attributes() -> List[LogData]: + def _get_test_logs_dropped_attributes() -> Sequence[LogData]: log1 = LogData( log_record=SDKLogRecord( timestamp=1644650195189786880, diff --git a/tests/test_metrics_encoder.py b/tests/test_metrics_encoder.py index bb4b1d8..76464a6 100644 --- a/tests/test_metrics_encoder.py +++ b/tests/test_metrics_encoder.py @@ -33,10 +33,8 @@ from opentelemetry.sdk.metrics.export import AggregationTemporality, Buckets from opentelemetry.sdk.metrics.export import ( ExponentialHistogram as ExponentialHistogramType, -) -from opentelemetry.sdk.metrics.export import ExponentialHistogramDataPoint -from opentelemetry.sdk.metrics.export import Histogram as HistogramType -from opentelemetry.sdk.metrics.export import ( + Histogram as HistogramType, + ExponentialHistogramDataPoint, HistogramDataPoint, Metric, MetricsData, diff --git a/tests/test_trace_encoder.py b/tests/test_trace_encoder.py index 0fc767f..322a521 100644 --- a/tests/test_trace_encoder.py +++ b/tests/test_trace_encoder.py @@ -40,10 +40,11 @@ from opentelemetry.proto.trace.v1.trace_pb2 import ( ResourceSpans as PB2ResourceSpans, TracesData as PB2TracesData, + ScopeSpans as PB2ScopeSpans, + Span as PB2Span, + SpanFlags as PB2SpanFlags, + Status as PB2Status, ) -from opentelemetry.proto.trace.v1.trace_pb2 import ScopeSpans as PB2ScopeSpans -from opentelemetry.proto.trace.v1.trace_pb2 import Span as PB2SPan -from opentelemetry.proto.trace.v1.trace_pb2 import Status as PB2Status from opentelemetry.sdk.trace import Event as SDKEvent from opentelemetry.sdk.trace import Resource as SDKResource from opentelemetry.sdk.trace import SpanContext as SDKSpanContext @@ -97,7 +98,7 @@ def get_exhaustive_otel_span_list() -> List[SDKSpan]: ) parent_span_context = SDKSpanContext( - trace_id, 0x1111111111111111, is_remote=False + trace_id, 0x1111111111111111, is_remote=True ) other_context = SDKSpanContext( @@ -185,15 +186,15 @@ def get_exhaustive_test_spans( PB2ScopeSpans( scope=PB2InstrumentationScope(), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[0].context.span_id ), - trace_state=None, parent_span_id=_encode_span_id( otel_spans[0].parent.span_id ), + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK|PB2SpanFlags.SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK, name=otel_spans[0].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -221,7 +222,7 @@ def get_exhaustive_test_spans( ), ], events=[ - PB2SPan.Event( + PB2Span.Event( name="event0", time_unix_nano=otel_spans[0] .events[0] @@ -249,7 +250,7 @@ def get_exhaustive_test_spans( ) ], links=[ - PB2SPan.Link( + PB2Span.Link( trace_id=_encode_trace_id( otel_spans[0] .links[0] @@ -260,6 +261,7 @@ def get_exhaustive_test_spans( .links[0] .context.span_id ), + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, attributes=[ PB2KeyValue( key="key_bool", @@ -283,13 +285,12 @@ def get_exhaustive_test_spans( version="version", ), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[3].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[3].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -320,13 +321,12 @@ def get_exhaustive_test_spans( PB2ScopeSpans( scope=PB2InstrumentationScope(), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[1].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[1].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -338,13 +338,12 @@ def get_exhaustive_test_spans( links=None, status={}, ), - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[2].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[2].name, kind=span_kind, start_time_unix_nano=otel_spans[ From 0fd842e4f2dd48843340b88d0506ae41b8640166 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Tue, 29 Oct 2024 15:56:14 -0700 Subject: [PATCH 03/11] Add code generator for otel proto (#39) --- .github/workflows/check-codegen.yml | 36 ++ README.md | 8 + scripts/plugin.py | 237 +++++++++ scripts/proto_codegen.sh | 64 +++ scripts/templates/template.py.jinja2 | 44 ++ .../proto/collector/logs/v1/logs_service.py | 42 ++ .../collector/metrics/v1/metrics_service.py | 42 ++ .../proto/collector/trace/v1/trace_service.py | 42 ++ .../opentelemetry/proto/common/v1/common.py | 87 ++++ .../opentelemetry/proto/logs/v1/logs.py | 120 +++++ .../opentelemetry/proto/metrics/v1/metrics.py | 341 +++++++++++++ .../proto/resource/v1/resource.py | 24 + .../opentelemetry/proto/trace/v1/trace.py | 181 +++++++ .../telemetry/_internal/serialize/__init__.py | 145 ++++++ tests/snowflake-telemetry-test-utils/setup.py | 5 + tests/test_proto_serialization.py | 472 ++++++++++++++++++ tests/test_protoc_plugin.py | 91 ++++ 17 files changed, 1981 insertions(+) create mode 100644 .github/workflows/check-codegen.yml create mode 100755 scripts/plugin.py create mode 100755 scripts/proto_codegen.sh create mode 100644 scripts/templates/template.py.jinja2 create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py create mode 100644 src/snowflake/telemetry/_internal/serialize/__init__.py create mode 100644 tests/test_proto_serialization.py create mode 100644 tests/test_protoc_plugin.py diff --git a/.github/workflows/check-codegen.yml b/.github/workflows/check-codegen.yml new file mode 100644 index 0000000..17b4c6f --- /dev/null +++ b/.github/workflows/check-codegen.yml @@ -0,0 +1,36 @@ +# This workflow will delete and regenerate the opentelemetry marshaling code using scripts/proto_codegen.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check Codegen + +on: + push: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - ".github/workflows/check-codegen.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - ".github/workflows/check-codegen.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Run codegen script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/proto/ + ./scripts/proto_codegen.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/proto_codegen.sh"; exit 1; } diff --git a/README.md b/README.md index ef6e6ae..ca9d2b3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ pip install --upgrade pip pip install . ``` +## Development + To develop this package, run ```bash @@ -33,3 +35,9 @@ source .venv/bin/activate pip install --upgrade pip pip install . ./tests/snowflake-telemetry-test-utils ``` + +### Code generation + +To regenerate the code under `src/snowflake/_internal/opentelemetry/proto/`, execute the script `./scripts/proto_codegen.sh`. The script expects the `src/snowflake/_internal/opentelemetry/proto/` directory to exist, and will delete all .py files in it before regerating the code. + +The commit/branch/tag of [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-proto) that the code is generated from is pinned to PROTO_REPO_BRANCH_OR_COMMIT, which can be configured in the script. It is currently pinned to the same tag as [opentelemetry-python](https://github.com/open-telemetry/opentelemetry-python/blob/main/scripts/proto_codegen.sh#L15). diff --git a/scripts/plugin.py b/scripts/plugin.py new file mode 100755 index 0000000..588c2fc --- /dev/null +++ b/scripts/plugin.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +import os +import sys +from dataclasses import dataclass, field +from typing import List, Optional +from enum import IntEnum + +from google.protobuf.compiler import plugin_pb2 as plugin +from google.protobuf.descriptor_pb2 import ( + FileDescriptorProto, + FieldDescriptorProto, + EnumDescriptorProto, + EnumValueDescriptorProto, + MethodDescriptorProto, + ServiceDescriptorProto, + DescriptorProto, +) +from jinja2 import Environment, FileSystemLoader +import black +import isort.api + +class WireType(IntEnum): + VARINT = 0 + I64 = 1 + LEN = 2 + I32 = 5 + +@dataclass +class ProtoTypeDescriptor: + name: str + wire_type: WireType + python_type: str + +proto_type_to_descriptor = { + FieldDescriptorProto.TYPE_BOOL: ProtoTypeDescriptor("bool", WireType.VARINT, "bool"), + FieldDescriptorProto.TYPE_ENUM: ProtoTypeDescriptor("enum", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_INT32: ProtoTypeDescriptor("int32", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_INT64: ProtoTypeDescriptor("int64", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_UINT32: ProtoTypeDescriptor("uint32", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_UINT64: ProtoTypeDescriptor("uint64", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_SINT32: ProtoTypeDescriptor("sint32", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_SINT64: ProtoTypeDescriptor("sint64", WireType.VARINT, "int"), + FieldDescriptorProto.TYPE_FIXED32: ProtoTypeDescriptor("fixed32", WireType.I32, "int"), + FieldDescriptorProto.TYPE_FIXED64: ProtoTypeDescriptor("fixed64", WireType.I64, "int"), + FieldDescriptorProto.TYPE_SFIXED32: ProtoTypeDescriptor("sfixed32", WireType.I32, "int"), + FieldDescriptorProto.TYPE_SFIXED64: ProtoTypeDescriptor("sfixed64", WireType.I64, "int"), + FieldDescriptorProto.TYPE_FLOAT: ProtoTypeDescriptor("float", WireType.I32, "float"), + FieldDescriptorProto.TYPE_DOUBLE: ProtoTypeDescriptor("double", WireType.I64, "float"), + FieldDescriptorProto.TYPE_STRING: ProtoTypeDescriptor("string", WireType.LEN, "str"), + FieldDescriptorProto.TYPE_BYTES: ProtoTypeDescriptor("bytes", WireType.LEN, "bytes"), + FieldDescriptorProto.TYPE_MESSAGE: ProtoTypeDescriptor("message", WireType.LEN, "bytes"), +} + +@dataclass +class EnumValueTemplate: + name: str + number: int + + @staticmethod + def from_descriptor(descriptor: EnumValueDescriptorProto) -> "EnumValueTemplate": + return EnumValueTemplate( + name=descriptor.name, + number=descriptor.number, + ) + +@dataclass +class EnumTemplate: + name: str + values: List["EnumValueTemplate"] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: EnumDescriptorProto, parent: str = "") -> "EnumTemplate": + return EnumTemplate( + name=parent + "_" + descriptor.name if parent else descriptor.name, + values=[EnumValueTemplate.from_descriptor(value) for value in descriptor.value], + ) + +def tag_to_repr_varint(tag: int) -> str: + out = bytearray() + while tag >= 128: + out.append((tag & 0x7F) | 0x80) + tag >>= 7 + out.append(tag) + return repr(bytes(out)) + +@dataclass +class FieldTemplate: + name: str + number: int + tag: str + python_type: str + proto_type: str + repeated: bool + group: str + encode_presence: bool + + @staticmethod + def from_descriptor(descriptor: FieldDescriptorProto, group: Optional[str] = None) -> "FieldTemplate": + repeated = descriptor.label == FieldDescriptorProto.LABEL_REPEATED + type_descriptor = proto_type_to_descriptor[descriptor.type] + + python_type = type_descriptor.python_type + proto_type = type_descriptor.name + + if repeated: + python_type = f"List[{python_type}]" + proto_type = f"repeated_{proto_type}" + + tag = (descriptor.number << 3) | type_descriptor.wire_type.value + if repeated and type_descriptor.wire_type != WireType.LEN: + # Special case: repeated primitive fields are packed + # So we need to use the length-delimited wire type + tag = (descriptor.number << 3) | WireType.LEN.value + # Convert the tag to a varint representation + # Saves us from having to calculate the tag at runtime + tag = tag_to_repr_varint(tag) + + # For group / oneof fields, we need to encode the presence of the field + # For message fields, we need to encode the presence of the field if it is not None + encode_presence = group is not None or proto_type == "message" + + return FieldTemplate( + name=descriptor.name, + tag=tag, + number=descriptor.number, + python_type=python_type, + proto_type=proto_type, + repeated=repeated, + group=group, + encode_presence=encode_presence, + ) + +@dataclass +class MessageTemplate: + name: str + fields: List[FieldTemplate] = field(default_factory=list) + enums: List["EnumTemplate"] = field(default_factory=list) + messages: List["MessageTemplate"] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: DescriptorProto, parent: str = "") -> "MessageTemplate": + def get_group(field: FieldDescriptorProto) -> str: + return descriptor.oneof_decl[field.oneof_index].name if field.HasField("oneof_index") else None + fields = [FieldTemplate.from_descriptor(field, get_group(field)) for field in descriptor.field] + fields.sort(key=lambda field: field.number) + + name = parent + "_" + descriptor.name if parent else descriptor.name + return MessageTemplate( + name=name, + fields=fields, + enums=[EnumTemplate.from_descriptor(enum, name) for enum in descriptor.enum_type], + messages=[MessageTemplate.from_descriptor(message, name) for message in descriptor.nested_type], + ) + +@dataclass +class MethodTemplate: + name: str + input_message: MessageTemplate + output_message: MessageTemplate + + @staticmethod + def from_descriptor(descriptor: MethodDescriptorProto) -> "MethodTemplate": + return MethodTemplate( + name=descriptor.name, + input_message=MessageTemplate(name=descriptor.input_type), + output_message=MessageTemplate(name=descriptor.output_type), + ) + +@dataclass +class ServiceTemplate: + name: str + methods: List["MethodTemplate"] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: ServiceDescriptorProto) -> "ServiceTemplate": + return ServiceTemplate( + name=descriptor.name, + methods=[MethodTemplate.from_descriptor(method) for method in descriptor.method], + ) + +@dataclass +class FileTemplate: + messages: List["MessageTemplate"] = field(default_factory=list) + enums: List["EnumTemplate"] = field(default_factory=list) + services: List["ServiceTemplate"] = field(default_factory=list) + name: str = "" + + @staticmethod + def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": + return FileTemplate( + messages=[MessageTemplate.from_descriptor(message) for message in descriptor.message_type], + enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], + services=[ServiceTemplate.from_descriptor(service) for service in descriptor.service], + name=descriptor.name, + ) + +def main(): + request = plugin.CodeGeneratorRequest() + request.ParseFromString(sys.stdin.buffer.read()) + + response = plugin.CodeGeneratorResponse() + # needed since metrics.proto uses proto3 optional fields + response.supported_features = plugin.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + + template_env = Environment(loader=FileSystemLoader(f"{os.path.dirname(os.path.realpath(__file__))}/templates")) + jinja_body_template = template_env.get_template("template.py.jinja2") + + for proto_file in request.proto_file: + file_name = proto_file.name.replace('.proto', '.py') + file_descriptor_proto = proto_file + + file_template = FileTemplate.from_descriptor(file_descriptor_proto) + + code = jinja_body_template.render(file_template=file_template) + code = isort.api.sort_code_string( + code = code, + show_diff=False, + profile="black", + combine_as_imports=True, + lines_after_imports=2, + quiet=True, + force_grid_wrap=2, + ) + code = black.format_str( + src_contents=code, + mode=black.Mode(), + ) + + response_file = response.file.add() + response_file.name = file_name + response_file.content = code + + sys.stdout.buffer.write(response.SerializeToString()) + +if __name__ == '__main__': + main() diff --git a/scripts/proto_codegen.sh b/scripts/proto_codegen.sh new file mode 100755 index 0000000..4d7dffa --- /dev/null +++ b/scripts/proto_codegen.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Regenerate python code from OTLP protos in +# https://github.com/open-telemetry/opentelemetry-proto +# +# To use, update PROTO_REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-proto repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. +# +# Optional envars: +# PROTO_REPO_DIR - the path to an existing checkout of the opentelemetry-proto repo + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +PROTO_REPO_BRANCH_OR_COMMIT="v1.2.0" + +set -e + +PROTO_REPO_DIR=${PROTO_REPO_DIR:-"/tmp/opentelemetry-proto"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" +venv_dir="/tmp/proto_codegen_venv" + +# run on exit even if crash +cleanup() { + echo "Deleting $venv_dir" + rm -rf $venv_dir +} +trap cleanup EXIT + +echo "Creating temporary virtualenv at $venv_dir using $(python3 --version)" +python3 -m venv $venv_dir +source $venv_dir/bin/activate +python -m pip install protobuf Jinja2 grpcio-tools black isort +echo 'python -m grpc_tools.protoc --version' +python -m grpc_tools.protoc --version + +# Clone the proto repo if it doesn't exist +if [ ! -d "$PROTO_REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-proto.git $PROTO_REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $PROTO_REPO_DIR + git fetch --all + git checkout $PROTO_REPO_BRANCH_OR_COMMIT + # pull if PROTO_REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# clean up old generated code +mkdir -p opentelemetry/proto +find opentelemetry/proto/ -regex ".*\.py?" -exec rm {} + + +# generate proto code for all protos +all_protos=$(find $PROTO_REPO_DIR/ -iname "*.proto") +python -m grpc_tools.protoc \ + -I $PROTO_REPO_DIR \ + --plugin=protoc-gen-custom-plugin=$repo_root/scripts/plugin.py \ + --custom-plugin_out=. \ + $all_protos diff --git a/scripts/templates/template.py.jinja2 b/scripts/templates/template.py.jinja2 new file mode 100644 index 0000000..2246245 --- /dev/null +++ b/scripts/templates/template.py.jinja2 @@ -0,0 +1,44 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: {{ file_template.name }} + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) +from typing import List, Optional + +{% for enum in file_template.enums %} +class {{ enum.name }}(Enum): +{%- for value in enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% macro render_message(message) %} +def {{ message.name }}( +{%- for field in message.fields %} + {{ field.name }}: Optional[{{ field.python_type }}] = None, +{%- endfor %} +) -> bytes: + proto_serializer = ProtoSerializer() +{%- for field in message.fields %} + if {{ field.name }}{% if field.encode_presence %} is not None{% endif %}: {% if field.group %}# oneof group {{ field.group }}{% endif %} + proto_serializer.serialize_{{ field.proto_type }}({{ field.tag }}, {{ field.name }}) +{%- endfor %} + return proto_serializer.out + +{% for nested_enum in message.enums %} +class {{ nested_enum.name }}(Enum): +{%- for value in nested_enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% for nested_message in message.messages %} +{{ render_message(nested_message) }} +{% endfor %} +{% endmacro %} + +{% for message in file_template.messages %} +{{ render_message(message) }} +{% endfor %} \ No newline at end of file diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py new file mode 100644 index 0000000..2cfad4a --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py @@ -0,0 +1,42 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/logs/v1/logs_service.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +def ExportLogsServiceRequest( + resource_logs: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_logs: + proto_serializer.serialize_repeated_message(b"\n", resource_logs) + return proto_serializer.out + + +def ExportLogsServiceResponse( + partial_success: Optional[bytes] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if partial_success is not None: + proto_serializer.serialize_message(b"\n", partial_success) + return proto_serializer.out + + +def ExportLogsPartialSuccess( + rejected_log_records: Optional[int] = None, + error_message: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if rejected_log_records: + proto_serializer.serialize_int64(b"\x08", rejected_log_records) + if error_message: + proto_serializer.serialize_string(b"\x12", error_message) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py new file mode 100644 index 0000000..0c31719 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py @@ -0,0 +1,42 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/metrics/v1/metrics_service.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +def ExportMetricsServiceRequest( + resource_metrics: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_metrics: + proto_serializer.serialize_repeated_message(b"\n", resource_metrics) + return proto_serializer.out + + +def ExportMetricsServiceResponse( + partial_success: Optional[bytes] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if partial_success is not None: + proto_serializer.serialize_message(b"\n", partial_success) + return proto_serializer.out + + +def ExportMetricsPartialSuccess( + rejected_data_points: Optional[int] = None, + error_message: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if rejected_data_points: + proto_serializer.serialize_int64(b"\x08", rejected_data_points) + if error_message: + proto_serializer.serialize_string(b"\x12", error_message) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py new file mode 100644 index 0000000..c4e2496 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py @@ -0,0 +1,42 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/trace/v1/trace_service.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +def ExportTraceServiceRequest( + resource_spans: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_spans: + proto_serializer.serialize_repeated_message(b"\n", resource_spans) + return proto_serializer.out + + +def ExportTraceServiceResponse( + partial_success: Optional[bytes] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if partial_success is not None: + proto_serializer.serialize_message(b"\n", partial_success) + return proto_serializer.out + + +def ExportTracePartialSuccess( + rejected_spans: Optional[int] = None, + error_message: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if rejected_spans: + proto_serializer.serialize_int64(b"\x08", rejected_spans) + if error_message: + proto_serializer.serialize_string(b"\x12", error_message) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py new file mode 100644 index 0000000..27f8c02 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py @@ -0,0 +1,87 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/common/v1/common.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +def AnyValue( + string_value: Optional[str] = None, + bool_value: Optional[bool] = None, + int_value: Optional[int] = None, + double_value: Optional[float] = None, + array_value: Optional[bytes] = None, + kvlist_value: Optional[bytes] = None, + bytes_value: Optional[bytes] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if string_value is not None: # oneof group value + proto_serializer.serialize_string(b"\n", string_value) + if bool_value is not None: # oneof group value + proto_serializer.serialize_bool(b"\x10", bool_value) + if int_value is not None: # oneof group value + proto_serializer.serialize_int64(b"\x18", int_value) + if double_value is not None: # oneof group value + proto_serializer.serialize_double(b"!", double_value) + if array_value is not None: # oneof group value + proto_serializer.serialize_message(b"*", array_value) + if kvlist_value is not None: # oneof group value + proto_serializer.serialize_message(b"2", kvlist_value) + if bytes_value is not None: # oneof group value + proto_serializer.serialize_bytes(b":", bytes_value) + return proto_serializer.out + + +def ArrayValue( + values: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if values: + proto_serializer.serialize_repeated_message(b"\n", values) + return proto_serializer.out + + +def KeyValueList( + values: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if values: + proto_serializer.serialize_repeated_message(b"\n", values) + return proto_serializer.out + + +def KeyValue( + key: Optional[str] = None, + value: Optional[bytes] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if key: + proto_serializer.serialize_string(b"\n", key) + if value is not None: + proto_serializer.serialize_message(b"\x12", value) + return proto_serializer.out + + +def InstrumentationScope( + name: Optional[str] = None, + version: Optional[str] = None, + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if name: + proto_serializer.serialize_string(b"\n", name) + if version: + proto_serializer.serialize_string(b"\x12", version) + if attributes: + proto_serializer.serialize_repeated_message(b"\x1a", attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b" ", dropped_attributes_count) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py new file mode 100644 index 0000000..66b0e47 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py @@ -0,0 +1,120 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/logs/v1/logs.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +class SeverityNumber(Enum): + SEVERITY_NUMBER_UNSPECIFIED = 0 + SEVERITY_NUMBER_TRACE = 1 + SEVERITY_NUMBER_TRACE2 = 2 + SEVERITY_NUMBER_TRACE3 = 3 + SEVERITY_NUMBER_TRACE4 = 4 + SEVERITY_NUMBER_DEBUG = 5 + SEVERITY_NUMBER_DEBUG2 = 6 + SEVERITY_NUMBER_DEBUG3 = 7 + SEVERITY_NUMBER_DEBUG4 = 8 + SEVERITY_NUMBER_INFO = 9 + SEVERITY_NUMBER_INFO2 = 10 + SEVERITY_NUMBER_INFO3 = 11 + SEVERITY_NUMBER_INFO4 = 12 + SEVERITY_NUMBER_WARN = 13 + SEVERITY_NUMBER_WARN2 = 14 + SEVERITY_NUMBER_WARN3 = 15 + SEVERITY_NUMBER_WARN4 = 16 + SEVERITY_NUMBER_ERROR = 17 + SEVERITY_NUMBER_ERROR2 = 18 + SEVERITY_NUMBER_ERROR3 = 19 + SEVERITY_NUMBER_ERROR4 = 20 + SEVERITY_NUMBER_FATAL = 21 + SEVERITY_NUMBER_FATAL2 = 22 + SEVERITY_NUMBER_FATAL3 = 23 + SEVERITY_NUMBER_FATAL4 = 24 + + +class LogRecordFlags(Enum): + LOG_RECORD_FLAGS_DO_NOT_USE = 0 + LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 255 + + +def LogsData( + resource_logs: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_logs: + proto_serializer.serialize_repeated_message(b"\n", resource_logs) + return proto_serializer.out + + +def ResourceLogs( + resource: Optional[bytes] = None, + scope_logs: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource is not None: + proto_serializer.serialize_message(b"\n", resource) + if scope_logs: + proto_serializer.serialize_repeated_message(b"\x12", scope_logs) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def ScopeLogs( + scope: Optional[bytes] = None, + log_records: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if scope is not None: + proto_serializer.serialize_message(b"\n", scope) + if log_records: + proto_serializer.serialize_repeated_message(b"\x12", log_records) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def LogRecord( + time_unix_nano: Optional[int] = None, + severity_number: Optional[int] = None, + severity_text: Optional[str] = None, + body: Optional[bytes] = None, + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, + flags: Optional[int] = None, + trace_id: Optional[bytes] = None, + span_id: Optional[bytes] = None, + observed_time_unix_nano: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\t", time_unix_nano) + if severity_number: + proto_serializer.serialize_enum(b"\x10", severity_number) + if severity_text: + proto_serializer.serialize_string(b"\x1a", severity_text) + if body is not None: + proto_serializer.serialize_message(b"*", body) + if attributes: + proto_serializer.serialize_repeated_message(b"2", attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b"8", dropped_attributes_count) + if flags: + proto_serializer.serialize_fixed32(b"E", flags) + if trace_id: + proto_serializer.serialize_bytes(b"J", trace_id) + if span_id: + proto_serializer.serialize_bytes(b"R", span_id) + if observed_time_unix_nano: + proto_serializer.serialize_fixed64(b"Y", observed_time_unix_nano) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py new file mode 100644 index 0000000..d71f1e9 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py @@ -0,0 +1,341 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/metrics/v1/metrics.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +class AggregationTemporality(Enum): + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0 + AGGREGATION_TEMPORALITY_DELTA = 1 + AGGREGATION_TEMPORALITY_CUMULATIVE = 2 + + +class DataPointFlags(Enum): + DATA_POINT_FLAGS_DO_NOT_USE = 0 + DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1 + + +def MetricsData( + resource_metrics: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_metrics: + proto_serializer.serialize_repeated_message(b"\n", resource_metrics) + return proto_serializer.out + + +def ResourceMetrics( + resource: Optional[bytes] = None, + scope_metrics: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource is not None: + proto_serializer.serialize_message(b"\n", resource) + if scope_metrics: + proto_serializer.serialize_repeated_message(b"\x12", scope_metrics) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def ScopeMetrics( + scope: Optional[bytes] = None, + metrics: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if scope is not None: + proto_serializer.serialize_message(b"\n", scope) + if metrics: + proto_serializer.serialize_repeated_message(b"\x12", metrics) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def Metric( + name: Optional[str] = None, + description: Optional[str] = None, + unit: Optional[str] = None, + gauge: Optional[bytes] = None, + sum: Optional[bytes] = None, + histogram: Optional[bytes] = None, + exponential_histogram: Optional[bytes] = None, + summary: Optional[bytes] = None, + metadata: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if name: + proto_serializer.serialize_string(b"\n", name) + if description: + proto_serializer.serialize_string(b"\x12", description) + if unit: + proto_serializer.serialize_string(b"\x1a", unit) + if gauge is not None: # oneof group data + proto_serializer.serialize_message(b"*", gauge) + if sum is not None: # oneof group data + proto_serializer.serialize_message(b":", sum) + if histogram is not None: # oneof group data + proto_serializer.serialize_message(b"J", histogram) + if exponential_histogram is not None: # oneof group data + proto_serializer.serialize_message(b"R", exponential_histogram) + if summary is not None: # oneof group data + proto_serializer.serialize_message(b"Z", summary) + if metadata: + proto_serializer.serialize_repeated_message(b"b", metadata) + return proto_serializer.out + + +def Gauge( + data_points: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if data_points: + proto_serializer.serialize_repeated_message(b"\n", data_points) + return proto_serializer.out + + +def Sum( + data_points: Optional[List[bytes]] = None, + aggregation_temporality: Optional[int] = None, + is_monotonic: Optional[bool] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if data_points: + proto_serializer.serialize_repeated_message(b"\n", data_points) + if aggregation_temporality: + proto_serializer.serialize_enum(b"\x10", aggregation_temporality) + if is_monotonic: + proto_serializer.serialize_bool(b"\x18", is_monotonic) + return proto_serializer.out + + +def Histogram( + data_points: Optional[List[bytes]] = None, + aggregation_temporality: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if data_points: + proto_serializer.serialize_repeated_message(b"\n", data_points) + if aggregation_temporality: + proto_serializer.serialize_enum(b"\x10", aggregation_temporality) + return proto_serializer.out + + +def ExponentialHistogram( + data_points: Optional[List[bytes]] = None, + aggregation_temporality: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if data_points: + proto_serializer.serialize_repeated_message(b"\n", data_points) + if aggregation_temporality: + proto_serializer.serialize_enum(b"\x10", aggregation_temporality) + return proto_serializer.out + + +def Summary( + data_points: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if data_points: + proto_serializer.serialize_repeated_message(b"\n", data_points) + return proto_serializer.out + + +def NumberDataPoint( + start_time_unix_nano: Optional[int] = None, + time_unix_nano: Optional[int] = None, + as_double: Optional[float] = None, + exemplars: Optional[List[bytes]] = None, + as_int: Optional[int] = None, + attributes: Optional[List[bytes]] = None, + flags: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if start_time_unix_nano: + proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) + if as_double is not None: # oneof group value + proto_serializer.serialize_double(b"!", as_double) + if exemplars: + proto_serializer.serialize_repeated_message(b"*", exemplars) + if as_int is not None: # oneof group value + proto_serializer.serialize_sfixed64(b"1", as_int) + if attributes: + proto_serializer.serialize_repeated_message(b":", attributes) + if flags: + proto_serializer.serialize_uint32(b"@", flags) + return proto_serializer.out + + +def HistogramDataPoint( + start_time_unix_nano: Optional[int] = None, + time_unix_nano: Optional[int] = None, + count: Optional[int] = None, + sum: Optional[float] = None, + bucket_counts: Optional[List[int]] = None, + explicit_bounds: Optional[List[float]] = None, + exemplars: Optional[List[bytes]] = None, + attributes: Optional[List[bytes]] = None, + flags: Optional[int] = None, + min: Optional[float] = None, + max: Optional[float] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if start_time_unix_nano: + proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) + if count: + proto_serializer.serialize_fixed64(b"!", count) + if sum is not None: # oneof group _sum + proto_serializer.serialize_double(b")", sum) + if bucket_counts: + proto_serializer.serialize_repeated_fixed64(b"2", bucket_counts) + if explicit_bounds: + proto_serializer.serialize_repeated_double(b":", explicit_bounds) + if exemplars: + proto_serializer.serialize_repeated_message(b"B", exemplars) + if attributes: + proto_serializer.serialize_repeated_message(b"J", attributes) + if flags: + proto_serializer.serialize_uint32(b"P", flags) + if min is not None: # oneof group _min + proto_serializer.serialize_double(b"Y", min) + if max is not None: # oneof group _max + proto_serializer.serialize_double(b"a", max) + return proto_serializer.out + + +def ExponentialHistogramDataPoint( + attributes: Optional[List[bytes]] = None, + start_time_unix_nano: Optional[int] = None, + time_unix_nano: Optional[int] = None, + count: Optional[int] = None, + sum: Optional[float] = None, + scale: Optional[int] = None, + zero_count: Optional[int] = None, + positive: Optional[bytes] = None, + negative: Optional[bytes] = None, + flags: Optional[int] = None, + exemplars: Optional[List[bytes]] = None, + min: Optional[float] = None, + max: Optional[float] = None, + zero_threshold: Optional[float] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if attributes: + proto_serializer.serialize_repeated_message(b"\n", attributes) + if start_time_unix_nano: + proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) + if count: + proto_serializer.serialize_fixed64(b"!", count) + if sum is not None: # oneof group _sum + proto_serializer.serialize_double(b")", sum) + if scale: + proto_serializer.serialize_sint32(b"0", scale) + if zero_count: + proto_serializer.serialize_fixed64(b"9", zero_count) + if positive is not None: + proto_serializer.serialize_message(b"B", positive) + if negative is not None: + proto_serializer.serialize_message(b"J", negative) + if flags: + proto_serializer.serialize_uint32(b"P", flags) + if exemplars: + proto_serializer.serialize_repeated_message(b"Z", exemplars) + if min is not None: # oneof group _min + proto_serializer.serialize_double(b"a", min) + if max is not None: # oneof group _max + proto_serializer.serialize_double(b"i", max) + if zero_threshold: + proto_serializer.serialize_double(b"q", zero_threshold) + return proto_serializer.out + + +def ExponentialHistogramDataPoint_Buckets( + offset: Optional[int] = None, + bucket_counts: Optional[List[int]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if offset: + proto_serializer.serialize_sint32(b"\x08", offset) + if bucket_counts: + proto_serializer.serialize_repeated_uint64(b"\x12", bucket_counts) + return proto_serializer.out + + +def SummaryDataPoint( + start_time_unix_nano: Optional[int] = None, + time_unix_nano: Optional[int] = None, + count: Optional[int] = None, + sum: Optional[float] = None, + quantile_values: Optional[List[bytes]] = None, + attributes: Optional[List[bytes]] = None, + flags: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if start_time_unix_nano: + proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) + if count: + proto_serializer.serialize_fixed64(b"!", count) + if sum: + proto_serializer.serialize_double(b")", sum) + if quantile_values: + proto_serializer.serialize_repeated_message(b"2", quantile_values) + if attributes: + proto_serializer.serialize_repeated_message(b":", attributes) + if flags: + proto_serializer.serialize_uint32(b"@", flags) + return proto_serializer.out + + +def SummaryDataPoint_ValueAtQuantile( + quantile: Optional[float] = None, + value: Optional[float] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if quantile: + proto_serializer.serialize_double(b"\t", quantile) + if value: + proto_serializer.serialize_double(b"\x11", value) + return proto_serializer.out + + +def Exemplar( + time_unix_nano: Optional[int] = None, + as_double: Optional[float] = None, + span_id: Optional[bytes] = None, + trace_id: Optional[bytes] = None, + as_int: Optional[int] = None, + filtered_attributes: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\x11", time_unix_nano) + if as_double is not None: # oneof group value + proto_serializer.serialize_double(b"\x19", as_double) + if span_id: + proto_serializer.serialize_bytes(b'"', span_id) + if trace_id: + proto_serializer.serialize_bytes(b"*", trace_id) + if as_int is not None: # oneof group value + proto_serializer.serialize_sfixed64(b"1", as_int) + if filtered_attributes: + proto_serializer.serialize_repeated_message(b":", filtered_attributes) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py new file mode 100644 index 0000000..683727c --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py @@ -0,0 +1,24 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/resource/v1/resource.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +def Resource( + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if attributes: + proto_serializer.serialize_repeated_message(b"\n", attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b"\x10", dropped_attributes_count) + return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py new file mode 100644 index 0000000..48f2908 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py @@ -0,0 +1,181 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/trace/v1/trace.proto + +from typing import ( + List, + Optional, +) + +from snowflake.telemetry._internal.serialize import ( + Enum, + ProtoSerializer, +) + + +class SpanFlags(Enum): + SPAN_FLAGS_DO_NOT_USE = 0 + SPAN_FLAGS_TRACE_FLAGS_MASK = 255 + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 256 + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 512 + + +def TracesData( + resource_spans: Optional[List[bytes]] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource_spans: + proto_serializer.serialize_repeated_message(b"\n", resource_spans) + return proto_serializer.out + + +def ResourceSpans( + resource: Optional[bytes] = None, + scope_spans: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if resource is not None: + proto_serializer.serialize_message(b"\n", resource) + if scope_spans: + proto_serializer.serialize_repeated_message(b"\x12", scope_spans) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def ScopeSpans( + scope: Optional[bytes] = None, + spans: Optional[List[bytes]] = None, + schema_url: Optional[str] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if scope is not None: + proto_serializer.serialize_message(b"\n", scope) + if spans: + proto_serializer.serialize_repeated_message(b"\x12", spans) + if schema_url: + proto_serializer.serialize_string(b"\x1a", schema_url) + return proto_serializer.out + + +def Span( + trace_id: Optional[bytes] = None, + span_id: Optional[bytes] = None, + trace_state: Optional[str] = None, + parent_span_id: Optional[bytes] = None, + name: Optional[str] = None, + kind: Optional[int] = None, + start_time_unix_nano: Optional[int] = None, + end_time_unix_nano: Optional[int] = None, + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, + events: Optional[List[bytes]] = None, + dropped_events_count: Optional[int] = None, + links: Optional[List[bytes]] = None, + dropped_links_count: Optional[int] = None, + status: Optional[bytes] = None, + flags: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if trace_id: + proto_serializer.serialize_bytes(b"\n", trace_id) + if span_id: + proto_serializer.serialize_bytes(b"\x12", span_id) + if trace_state: + proto_serializer.serialize_string(b"\x1a", trace_state) + if parent_span_id: + proto_serializer.serialize_bytes(b'"', parent_span_id) + if name: + proto_serializer.serialize_string(b"*", name) + if kind: + proto_serializer.serialize_enum(b"0", kind) + if start_time_unix_nano: + proto_serializer.serialize_fixed64(b"9", start_time_unix_nano) + if end_time_unix_nano: + proto_serializer.serialize_fixed64(b"A", end_time_unix_nano) + if attributes: + proto_serializer.serialize_repeated_message(b"J", attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b"P", dropped_attributes_count) + if events: + proto_serializer.serialize_repeated_message(b"Z", events) + if dropped_events_count: + proto_serializer.serialize_uint32(b"`", dropped_events_count) + if links: + proto_serializer.serialize_repeated_message(b"j", links) + if dropped_links_count: + proto_serializer.serialize_uint32(b"p", dropped_links_count) + if status is not None: + proto_serializer.serialize_message(b"z", status) + if flags: + proto_serializer.serialize_fixed32(b"\x85\x01", flags) + return proto_serializer.out + + +class Span_SpanKind(Enum): + SPAN_KIND_UNSPECIFIED = 0 + SPAN_KIND_INTERNAL = 1 + SPAN_KIND_SERVER = 2 + SPAN_KIND_CLIENT = 3 + SPAN_KIND_PRODUCER = 4 + SPAN_KIND_CONSUMER = 5 + + +def Span_Event( + time_unix_nano: Optional[int] = None, + name: Optional[str] = None, + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if time_unix_nano: + proto_serializer.serialize_fixed64(b"\t", time_unix_nano) + if name: + proto_serializer.serialize_string(b"\x12", name) + if attributes: + proto_serializer.serialize_repeated_message(b"\x1a", attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b" ", dropped_attributes_count) + return proto_serializer.out + + +def Span_Link( + trace_id: Optional[bytes] = None, + span_id: Optional[bytes] = None, + trace_state: Optional[str] = None, + attributes: Optional[List[bytes]] = None, + dropped_attributes_count: Optional[int] = None, + flags: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if trace_id: + proto_serializer.serialize_bytes(b"\n", trace_id) + if span_id: + proto_serializer.serialize_bytes(b"\x12", span_id) + if trace_state: + proto_serializer.serialize_string(b"\x1a", trace_state) + if attributes: + proto_serializer.serialize_repeated_message(b'"', attributes) + if dropped_attributes_count: + proto_serializer.serialize_uint32(b"(", dropped_attributes_count) + if flags: + proto_serializer.serialize_fixed32(b"5", flags) + return proto_serializer.out + + +def Status( + message: Optional[str] = None, + code: Optional[int] = None, +) -> bytes: + proto_serializer = ProtoSerializer() + if message: + proto_serializer.serialize_string(b"\x12", message) + if code: + proto_serializer.serialize_enum(b"\x18", code) + return proto_serializer.out + + +class Status_StatusCode(Enum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 diff --git a/src/snowflake/telemetry/_internal/serialize/__init__.py b/src/snowflake/telemetry/_internal/serialize/__init__.py new file mode 100644 index 0000000..9d9063c --- /dev/null +++ b/src/snowflake/telemetry/_internal/serialize/__init__.py @@ -0,0 +1,145 @@ +import struct +from enum import IntEnum +from typing import List, Union + +Enum = IntEnum + +class ProtoSerializer: + __slots__ = ("out") + + def __init__(self) -> None: + self.out = bytearray() + + def __bytes__(self) -> bytes: + return bytes(self.out) + + def serialize_bool(self, tag: bytes, value: bool) -> None: + self.out += tag + self._write_varint_unsigned(1 if value else 0) + + def serialize_enum(self, tag: bytes, value: Union[Enum, int]) -> None: + if not isinstance(value, int): + value = value.value + self.out += tag + self._write_varint_unsigned(value) + + def serialize_uint32(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value) + + def serialize_uint64(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value) + + def serialize_sint32(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value << 1 if value >= 0 else (value << 1) ^ (~0)) + + def serialize_sint64(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value << 1 if value >= 0 else (value << 1) ^ (~0)) + + def serialize_int32(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value + (1 << 32) if value < 0 else value) + + def serialize_int64(self, tag: bytes, value: int) -> None: + self.out += tag + self._write_varint_unsigned(value + (1 << 64) if value < 0 else value) + + def serialize_fixed32(self, tag: bytes, value: int) -> None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self.out += struct.pack(" None: + self.out += tag + self._write_varint_unsigned(len(value)) + self.out += value + + def serialize_string(self, tag: bytes, value: str) -> None: + self.serialize_bytes(tag, value.encode("utf-8")) + + def serialize_message( + self, + tag: bytes, + value: bytes, + ) -> None: + # If value is None, omit message entirely + if value is None: + return + # Otherwise, write the message + # Even if all fields are default (ommnited) + # The presence of the message is still encoded + self.out += tag + self._write_varint_unsigned(len(value)) + self.out += value + + def serialize_repeated_message( + self, + tag: bytes, + values: List[bytes], + ) -> None: + if not values: + return + # local reference to avoid repeated lookups + _self_serialize = self.serialize_message + for value in values: + _self_serialize(tag, value) + + def serialize_repeated_double(self, tag: bytes, values: List[float]) -> None: + if not values: + return + self.out += tag + self._write_varint_unsigned(len(values) * 8) + for value in values: + self.write_double_no_tag(value) + + def serialize_repeated_fixed64(self, tag: bytes, values: List[int]) -> None: + if not values: + return + self.out += tag + self._write_varint_unsigned(len(values) * 8) + for value in values: + self.write_fixed64_no_tag(value) + + def serialize_repeated_uint64(self, tag: bytes, values: List[int]) -> None: + if not values: + return + self.out += tag + tmp = ProtoSerializer() + for value in values: + tmp._write_varint_unsigned(value) + self._write_varint_unsigned(len(tmp.out)) + self.out += tmp.out + + def _write_varint_unsigned(self, value: int) -> None: + while value >= 128: + self.out.append((value & 0x7F) | 0x80) + value >>= 7 + self.out.append(value) + + def write_double_no_tag(self, value: float) -> None: + self.out += struct.pack(" None: + self.out += struct.pack("= 7.0.0", "snowflake-telemetry-python == 0.6.0.dev", + "Jinja2", + "grpcio-tools", + "black", + "isort", + "hypothesis >= 6.0.0", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_proto_serialization.py b/tests/test_proto_serialization.py new file mode 100644 index 0000000..c9f4cc5 --- /dev/null +++ b/tests/test_proto_serialization.py @@ -0,0 +1,472 @@ +import unittest + +from dataclasses import dataclass +from typing import Any, Dict, Callable +import hypothesis +from hypothesis.strategies import composite, text, booleans, integers, floats, lists, binary, sampled_from +from hypothesis.control import assume + +import hypothesis.strategies +import opentelemetry.proto.logs.v1.logs_pb2 as logs_pb2 +import opentelemetry.proto.trace.v1.trace_pb2 as trace_pb2 +import opentelemetry.proto.common.v1.common_pb2 as common_pb2 +import opentelemetry.proto.metrics.v1.metrics_pb2 as metrics_pb2 +import opentelemetry.proto.resource.v1.resource_pb2 as resource_pb2 + +import snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs as logs_sf +import snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace as trace_sf +import snowflake.telemetry._internal.opentelemetry.proto.common.v1.common as common_sf +import snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics as metrics_sf +import snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource as resource_sf + +# Strategy for generating protobuf types +def pb_uint32(): return integers(min_value=0, max_value=2**32-1) +def pb_uint64(): return integers(min_value=0, max_value=2**64-1) +def pb_int32(): return integers(min_value=-2**31, max_value=2**31-1) +def pb_int64(): return integers(min_value=-2**63, max_value=2**63-1) +def pb_sint32(): return integers(min_value=-2**31, max_value=2**31-1) +def pb_sint64(): return integers(min_value=-2**63, max_value=2**63-1) +def pb_float(): return floats(allow_nan=False, allow_infinity=False, width=32) +def pb_double(): return floats(allow_nan=False, allow_infinity=False, width=64) +def draw_pb_double(draw): + # -0.0 is an edge case that is not handled by the snowflake serialization library + # Protobuf serialization will serialize -0.0 as "-0.0", and omit 0.0 + # Snowflake will omit both -0.0 and 0.0 + double = draw(pb_double()) + assume(str(double) != "-0.0") + return double +def pb_fixed64(): return pb_uint64() +def pb_fixed32(): return pb_uint32() +def pb_sfixed64(): return pb_int64() +def pb_sfixed32(): return pb_int32() +def pb_bool(): return booleans() +def pb_string(): return text(max_size=20) +def pb_bytes(): return binary(max_size=20) +def pb_enum(enum): return sampled_from([member.value for member in enum]) +def pb_repeated(type): return lists(type, max_size=3) # limit the size of the repeated field to speed up testing +def pb_span_id(): return binary(min_size=8, max_size=8) +def pb_trace_id(): return binary(min_size=16, max_size=16) + +# Maps protobuf types to their serialization functions, from the protobuf and snowflake serialization libraries +@dataclass +class EncodeStrategy: + pb2: Callable[[Any], Any] + sf: Callable[[Any], Any] + +Resource = EncodeStrategy(pb2=resource_pb2.Resource, sf=resource_sf.Resource) + +InstrumentationScope = EncodeStrategy(pb2=common_pb2.InstrumentationScope, sf=common_sf.InstrumentationScope) +AnyValue = EncodeStrategy(pb2=common_pb2.AnyValue, sf=common_sf.AnyValue) +ArrayValue = EncodeStrategy(pb2=common_pb2.ArrayValue, sf=common_sf.ArrayValue) +KeyValue = EncodeStrategy(pb2=common_pb2.KeyValue, sf=common_sf.KeyValue) +KeyValueList = EncodeStrategy(pb2=common_pb2.KeyValueList, sf=common_sf.KeyValueList) + +LogRecord = EncodeStrategy(pb2=logs_pb2.LogRecord, sf=logs_sf.LogRecord) +ScopeLogs = EncodeStrategy(pb2=logs_pb2.ScopeLogs, sf=logs_sf.ScopeLogs) +ResourceLogs = EncodeStrategy(pb2=logs_pb2.ResourceLogs, sf=logs_sf.ResourceLogs) +LogsData = EncodeStrategy(pb2=logs_pb2.LogsData, sf=logs_sf.LogsData) + +TracesData = EncodeStrategy(pb2=trace_pb2.TracesData, sf=trace_sf.TracesData) +ScopeSpans = EncodeStrategy(pb2=trace_pb2.ScopeSpans, sf=trace_sf.ScopeSpans) +ResourceSpans = EncodeStrategy(pb2=trace_pb2.ResourceSpans, sf=trace_sf.ResourceSpans) +Span = EncodeStrategy(pb2=trace_pb2.Span, sf=trace_sf.Span) +Event = EncodeStrategy(pb2=trace_pb2.Span.Event, sf=trace_sf.Span_Event) +Link = EncodeStrategy(pb2=trace_pb2.Span.Link, sf=trace_sf.Span_Link) +Status = EncodeStrategy(pb2=trace_pb2.Status, sf=trace_sf.Status) + +Metric = EncodeStrategy(pb2=metrics_pb2.Metric, sf=metrics_sf.Metric) +ScopeMetrics = EncodeStrategy(pb2=metrics_pb2.ScopeMetrics, sf=metrics_sf.ScopeMetrics) +ResourceMetrics = EncodeStrategy(pb2=metrics_pb2.ResourceMetrics, sf=metrics_sf.ResourceMetrics) +MetricsData = EncodeStrategy(pb2=metrics_pb2.MetricsData, sf=metrics_sf.MetricsData) +Gauge = EncodeStrategy(pb2=metrics_pb2.Gauge, sf=metrics_sf.Gauge) +Sum = EncodeStrategy(pb2=metrics_pb2.Sum, sf=metrics_sf.Sum) +Histogram = EncodeStrategy(pb2=metrics_pb2.Histogram, sf=metrics_sf.Histogram) +ExponentialHistogram = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogram, sf=metrics_sf.ExponentialHistogram) +Summary = EncodeStrategy(pb2=metrics_pb2.Summary, sf=metrics_sf.Summary) +NumberDataPoint = EncodeStrategy(pb2=metrics_pb2.NumberDataPoint, sf=metrics_sf.NumberDataPoint) +Exemplar = EncodeStrategy(pb2=metrics_pb2.Exemplar, sf=metrics_sf.Exemplar) +HistogramDataPoint = EncodeStrategy(pb2=metrics_pb2.HistogramDataPoint, sf=metrics_sf.HistogramDataPoint) +ExponentialHistogramDataPoint = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogramDataPoint, sf=metrics_sf.ExponentialHistogramDataPoint) +SummaryDataPoint = EncodeStrategy(pb2=metrics_pb2.SummaryDataPoint, sf=metrics_sf.SummaryDataPoint) +ValueAtQuantile = EncodeStrategy(pb2=metrics_pb2.SummaryDataPoint.ValueAtQuantile, sf=metrics_sf.SummaryDataPoint_ValueAtQuantile) +Buckets = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogramDataPoint.Buckets, sf=metrics_sf.ExponentialHistogramDataPoint_Buckets) + + +# Package the protobuf type with its arguments for serialization +@dataclass +class EncodeWithArgs: + kwargs: Dict[str, Any] + cls: EncodeStrategy + +# Strategies for generating opentelemetry-proto types +@composite +def instrumentation_scope(draw): + return EncodeWithArgs({ + "name": draw(pb_string()), + "version": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + }, InstrumentationScope) + +@composite +def resource(draw): + return EncodeWithArgs({ + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + }, Resource) + +@composite +def any_value(draw): + # oneof field so only set one + oneof = draw(integers(min_value=0, max_value=6)) + if oneof == 0: + kwargs = {"string_value": draw(pb_string())} + elif oneof == 1: + kwargs = {"bool_value": draw(pb_bool())} + elif oneof == 2: + kwargs = {"int_value": draw(pb_int64())} + elif oneof == 3: + kwargs = {"double_value": draw_pb_double(draw)} + elif oneof == 4: + kwargs = {"array_value": draw(array_value())} + elif oneof == 5: + kwargs = {"kvlist_value": draw(key_value_list())} + elif oneof == 6: + kwargs = {"bytes_value": draw(pb_bytes())} + return EncodeWithArgs(kwargs, AnyValue) + +@composite +def array_value(draw): + return EncodeWithArgs({ + "values": draw(pb_repeated(any_value())), + }, ArrayValue) + +@composite +def key_value(draw): + return EncodeWithArgs({ + "key": draw(pb_string()), + "value": draw(any_value()), + }, KeyValue) + +@composite +def key_value_list(draw): + return EncodeWithArgs({ + "values": draw(pb_repeated(key_value())), + }, KeyValueList) + +@composite +def logs_data(draw): + @composite + def log_record(draw): + return EncodeWithArgs({ + "time_unix_nano": draw(pb_fixed64()), + "observed_time_unix_nano": draw(pb_fixed64()), + "severity_number": draw(pb_enum(logs_sf.SeverityNumber)), + "severity_text": draw(pb_string()), + "body": draw(any_value()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + "span_id": draw(pb_span_id()), + "trace_id": draw(pb_trace_id()), + }, LogRecord) + + @composite + def scope_logs(draw): + return EncodeWithArgs({ + "scope": draw(instrumentation_scope()), + "log_records": draw(pb_repeated(log_record())), + "schema_url": draw(pb_string()), + }, ScopeLogs) + + @composite + def resource_logs(draw): + return EncodeWithArgs({ + "resource": draw(resource()), + "scope_logs": draw(pb_repeated(scope_logs())), + "schema_url": draw(pb_string()), + }, ResourceLogs) + + return EncodeWithArgs({ + "resource_logs": draw(pb_repeated(resource_logs())), + }, LogsData) + +@composite +def traces_data(draw): + @composite + def event(draw): + return EncodeWithArgs({ + "time_unix_nano": draw(pb_fixed64()), + "name": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + }, Event) + + @composite + def link(draw): + return EncodeWithArgs({ + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + }, Link) + + @composite + def status(draw): + return EncodeWithArgs({ + "code": draw(pb_enum(trace_sf.Status_StatusCode)), + "message": draw(pb_string()), + }, Status) + + @composite + def span(draw): + return EncodeWithArgs({ + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "parent_span_id": draw(pb_span_id()), + "name": draw(pb_string()), + "kind": draw(pb_enum(trace_sf.Span_SpanKind)), + "start_time_unix_nano": draw(pb_fixed64()), + "end_time_unix_nano": draw(pb_fixed64()), + "attributes": draw(pb_repeated(key_value())), + "events": draw(pb_repeated(event())), + "links": draw(pb_repeated(link())), + "status": draw(status()), + "dropped_attributes_count": draw(pb_uint32()), + "dropped_events_count": draw(pb_uint32()), + "dropped_links_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + }, Span) + + @composite + def scope_spans(draw): + return EncodeWithArgs({ + "scope": draw(instrumentation_scope()), + "spans": draw(pb_repeated(span())), + "schema_url": draw(pb_string()), + }, ScopeSpans) + + @composite + def resource_spans(draw): + return EncodeWithArgs({ + "resource": draw(resource()), + "scope_spans": draw(pb_repeated(scope_spans())), + "schema_url": draw(pb_string()), + }, ResourceSpans) + + return EncodeWithArgs({ + "resource_spans": draw(pb_repeated(resource_spans())), + }, TracesData) + +@composite +def metrics_data(draw): + @composite + def exemplar(draw): + kwargs = {} + oneof = draw(integers(min_value=0, max_value=1)) + if oneof == 0: + kwargs["as_double"] = draw(pb_double()) + elif oneof == 1: + kwargs["as_int"] = draw(pb_sfixed64()) + + return EncodeWithArgs({ + **kwargs, + "time_unix_nano": draw(pb_fixed64()), + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "filtered_attributes": draw(pb_repeated(key_value())), + }, Exemplar) + + @composite + def value_at_quantile(draw): + return EncodeWithArgs({ + "quantile": draw_pb_double(draw), + "value": draw_pb_double(draw), + }, ValueAtQuantile) + + @composite + def summary_data_point(draw): + return EncodeWithArgs({ + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "quantile_values": draw(pb_repeated(value_at_quantile())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + }, SummaryDataPoint) + + @composite + def buckets(draw): + return EncodeWithArgs({ + "offset": draw(pb_sint32()), + "bucket_counts": draw(pb_repeated(pb_uint64())), + }, Buckets) + + @composite + def exponential_histogram_data_point(draw): + return EncodeWithArgs({ + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "positive": draw(buckets()), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "max": draw_pb_double(draw), + "zero_threshold": draw_pb_double(draw), + }, ExponentialHistogramDataPoint) + + @composite + def histogram_data_point(draw): + return EncodeWithArgs({ + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "bucket_counts": draw(pb_repeated(pb_uint64())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "explicit_bounds": draw(pb_repeated(pb_double())), + "max": draw_pb_double(draw), + }, HistogramDataPoint) + + @composite + def number_data_point(draw): + oneof = draw(integers(min_value=0, max_value=3)) + kwargs = {} + if oneof == 0: + kwargs["as_int"] = draw(pb_sfixed32()) + elif oneof == 1: + kwargs["as_double"] = draw(pb_double()) + + return EncodeWithArgs({ + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + **kwargs, + "exemplars": draw(pb_repeated(exemplar())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + }, NumberDataPoint) + + @composite + def summary(draw): + return EncodeWithArgs({ + "data_points": draw(pb_repeated(summary_data_point())), + }, Summary) + + @composite + def exponential_histogram(draw): + return EncodeWithArgs({ + "data_points": draw(pb_repeated(exponential_histogram_data_point())), + "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), + }, ExponentialHistogram) + + @composite + def histogram(draw): + return EncodeWithArgs({ + "data_points": draw(pb_repeated(histogram_data_point())), + "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), + }, Histogram) + + @composite + def sum(draw): + return EncodeWithArgs({ + "data_points": draw(pb_repeated(number_data_point())), + "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), + "is_monotonic": draw(pb_bool()), + }, Sum) + + @composite + def gauge(draw): + return EncodeWithArgs({ + "data_points": draw(pb_repeated(number_data_point())), + }, Gauge) + + @composite + def metric(draw): + oneof = draw(integers(min_value=0, max_value=3)) + kwargs = {} + if oneof == 0: + kwargs["gauge"] = draw(gauge()) + elif oneof == 1: + kwargs["sum"] = draw(sum()) + elif oneof == 2: + kwargs["histogram"] = draw(histogram()) + elif oneof == 3: + kwargs["exponential_histogram"] = draw(exponential_histogram()) + + return EncodeWithArgs({ + "name": draw(pb_string()), + "description": draw(pb_string()), + "unit": draw(pb_string()), + **kwargs, + "metadata": draw(pb_repeated(key_value())), + }, Metric) + + @composite + def scope_metrics(draw): + return EncodeWithArgs({ + "scope": draw(instrumentation_scope()), + "metrics": draw(pb_repeated(metric())), + "schema_url": draw(pb_string()), + }, ScopeMetrics) + + @composite + def resource_metrics(draw): + return EncodeWithArgs({ + "resource": draw(resource()), + "scope_metrics": draw(pb_repeated(scope_metrics())), + "schema_url": draw(pb_string()), + }, ResourceMetrics) + + return EncodeWithArgs({ + "resource_metrics": draw(pb_repeated(resource_metrics())), + }, MetricsData) + + +# Helper function to recursively encode protobuf types using the generated args +# and the given serialization strategy +def encode_recurse(obj: EncodeWithArgs, strategy: str) -> Any: + kwargs = {} + for key, value in obj.kwargs.items(): + if isinstance(value, EncodeWithArgs): + kwargs[key] = encode_recurse(value, strategy) + elif isinstance(value, list) and value and isinstance(value[0], EncodeWithArgs): + kwargs[key] = [encode_recurse(v, strategy) for v in value] + else: + kwargs[key] = value + if strategy == "pb2": + return obj.cls.pb2(**kwargs) + elif strategy == "sf": + return obj.cls.sf(**kwargs) + +class TestProtoSerialization(unittest.TestCase): + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(logs_data()) + def test_log_data(self, logs_data): + self.assertEqual( + encode_recurse(logs_data, "pb2").SerializeToString(deterministic=True), + bytes(encode_recurse(logs_data, "sf")) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(traces_data()) + def test_trace_data(self, traces_data): + self.assertEqual( + encode_recurse(traces_data, "pb2").SerializeToString(deterministic=True), + bytes(encode_recurse(traces_data, "sf")) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(metrics_data()) + def test_metrics_data(self, metrics_data): + self.assertEqual( + encode_recurse(metrics_data, "pb2").SerializeToString(deterministic=True), + bytes(encode_recurse(metrics_data, "sf")) + ) diff --git a/tests/test_protoc_plugin.py b/tests/test_protoc_plugin.py new file mode 100644 index 0000000..a312995 --- /dev/null +++ b/tests/test_protoc_plugin.py @@ -0,0 +1,91 @@ +""" +Test protoc code generator plugin for custom protoc message types +""" +import unittest +import tempfile +import subprocess +import os + +# Import into globals() so generated code string can be compiled +from snowflake.telemetry._internal.serialize import ProtoSerializer, Enum + +class TestProtocPlugin(unittest.TestCase): + def namespace_serialize_message(self, message_type: str, local_namespace: dict, **kwargs) -> bytes: + assert message_type in local_namespace, f"Message type {message_type} not found in local namespace" + return local_namespace[message_type](**kwargs) + + def test_protoc_plugin(self): + with tempfile.NamedTemporaryFile(suffix=".proto", mode="w", delete=False) as proto_file: + # Define a simple proto file + proto_file.write( + """syntax = "proto3"; +package opentelemetry.proto.common.v1; + +message AnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +message ArrayValue { + repeated AnyValue values = 1; +} + +message KeyValueList { + repeated KeyValue values = 1; +} + +message KeyValue { + string key = 1; + AnyValue value = 2; +} + +message InstrumentationScope { + string name = 1; + string version = 2; + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} +""" + ) + proto_file.flush() + proto_file.close() + + proto_file_dir = os.path.dirname(proto_file.name) + proto_file_name = os.path.basename(proto_file.name) + + # Run protoc with custom plugin to generate serialization code for messages + result = subprocess.run([ + "python", + "-m", + "grpc_tools.protoc", + "-I", + proto_file_dir, + "--plugin=protoc-gen-custom-plugin=scripts/plugin.py", + f"--custom-plugin_out={tempfile.gettempdir()}", + proto_file_name, + ], capture_output=True) + + # Ensure protoc ran successfully + self.assertEqual(result.returncode, 0) + + generated_code_file_name = proto_file_name.replace(".proto", ".py") + generated_code_file_dir = tempfile.gettempdir() + generated_code_file = os.path.join(generated_code_file_dir, generated_code_file_name) + + # Ensure generated code file exists + self.assertTrue(os.path.exists(os.path.join(generated_code_file_dir, generated_code_file_name))) + + # Ensure code can be executed and serializes correctly + with open(generated_code_file, "r") as f: + generated_code = f.read() + local_namespace = {} + eval(compile(generated_code, generated_code_file, "exec"), globals(), local_namespace) + + self.assertEqual(b'\n\x04test', self.namespace_serialize_message("AnyValue", local_namespace, string_value="test")) From aa6f8dac41100910cd188ba6f16fb3cd4ea22646 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Thu, 7 Nov 2024 11:02:35 -0800 Subject: [PATCH 04/11] Revert "Add code generator for otel proto (#39)" (#41) --- .github/workflows/check-codegen.yml | 36 -- README.md | 8 - scripts/plugin.py | 237 --------- scripts/proto_codegen.sh | 64 --- scripts/templates/template.py.jinja2 | 44 -- .../proto/collector/logs/v1/logs_service.py | 42 -- .../collector/metrics/v1/metrics_service.py | 42 -- .../proto/collector/trace/v1/trace_service.py | 42 -- .../opentelemetry/proto/common/v1/common.py | 87 ---- .../opentelemetry/proto/logs/v1/logs.py | 120 ----- .../opentelemetry/proto/metrics/v1/metrics.py | 341 ------------- .../proto/resource/v1/resource.py | 24 - .../opentelemetry/proto/trace/v1/trace.py | 181 ------- .../telemetry/_internal/serialize/__init__.py | 145 ------ tests/snowflake-telemetry-test-utils/setup.py | 5 - tests/test_proto_serialization.py | 472 ------------------ tests/test_protoc_plugin.py | 91 ---- 17 files changed, 1981 deletions(-) delete mode 100644 .github/workflows/check-codegen.yml delete mode 100755 scripts/plugin.py delete mode 100755 scripts/proto_codegen.sh delete mode 100644 scripts/templates/template.py.jinja2 delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py delete mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py delete mode 100644 src/snowflake/telemetry/_internal/serialize/__init__.py delete mode 100644 tests/test_proto_serialization.py delete mode 100644 tests/test_protoc_plugin.py diff --git a/.github/workflows/check-codegen.yml b/.github/workflows/check-codegen.yml deleted file mode 100644 index 17b4c6f..0000000 --- a/.github/workflows/check-codegen.yml +++ /dev/null @@ -1,36 +0,0 @@ -# This workflow will delete and regenerate the opentelemetry marshaling code using scripts/proto_codegen.sh. -# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Check Codegen - -on: - push: - branches: [ "main" ] - paths: - - "scripts/**" - - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" - - ".github/workflows/check-codegen.yml" - pull_request: - branches: [ "main" ] - paths: - - "scripts/**" - - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" - - ".github/workflows/check-codegen.yml" - -jobs: - check-codegen: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: "3.11" - - name: Run codegen script - run: | - rm -rf src/snowflake/telemetry/_internal/opentelemetry/proto/ - ./scripts/proto_codegen.sh - - name: Check for changes - run: | - git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/proto_codegen.sh"; exit 1; } diff --git a/README.md b/README.md index ca9d2b3..ef6e6ae 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,6 @@ pip install --upgrade pip pip install . ``` -## Development - To develop this package, run ```bash @@ -35,9 +33,3 @@ source .venv/bin/activate pip install --upgrade pip pip install . ./tests/snowflake-telemetry-test-utils ``` - -### Code generation - -To regenerate the code under `src/snowflake/_internal/opentelemetry/proto/`, execute the script `./scripts/proto_codegen.sh`. The script expects the `src/snowflake/_internal/opentelemetry/proto/` directory to exist, and will delete all .py files in it before regerating the code. - -The commit/branch/tag of [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-proto) that the code is generated from is pinned to PROTO_REPO_BRANCH_OR_COMMIT, which can be configured in the script. It is currently pinned to the same tag as [opentelemetry-python](https://github.com/open-telemetry/opentelemetry-python/blob/main/scripts/proto_codegen.sh#L15). diff --git a/scripts/plugin.py b/scripts/plugin.py deleted file mode 100755 index 588c2fc..0000000 --- a/scripts/plugin.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -from dataclasses import dataclass, field -from typing import List, Optional -from enum import IntEnum - -from google.protobuf.compiler import plugin_pb2 as plugin -from google.protobuf.descriptor_pb2 import ( - FileDescriptorProto, - FieldDescriptorProto, - EnumDescriptorProto, - EnumValueDescriptorProto, - MethodDescriptorProto, - ServiceDescriptorProto, - DescriptorProto, -) -from jinja2 import Environment, FileSystemLoader -import black -import isort.api - -class WireType(IntEnum): - VARINT = 0 - I64 = 1 - LEN = 2 - I32 = 5 - -@dataclass -class ProtoTypeDescriptor: - name: str - wire_type: WireType - python_type: str - -proto_type_to_descriptor = { - FieldDescriptorProto.TYPE_BOOL: ProtoTypeDescriptor("bool", WireType.VARINT, "bool"), - FieldDescriptorProto.TYPE_ENUM: ProtoTypeDescriptor("enum", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_INT32: ProtoTypeDescriptor("int32", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_INT64: ProtoTypeDescriptor("int64", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_UINT32: ProtoTypeDescriptor("uint32", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_UINT64: ProtoTypeDescriptor("uint64", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_SINT32: ProtoTypeDescriptor("sint32", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_SINT64: ProtoTypeDescriptor("sint64", WireType.VARINT, "int"), - FieldDescriptorProto.TYPE_FIXED32: ProtoTypeDescriptor("fixed32", WireType.I32, "int"), - FieldDescriptorProto.TYPE_FIXED64: ProtoTypeDescriptor("fixed64", WireType.I64, "int"), - FieldDescriptorProto.TYPE_SFIXED32: ProtoTypeDescriptor("sfixed32", WireType.I32, "int"), - FieldDescriptorProto.TYPE_SFIXED64: ProtoTypeDescriptor("sfixed64", WireType.I64, "int"), - FieldDescriptorProto.TYPE_FLOAT: ProtoTypeDescriptor("float", WireType.I32, "float"), - FieldDescriptorProto.TYPE_DOUBLE: ProtoTypeDescriptor("double", WireType.I64, "float"), - FieldDescriptorProto.TYPE_STRING: ProtoTypeDescriptor("string", WireType.LEN, "str"), - FieldDescriptorProto.TYPE_BYTES: ProtoTypeDescriptor("bytes", WireType.LEN, "bytes"), - FieldDescriptorProto.TYPE_MESSAGE: ProtoTypeDescriptor("message", WireType.LEN, "bytes"), -} - -@dataclass -class EnumValueTemplate: - name: str - number: int - - @staticmethod - def from_descriptor(descriptor: EnumValueDescriptorProto) -> "EnumValueTemplate": - return EnumValueTemplate( - name=descriptor.name, - number=descriptor.number, - ) - -@dataclass -class EnumTemplate: - name: str - values: List["EnumValueTemplate"] = field(default_factory=list) - - @staticmethod - def from_descriptor(descriptor: EnumDescriptorProto, parent: str = "") -> "EnumTemplate": - return EnumTemplate( - name=parent + "_" + descriptor.name if parent else descriptor.name, - values=[EnumValueTemplate.from_descriptor(value) for value in descriptor.value], - ) - -def tag_to_repr_varint(tag: int) -> str: - out = bytearray() - while tag >= 128: - out.append((tag & 0x7F) | 0x80) - tag >>= 7 - out.append(tag) - return repr(bytes(out)) - -@dataclass -class FieldTemplate: - name: str - number: int - tag: str - python_type: str - proto_type: str - repeated: bool - group: str - encode_presence: bool - - @staticmethod - def from_descriptor(descriptor: FieldDescriptorProto, group: Optional[str] = None) -> "FieldTemplate": - repeated = descriptor.label == FieldDescriptorProto.LABEL_REPEATED - type_descriptor = proto_type_to_descriptor[descriptor.type] - - python_type = type_descriptor.python_type - proto_type = type_descriptor.name - - if repeated: - python_type = f"List[{python_type}]" - proto_type = f"repeated_{proto_type}" - - tag = (descriptor.number << 3) | type_descriptor.wire_type.value - if repeated and type_descriptor.wire_type != WireType.LEN: - # Special case: repeated primitive fields are packed - # So we need to use the length-delimited wire type - tag = (descriptor.number << 3) | WireType.LEN.value - # Convert the tag to a varint representation - # Saves us from having to calculate the tag at runtime - tag = tag_to_repr_varint(tag) - - # For group / oneof fields, we need to encode the presence of the field - # For message fields, we need to encode the presence of the field if it is not None - encode_presence = group is not None or proto_type == "message" - - return FieldTemplate( - name=descriptor.name, - tag=tag, - number=descriptor.number, - python_type=python_type, - proto_type=proto_type, - repeated=repeated, - group=group, - encode_presence=encode_presence, - ) - -@dataclass -class MessageTemplate: - name: str - fields: List[FieldTemplate] = field(default_factory=list) - enums: List["EnumTemplate"] = field(default_factory=list) - messages: List["MessageTemplate"] = field(default_factory=list) - - @staticmethod - def from_descriptor(descriptor: DescriptorProto, parent: str = "") -> "MessageTemplate": - def get_group(field: FieldDescriptorProto) -> str: - return descriptor.oneof_decl[field.oneof_index].name if field.HasField("oneof_index") else None - fields = [FieldTemplate.from_descriptor(field, get_group(field)) for field in descriptor.field] - fields.sort(key=lambda field: field.number) - - name = parent + "_" + descriptor.name if parent else descriptor.name - return MessageTemplate( - name=name, - fields=fields, - enums=[EnumTemplate.from_descriptor(enum, name) for enum in descriptor.enum_type], - messages=[MessageTemplate.from_descriptor(message, name) for message in descriptor.nested_type], - ) - -@dataclass -class MethodTemplate: - name: str - input_message: MessageTemplate - output_message: MessageTemplate - - @staticmethod - def from_descriptor(descriptor: MethodDescriptorProto) -> "MethodTemplate": - return MethodTemplate( - name=descriptor.name, - input_message=MessageTemplate(name=descriptor.input_type), - output_message=MessageTemplate(name=descriptor.output_type), - ) - -@dataclass -class ServiceTemplate: - name: str - methods: List["MethodTemplate"] = field(default_factory=list) - - @staticmethod - def from_descriptor(descriptor: ServiceDescriptorProto) -> "ServiceTemplate": - return ServiceTemplate( - name=descriptor.name, - methods=[MethodTemplate.from_descriptor(method) for method in descriptor.method], - ) - -@dataclass -class FileTemplate: - messages: List["MessageTemplate"] = field(default_factory=list) - enums: List["EnumTemplate"] = field(default_factory=list) - services: List["ServiceTemplate"] = field(default_factory=list) - name: str = "" - - @staticmethod - def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": - return FileTemplate( - messages=[MessageTemplate.from_descriptor(message) for message in descriptor.message_type], - enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], - services=[ServiceTemplate.from_descriptor(service) for service in descriptor.service], - name=descriptor.name, - ) - -def main(): - request = plugin.CodeGeneratorRequest() - request.ParseFromString(sys.stdin.buffer.read()) - - response = plugin.CodeGeneratorResponse() - # needed since metrics.proto uses proto3 optional fields - response.supported_features = plugin.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL - - template_env = Environment(loader=FileSystemLoader(f"{os.path.dirname(os.path.realpath(__file__))}/templates")) - jinja_body_template = template_env.get_template("template.py.jinja2") - - for proto_file in request.proto_file: - file_name = proto_file.name.replace('.proto', '.py') - file_descriptor_proto = proto_file - - file_template = FileTemplate.from_descriptor(file_descriptor_proto) - - code = jinja_body_template.render(file_template=file_template) - code = isort.api.sort_code_string( - code = code, - show_diff=False, - profile="black", - combine_as_imports=True, - lines_after_imports=2, - quiet=True, - force_grid_wrap=2, - ) - code = black.format_str( - src_contents=code, - mode=black.Mode(), - ) - - response_file = response.file.add() - response_file.name = file_name - response_file.content = code - - sys.stdout.buffer.write(response.SerializeToString()) - -if __name__ == '__main__': - main() diff --git a/scripts/proto_codegen.sh b/scripts/proto_codegen.sh deleted file mode 100755 index 4d7dffa..0000000 --- a/scripts/proto_codegen.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Regenerate python code from OTLP protos in -# https://github.com/open-telemetry/opentelemetry-proto -# -# To use, update PROTO_REPO_BRANCH_OR_COMMIT variable below to a commit hash or -# tag in opentelemtry-proto repo that you want to build off of. Then, just run -# this script to update the proto files. Commit the changes as well as any -# fixes needed in the OTLP exporter. -# -# Optional envars: -# PROTO_REPO_DIR - the path to an existing checkout of the opentelemetry-proto repo - -# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. -PROTO_REPO_BRANCH_OR_COMMIT="v1.2.0" - -set -e - -PROTO_REPO_DIR=${PROTO_REPO_DIR:-"/tmp/opentelemetry-proto"} -# root of opentelemetry-python repo -repo_root="$(git rev-parse --show-toplevel)" -venv_dir="/tmp/proto_codegen_venv" - -# run on exit even if crash -cleanup() { - echo "Deleting $venv_dir" - rm -rf $venv_dir -} -trap cleanup EXIT - -echo "Creating temporary virtualenv at $venv_dir using $(python3 --version)" -python3 -m venv $venv_dir -source $venv_dir/bin/activate -python -m pip install protobuf Jinja2 grpcio-tools black isort -echo 'python -m grpc_tools.protoc --version' -python -m grpc_tools.protoc --version - -# Clone the proto repo if it doesn't exist -if [ ! -d "$PROTO_REPO_DIR" ]; then - git clone https://github.com/open-telemetry/opentelemetry-proto.git $PROTO_REPO_DIR -fi - -# Pull in changes and switch to requested branch -( - cd $PROTO_REPO_DIR - git fetch --all - git checkout $PROTO_REPO_BRANCH_OR_COMMIT - # pull if PROTO_REPO_BRANCH_OR_COMMIT is not a detached head - git symbolic-ref -q HEAD && git pull --ff-only || true -) - -cd $repo_root/src/snowflake/telemetry/_internal - -# clean up old generated code -mkdir -p opentelemetry/proto -find opentelemetry/proto/ -regex ".*\.py?" -exec rm {} + - -# generate proto code for all protos -all_protos=$(find $PROTO_REPO_DIR/ -iname "*.proto") -python -m grpc_tools.protoc \ - -I $PROTO_REPO_DIR \ - --plugin=protoc-gen-custom-plugin=$repo_root/scripts/plugin.py \ - --custom-plugin_out=. \ - $all_protos diff --git a/scripts/templates/template.py.jinja2 b/scripts/templates/template.py.jinja2 deleted file mode 100644 index 2246245..0000000 --- a/scripts/templates/template.py.jinja2 +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: {{ file_template.name }} - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) -from typing import List, Optional - -{% for enum in file_template.enums %} -class {{ enum.name }}(Enum): -{%- for value in enum.values %} - {{ value.name }} = {{ value.number }} -{%- endfor %} -{% endfor %} - -{% macro render_message(message) %} -def {{ message.name }}( -{%- for field in message.fields %} - {{ field.name }}: Optional[{{ field.python_type }}] = None, -{%- endfor %} -) -> bytes: - proto_serializer = ProtoSerializer() -{%- for field in message.fields %} - if {{ field.name }}{% if field.encode_presence %} is not None{% endif %}: {% if field.group %}# oneof group {{ field.group }}{% endif %} - proto_serializer.serialize_{{ field.proto_type }}({{ field.tag }}, {{ field.name }}) -{%- endfor %} - return proto_serializer.out - -{% for nested_enum in message.enums %} -class {{ nested_enum.name }}(Enum): -{%- for value in nested_enum.values %} - {{ value.name }} = {{ value.number }} -{%- endfor %} -{% endfor %} - -{% for nested_message in message.messages %} -{{ render_message(nested_message) }} -{% endfor %} -{% endmacro %} - -{% for message in file_template.messages %} -{{ render_message(message) }} -{% endfor %} \ No newline at end of file diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py deleted file mode 100644 index 2cfad4a..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/collector/logs/v1/logs_service.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -def ExportLogsServiceRequest( - resource_logs: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_logs: - proto_serializer.serialize_repeated_message(b"\n", resource_logs) - return proto_serializer.out - - -def ExportLogsServiceResponse( - partial_success: Optional[bytes] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if partial_success is not None: - proto_serializer.serialize_message(b"\n", partial_success) - return proto_serializer.out - - -def ExportLogsPartialSuccess( - rejected_log_records: Optional[int] = None, - error_message: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if rejected_log_records: - proto_serializer.serialize_int64(b"\x08", rejected_log_records) - if error_message: - proto_serializer.serialize_string(b"\x12", error_message) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py deleted file mode 100644 index 0c31719..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/collector/metrics/v1/metrics_service.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -def ExportMetricsServiceRequest( - resource_metrics: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_metrics: - proto_serializer.serialize_repeated_message(b"\n", resource_metrics) - return proto_serializer.out - - -def ExportMetricsServiceResponse( - partial_success: Optional[bytes] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if partial_success is not None: - proto_serializer.serialize_message(b"\n", partial_success) - return proto_serializer.out - - -def ExportMetricsPartialSuccess( - rejected_data_points: Optional[int] = None, - error_message: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if rejected_data_points: - proto_serializer.serialize_int64(b"\x08", rejected_data_points) - if error_message: - proto_serializer.serialize_string(b"\x12", error_message) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py deleted file mode 100644 index c4e2496..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/collector/trace/v1/trace_service.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -def ExportTraceServiceRequest( - resource_spans: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_spans: - proto_serializer.serialize_repeated_message(b"\n", resource_spans) - return proto_serializer.out - - -def ExportTraceServiceResponse( - partial_success: Optional[bytes] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if partial_success is not None: - proto_serializer.serialize_message(b"\n", partial_success) - return proto_serializer.out - - -def ExportTracePartialSuccess( - rejected_spans: Optional[int] = None, - error_message: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if rejected_spans: - proto_serializer.serialize_int64(b"\x08", rejected_spans) - if error_message: - proto_serializer.serialize_string(b"\x12", error_message) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py deleted file mode 100644 index 27f8c02..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common.py +++ /dev/null @@ -1,87 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/common/v1/common.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -def AnyValue( - string_value: Optional[str] = None, - bool_value: Optional[bool] = None, - int_value: Optional[int] = None, - double_value: Optional[float] = None, - array_value: Optional[bytes] = None, - kvlist_value: Optional[bytes] = None, - bytes_value: Optional[bytes] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if string_value is not None: # oneof group value - proto_serializer.serialize_string(b"\n", string_value) - if bool_value is not None: # oneof group value - proto_serializer.serialize_bool(b"\x10", bool_value) - if int_value is not None: # oneof group value - proto_serializer.serialize_int64(b"\x18", int_value) - if double_value is not None: # oneof group value - proto_serializer.serialize_double(b"!", double_value) - if array_value is not None: # oneof group value - proto_serializer.serialize_message(b"*", array_value) - if kvlist_value is not None: # oneof group value - proto_serializer.serialize_message(b"2", kvlist_value) - if bytes_value is not None: # oneof group value - proto_serializer.serialize_bytes(b":", bytes_value) - return proto_serializer.out - - -def ArrayValue( - values: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if values: - proto_serializer.serialize_repeated_message(b"\n", values) - return proto_serializer.out - - -def KeyValueList( - values: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if values: - proto_serializer.serialize_repeated_message(b"\n", values) - return proto_serializer.out - - -def KeyValue( - key: Optional[str] = None, - value: Optional[bytes] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if key: - proto_serializer.serialize_string(b"\n", key) - if value is not None: - proto_serializer.serialize_message(b"\x12", value) - return proto_serializer.out - - -def InstrumentationScope( - name: Optional[str] = None, - version: Optional[str] = None, - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if name: - proto_serializer.serialize_string(b"\n", name) - if version: - proto_serializer.serialize_string(b"\x12", version) - if attributes: - proto_serializer.serialize_repeated_message(b"\x1a", attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b" ", dropped_attributes_count) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py deleted file mode 100644 index 66b0e47..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs.py +++ /dev/null @@ -1,120 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/logs/v1/logs.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -class SeverityNumber(Enum): - SEVERITY_NUMBER_UNSPECIFIED = 0 - SEVERITY_NUMBER_TRACE = 1 - SEVERITY_NUMBER_TRACE2 = 2 - SEVERITY_NUMBER_TRACE3 = 3 - SEVERITY_NUMBER_TRACE4 = 4 - SEVERITY_NUMBER_DEBUG = 5 - SEVERITY_NUMBER_DEBUG2 = 6 - SEVERITY_NUMBER_DEBUG3 = 7 - SEVERITY_NUMBER_DEBUG4 = 8 - SEVERITY_NUMBER_INFO = 9 - SEVERITY_NUMBER_INFO2 = 10 - SEVERITY_NUMBER_INFO3 = 11 - SEVERITY_NUMBER_INFO4 = 12 - SEVERITY_NUMBER_WARN = 13 - SEVERITY_NUMBER_WARN2 = 14 - SEVERITY_NUMBER_WARN3 = 15 - SEVERITY_NUMBER_WARN4 = 16 - SEVERITY_NUMBER_ERROR = 17 - SEVERITY_NUMBER_ERROR2 = 18 - SEVERITY_NUMBER_ERROR3 = 19 - SEVERITY_NUMBER_ERROR4 = 20 - SEVERITY_NUMBER_FATAL = 21 - SEVERITY_NUMBER_FATAL2 = 22 - SEVERITY_NUMBER_FATAL3 = 23 - SEVERITY_NUMBER_FATAL4 = 24 - - -class LogRecordFlags(Enum): - LOG_RECORD_FLAGS_DO_NOT_USE = 0 - LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 255 - - -def LogsData( - resource_logs: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_logs: - proto_serializer.serialize_repeated_message(b"\n", resource_logs) - return proto_serializer.out - - -def ResourceLogs( - resource: Optional[bytes] = None, - scope_logs: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource is not None: - proto_serializer.serialize_message(b"\n", resource) - if scope_logs: - proto_serializer.serialize_repeated_message(b"\x12", scope_logs) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def ScopeLogs( - scope: Optional[bytes] = None, - log_records: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if scope is not None: - proto_serializer.serialize_message(b"\n", scope) - if log_records: - proto_serializer.serialize_repeated_message(b"\x12", log_records) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def LogRecord( - time_unix_nano: Optional[int] = None, - severity_number: Optional[int] = None, - severity_text: Optional[str] = None, - body: Optional[bytes] = None, - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, - flags: Optional[int] = None, - trace_id: Optional[bytes] = None, - span_id: Optional[bytes] = None, - observed_time_unix_nano: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\t", time_unix_nano) - if severity_number: - proto_serializer.serialize_enum(b"\x10", severity_number) - if severity_text: - proto_serializer.serialize_string(b"\x1a", severity_text) - if body is not None: - proto_serializer.serialize_message(b"*", body) - if attributes: - proto_serializer.serialize_repeated_message(b"2", attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b"8", dropped_attributes_count) - if flags: - proto_serializer.serialize_fixed32(b"E", flags) - if trace_id: - proto_serializer.serialize_bytes(b"J", trace_id) - if span_id: - proto_serializer.serialize_bytes(b"R", span_id) - if observed_time_unix_nano: - proto_serializer.serialize_fixed64(b"Y", observed_time_unix_nano) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py deleted file mode 100644 index d71f1e9..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics.py +++ /dev/null @@ -1,341 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/metrics/v1/metrics.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -class AggregationTemporality(Enum): - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0 - AGGREGATION_TEMPORALITY_DELTA = 1 - AGGREGATION_TEMPORALITY_CUMULATIVE = 2 - - -class DataPointFlags(Enum): - DATA_POINT_FLAGS_DO_NOT_USE = 0 - DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1 - - -def MetricsData( - resource_metrics: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_metrics: - proto_serializer.serialize_repeated_message(b"\n", resource_metrics) - return proto_serializer.out - - -def ResourceMetrics( - resource: Optional[bytes] = None, - scope_metrics: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource is not None: - proto_serializer.serialize_message(b"\n", resource) - if scope_metrics: - proto_serializer.serialize_repeated_message(b"\x12", scope_metrics) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def ScopeMetrics( - scope: Optional[bytes] = None, - metrics: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if scope is not None: - proto_serializer.serialize_message(b"\n", scope) - if metrics: - proto_serializer.serialize_repeated_message(b"\x12", metrics) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def Metric( - name: Optional[str] = None, - description: Optional[str] = None, - unit: Optional[str] = None, - gauge: Optional[bytes] = None, - sum: Optional[bytes] = None, - histogram: Optional[bytes] = None, - exponential_histogram: Optional[bytes] = None, - summary: Optional[bytes] = None, - metadata: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if name: - proto_serializer.serialize_string(b"\n", name) - if description: - proto_serializer.serialize_string(b"\x12", description) - if unit: - proto_serializer.serialize_string(b"\x1a", unit) - if gauge is not None: # oneof group data - proto_serializer.serialize_message(b"*", gauge) - if sum is not None: # oneof group data - proto_serializer.serialize_message(b":", sum) - if histogram is not None: # oneof group data - proto_serializer.serialize_message(b"J", histogram) - if exponential_histogram is not None: # oneof group data - proto_serializer.serialize_message(b"R", exponential_histogram) - if summary is not None: # oneof group data - proto_serializer.serialize_message(b"Z", summary) - if metadata: - proto_serializer.serialize_repeated_message(b"b", metadata) - return proto_serializer.out - - -def Gauge( - data_points: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if data_points: - proto_serializer.serialize_repeated_message(b"\n", data_points) - return proto_serializer.out - - -def Sum( - data_points: Optional[List[bytes]] = None, - aggregation_temporality: Optional[int] = None, - is_monotonic: Optional[bool] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if data_points: - proto_serializer.serialize_repeated_message(b"\n", data_points) - if aggregation_temporality: - proto_serializer.serialize_enum(b"\x10", aggregation_temporality) - if is_monotonic: - proto_serializer.serialize_bool(b"\x18", is_monotonic) - return proto_serializer.out - - -def Histogram( - data_points: Optional[List[bytes]] = None, - aggregation_temporality: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if data_points: - proto_serializer.serialize_repeated_message(b"\n", data_points) - if aggregation_temporality: - proto_serializer.serialize_enum(b"\x10", aggregation_temporality) - return proto_serializer.out - - -def ExponentialHistogram( - data_points: Optional[List[bytes]] = None, - aggregation_temporality: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if data_points: - proto_serializer.serialize_repeated_message(b"\n", data_points) - if aggregation_temporality: - proto_serializer.serialize_enum(b"\x10", aggregation_temporality) - return proto_serializer.out - - -def Summary( - data_points: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if data_points: - proto_serializer.serialize_repeated_message(b"\n", data_points) - return proto_serializer.out - - -def NumberDataPoint( - start_time_unix_nano: Optional[int] = None, - time_unix_nano: Optional[int] = None, - as_double: Optional[float] = None, - exemplars: Optional[List[bytes]] = None, - as_int: Optional[int] = None, - attributes: Optional[List[bytes]] = None, - flags: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if start_time_unix_nano: - proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) - if as_double is not None: # oneof group value - proto_serializer.serialize_double(b"!", as_double) - if exemplars: - proto_serializer.serialize_repeated_message(b"*", exemplars) - if as_int is not None: # oneof group value - proto_serializer.serialize_sfixed64(b"1", as_int) - if attributes: - proto_serializer.serialize_repeated_message(b":", attributes) - if flags: - proto_serializer.serialize_uint32(b"@", flags) - return proto_serializer.out - - -def HistogramDataPoint( - start_time_unix_nano: Optional[int] = None, - time_unix_nano: Optional[int] = None, - count: Optional[int] = None, - sum: Optional[float] = None, - bucket_counts: Optional[List[int]] = None, - explicit_bounds: Optional[List[float]] = None, - exemplars: Optional[List[bytes]] = None, - attributes: Optional[List[bytes]] = None, - flags: Optional[int] = None, - min: Optional[float] = None, - max: Optional[float] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if start_time_unix_nano: - proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) - if count: - proto_serializer.serialize_fixed64(b"!", count) - if sum is not None: # oneof group _sum - proto_serializer.serialize_double(b")", sum) - if bucket_counts: - proto_serializer.serialize_repeated_fixed64(b"2", bucket_counts) - if explicit_bounds: - proto_serializer.serialize_repeated_double(b":", explicit_bounds) - if exemplars: - proto_serializer.serialize_repeated_message(b"B", exemplars) - if attributes: - proto_serializer.serialize_repeated_message(b"J", attributes) - if flags: - proto_serializer.serialize_uint32(b"P", flags) - if min is not None: # oneof group _min - proto_serializer.serialize_double(b"Y", min) - if max is not None: # oneof group _max - proto_serializer.serialize_double(b"a", max) - return proto_serializer.out - - -def ExponentialHistogramDataPoint( - attributes: Optional[List[bytes]] = None, - start_time_unix_nano: Optional[int] = None, - time_unix_nano: Optional[int] = None, - count: Optional[int] = None, - sum: Optional[float] = None, - scale: Optional[int] = None, - zero_count: Optional[int] = None, - positive: Optional[bytes] = None, - negative: Optional[bytes] = None, - flags: Optional[int] = None, - exemplars: Optional[List[bytes]] = None, - min: Optional[float] = None, - max: Optional[float] = None, - zero_threshold: Optional[float] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if attributes: - proto_serializer.serialize_repeated_message(b"\n", attributes) - if start_time_unix_nano: - proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) - if count: - proto_serializer.serialize_fixed64(b"!", count) - if sum is not None: # oneof group _sum - proto_serializer.serialize_double(b")", sum) - if scale: - proto_serializer.serialize_sint32(b"0", scale) - if zero_count: - proto_serializer.serialize_fixed64(b"9", zero_count) - if positive is not None: - proto_serializer.serialize_message(b"B", positive) - if negative is not None: - proto_serializer.serialize_message(b"J", negative) - if flags: - proto_serializer.serialize_uint32(b"P", flags) - if exemplars: - proto_serializer.serialize_repeated_message(b"Z", exemplars) - if min is not None: # oneof group _min - proto_serializer.serialize_double(b"a", min) - if max is not None: # oneof group _max - proto_serializer.serialize_double(b"i", max) - if zero_threshold: - proto_serializer.serialize_double(b"q", zero_threshold) - return proto_serializer.out - - -def ExponentialHistogramDataPoint_Buckets( - offset: Optional[int] = None, - bucket_counts: Optional[List[int]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if offset: - proto_serializer.serialize_sint32(b"\x08", offset) - if bucket_counts: - proto_serializer.serialize_repeated_uint64(b"\x12", bucket_counts) - return proto_serializer.out - - -def SummaryDataPoint( - start_time_unix_nano: Optional[int] = None, - time_unix_nano: Optional[int] = None, - count: Optional[int] = None, - sum: Optional[float] = None, - quantile_values: Optional[List[bytes]] = None, - attributes: Optional[List[bytes]] = None, - flags: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if start_time_unix_nano: - proto_serializer.serialize_fixed64(b"\x11", start_time_unix_nano) - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\x19", time_unix_nano) - if count: - proto_serializer.serialize_fixed64(b"!", count) - if sum: - proto_serializer.serialize_double(b")", sum) - if quantile_values: - proto_serializer.serialize_repeated_message(b"2", quantile_values) - if attributes: - proto_serializer.serialize_repeated_message(b":", attributes) - if flags: - proto_serializer.serialize_uint32(b"@", flags) - return proto_serializer.out - - -def SummaryDataPoint_ValueAtQuantile( - quantile: Optional[float] = None, - value: Optional[float] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if quantile: - proto_serializer.serialize_double(b"\t", quantile) - if value: - proto_serializer.serialize_double(b"\x11", value) - return proto_serializer.out - - -def Exemplar( - time_unix_nano: Optional[int] = None, - as_double: Optional[float] = None, - span_id: Optional[bytes] = None, - trace_id: Optional[bytes] = None, - as_int: Optional[int] = None, - filtered_attributes: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\x11", time_unix_nano) - if as_double is not None: # oneof group value - proto_serializer.serialize_double(b"\x19", as_double) - if span_id: - proto_serializer.serialize_bytes(b'"', span_id) - if trace_id: - proto_serializer.serialize_bytes(b"*", trace_id) - if as_int is not None: # oneof group value - proto_serializer.serialize_sfixed64(b"1", as_int) - if filtered_attributes: - proto_serializer.serialize_repeated_message(b":", filtered_attributes) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py deleted file mode 100644 index 683727c..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/resource/v1/resource.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -def Resource( - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if attributes: - proto_serializer.serialize_repeated_message(b"\n", attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b"\x10", dropped_attributes_count) - return proto_serializer.out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py deleted file mode 100644 index 48f2908..0000000 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace.py +++ /dev/null @@ -1,181 +0,0 @@ -# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! -# sources: opentelemetry/proto/trace/v1/trace.proto - -from typing import ( - List, - Optional, -) - -from snowflake.telemetry._internal.serialize import ( - Enum, - ProtoSerializer, -) - - -class SpanFlags(Enum): - SPAN_FLAGS_DO_NOT_USE = 0 - SPAN_FLAGS_TRACE_FLAGS_MASK = 255 - SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 256 - SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 512 - - -def TracesData( - resource_spans: Optional[List[bytes]] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource_spans: - proto_serializer.serialize_repeated_message(b"\n", resource_spans) - return proto_serializer.out - - -def ResourceSpans( - resource: Optional[bytes] = None, - scope_spans: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if resource is not None: - proto_serializer.serialize_message(b"\n", resource) - if scope_spans: - proto_serializer.serialize_repeated_message(b"\x12", scope_spans) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def ScopeSpans( - scope: Optional[bytes] = None, - spans: Optional[List[bytes]] = None, - schema_url: Optional[str] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if scope is not None: - proto_serializer.serialize_message(b"\n", scope) - if spans: - proto_serializer.serialize_repeated_message(b"\x12", spans) - if schema_url: - proto_serializer.serialize_string(b"\x1a", schema_url) - return proto_serializer.out - - -def Span( - trace_id: Optional[bytes] = None, - span_id: Optional[bytes] = None, - trace_state: Optional[str] = None, - parent_span_id: Optional[bytes] = None, - name: Optional[str] = None, - kind: Optional[int] = None, - start_time_unix_nano: Optional[int] = None, - end_time_unix_nano: Optional[int] = None, - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, - events: Optional[List[bytes]] = None, - dropped_events_count: Optional[int] = None, - links: Optional[List[bytes]] = None, - dropped_links_count: Optional[int] = None, - status: Optional[bytes] = None, - flags: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if trace_id: - proto_serializer.serialize_bytes(b"\n", trace_id) - if span_id: - proto_serializer.serialize_bytes(b"\x12", span_id) - if trace_state: - proto_serializer.serialize_string(b"\x1a", trace_state) - if parent_span_id: - proto_serializer.serialize_bytes(b'"', parent_span_id) - if name: - proto_serializer.serialize_string(b"*", name) - if kind: - proto_serializer.serialize_enum(b"0", kind) - if start_time_unix_nano: - proto_serializer.serialize_fixed64(b"9", start_time_unix_nano) - if end_time_unix_nano: - proto_serializer.serialize_fixed64(b"A", end_time_unix_nano) - if attributes: - proto_serializer.serialize_repeated_message(b"J", attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b"P", dropped_attributes_count) - if events: - proto_serializer.serialize_repeated_message(b"Z", events) - if dropped_events_count: - proto_serializer.serialize_uint32(b"`", dropped_events_count) - if links: - proto_serializer.serialize_repeated_message(b"j", links) - if dropped_links_count: - proto_serializer.serialize_uint32(b"p", dropped_links_count) - if status is not None: - proto_serializer.serialize_message(b"z", status) - if flags: - proto_serializer.serialize_fixed32(b"\x85\x01", flags) - return proto_serializer.out - - -class Span_SpanKind(Enum): - SPAN_KIND_UNSPECIFIED = 0 - SPAN_KIND_INTERNAL = 1 - SPAN_KIND_SERVER = 2 - SPAN_KIND_CLIENT = 3 - SPAN_KIND_PRODUCER = 4 - SPAN_KIND_CONSUMER = 5 - - -def Span_Event( - time_unix_nano: Optional[int] = None, - name: Optional[str] = None, - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if time_unix_nano: - proto_serializer.serialize_fixed64(b"\t", time_unix_nano) - if name: - proto_serializer.serialize_string(b"\x12", name) - if attributes: - proto_serializer.serialize_repeated_message(b"\x1a", attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b" ", dropped_attributes_count) - return proto_serializer.out - - -def Span_Link( - trace_id: Optional[bytes] = None, - span_id: Optional[bytes] = None, - trace_state: Optional[str] = None, - attributes: Optional[List[bytes]] = None, - dropped_attributes_count: Optional[int] = None, - flags: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if trace_id: - proto_serializer.serialize_bytes(b"\n", trace_id) - if span_id: - proto_serializer.serialize_bytes(b"\x12", span_id) - if trace_state: - proto_serializer.serialize_string(b"\x1a", trace_state) - if attributes: - proto_serializer.serialize_repeated_message(b'"', attributes) - if dropped_attributes_count: - proto_serializer.serialize_uint32(b"(", dropped_attributes_count) - if flags: - proto_serializer.serialize_fixed32(b"5", flags) - return proto_serializer.out - - -def Status( - message: Optional[str] = None, - code: Optional[int] = None, -) -> bytes: - proto_serializer = ProtoSerializer() - if message: - proto_serializer.serialize_string(b"\x12", message) - if code: - proto_serializer.serialize_enum(b"\x18", code) - return proto_serializer.out - - -class Status_StatusCode(Enum): - STATUS_CODE_UNSET = 0 - STATUS_CODE_OK = 1 - STATUS_CODE_ERROR = 2 diff --git a/src/snowflake/telemetry/_internal/serialize/__init__.py b/src/snowflake/telemetry/_internal/serialize/__init__.py deleted file mode 100644 index 9d9063c..0000000 --- a/src/snowflake/telemetry/_internal/serialize/__init__.py +++ /dev/null @@ -1,145 +0,0 @@ -import struct -from enum import IntEnum -from typing import List, Union - -Enum = IntEnum - -class ProtoSerializer: - __slots__ = ("out") - - def __init__(self) -> None: - self.out = bytearray() - - def __bytes__(self) -> bytes: - return bytes(self.out) - - def serialize_bool(self, tag: bytes, value: bool) -> None: - self.out += tag - self._write_varint_unsigned(1 if value else 0) - - def serialize_enum(self, tag: bytes, value: Union[Enum, int]) -> None: - if not isinstance(value, int): - value = value.value - self.out += tag - self._write_varint_unsigned(value) - - def serialize_uint32(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value) - - def serialize_uint64(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value) - - def serialize_sint32(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value << 1 if value >= 0 else (value << 1) ^ (~0)) - - def serialize_sint64(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value << 1 if value >= 0 else (value << 1) ^ (~0)) - - def serialize_int32(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value + (1 << 32) if value < 0 else value) - - def serialize_int64(self, tag: bytes, value: int) -> None: - self.out += tag - self._write_varint_unsigned(value + (1 << 64) if value < 0 else value) - - def serialize_fixed32(self, tag: bytes, value: int) -> None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self.out += struct.pack(" None: - self.out += tag - self._write_varint_unsigned(len(value)) - self.out += value - - def serialize_string(self, tag: bytes, value: str) -> None: - self.serialize_bytes(tag, value.encode("utf-8")) - - def serialize_message( - self, - tag: bytes, - value: bytes, - ) -> None: - # If value is None, omit message entirely - if value is None: - return - # Otherwise, write the message - # Even if all fields are default (ommnited) - # The presence of the message is still encoded - self.out += tag - self._write_varint_unsigned(len(value)) - self.out += value - - def serialize_repeated_message( - self, - tag: bytes, - values: List[bytes], - ) -> None: - if not values: - return - # local reference to avoid repeated lookups - _self_serialize = self.serialize_message - for value in values: - _self_serialize(tag, value) - - def serialize_repeated_double(self, tag: bytes, values: List[float]) -> None: - if not values: - return - self.out += tag - self._write_varint_unsigned(len(values) * 8) - for value in values: - self.write_double_no_tag(value) - - def serialize_repeated_fixed64(self, tag: bytes, values: List[int]) -> None: - if not values: - return - self.out += tag - self._write_varint_unsigned(len(values) * 8) - for value in values: - self.write_fixed64_no_tag(value) - - def serialize_repeated_uint64(self, tag: bytes, values: List[int]) -> None: - if not values: - return - self.out += tag - tmp = ProtoSerializer() - for value in values: - tmp._write_varint_unsigned(value) - self._write_varint_unsigned(len(tmp.out)) - self.out += tmp.out - - def _write_varint_unsigned(self, value: int) -> None: - while value >= 128: - self.out.append((value & 0x7F) | 0x80) - value >>= 7 - self.out.append(value) - - def write_double_no_tag(self, value: float) -> None: - self.out += struct.pack(" None: - self.out += struct.pack("= 7.0.0", "snowflake-telemetry-python == 0.6.0.dev", - "Jinja2", - "grpcio-tools", - "black", - "isort", - "hypothesis >= 6.0.0", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_proto_serialization.py b/tests/test_proto_serialization.py deleted file mode 100644 index c9f4cc5..0000000 --- a/tests/test_proto_serialization.py +++ /dev/null @@ -1,472 +0,0 @@ -import unittest - -from dataclasses import dataclass -from typing import Any, Dict, Callable -import hypothesis -from hypothesis.strategies import composite, text, booleans, integers, floats, lists, binary, sampled_from -from hypothesis.control import assume - -import hypothesis.strategies -import opentelemetry.proto.logs.v1.logs_pb2 as logs_pb2 -import opentelemetry.proto.trace.v1.trace_pb2 as trace_pb2 -import opentelemetry.proto.common.v1.common_pb2 as common_pb2 -import opentelemetry.proto.metrics.v1.metrics_pb2 as metrics_pb2 -import opentelemetry.proto.resource.v1.resource_pb2 as resource_pb2 - -import snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs as logs_sf -import snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace as trace_sf -import snowflake.telemetry._internal.opentelemetry.proto.common.v1.common as common_sf -import snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics as metrics_sf -import snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource as resource_sf - -# Strategy for generating protobuf types -def pb_uint32(): return integers(min_value=0, max_value=2**32-1) -def pb_uint64(): return integers(min_value=0, max_value=2**64-1) -def pb_int32(): return integers(min_value=-2**31, max_value=2**31-1) -def pb_int64(): return integers(min_value=-2**63, max_value=2**63-1) -def pb_sint32(): return integers(min_value=-2**31, max_value=2**31-1) -def pb_sint64(): return integers(min_value=-2**63, max_value=2**63-1) -def pb_float(): return floats(allow_nan=False, allow_infinity=False, width=32) -def pb_double(): return floats(allow_nan=False, allow_infinity=False, width=64) -def draw_pb_double(draw): - # -0.0 is an edge case that is not handled by the snowflake serialization library - # Protobuf serialization will serialize -0.0 as "-0.0", and omit 0.0 - # Snowflake will omit both -0.0 and 0.0 - double = draw(pb_double()) - assume(str(double) != "-0.0") - return double -def pb_fixed64(): return pb_uint64() -def pb_fixed32(): return pb_uint32() -def pb_sfixed64(): return pb_int64() -def pb_sfixed32(): return pb_int32() -def pb_bool(): return booleans() -def pb_string(): return text(max_size=20) -def pb_bytes(): return binary(max_size=20) -def pb_enum(enum): return sampled_from([member.value for member in enum]) -def pb_repeated(type): return lists(type, max_size=3) # limit the size of the repeated field to speed up testing -def pb_span_id(): return binary(min_size=8, max_size=8) -def pb_trace_id(): return binary(min_size=16, max_size=16) - -# Maps protobuf types to their serialization functions, from the protobuf and snowflake serialization libraries -@dataclass -class EncodeStrategy: - pb2: Callable[[Any], Any] - sf: Callable[[Any], Any] - -Resource = EncodeStrategy(pb2=resource_pb2.Resource, sf=resource_sf.Resource) - -InstrumentationScope = EncodeStrategy(pb2=common_pb2.InstrumentationScope, sf=common_sf.InstrumentationScope) -AnyValue = EncodeStrategy(pb2=common_pb2.AnyValue, sf=common_sf.AnyValue) -ArrayValue = EncodeStrategy(pb2=common_pb2.ArrayValue, sf=common_sf.ArrayValue) -KeyValue = EncodeStrategy(pb2=common_pb2.KeyValue, sf=common_sf.KeyValue) -KeyValueList = EncodeStrategy(pb2=common_pb2.KeyValueList, sf=common_sf.KeyValueList) - -LogRecord = EncodeStrategy(pb2=logs_pb2.LogRecord, sf=logs_sf.LogRecord) -ScopeLogs = EncodeStrategy(pb2=logs_pb2.ScopeLogs, sf=logs_sf.ScopeLogs) -ResourceLogs = EncodeStrategy(pb2=logs_pb2.ResourceLogs, sf=logs_sf.ResourceLogs) -LogsData = EncodeStrategy(pb2=logs_pb2.LogsData, sf=logs_sf.LogsData) - -TracesData = EncodeStrategy(pb2=trace_pb2.TracesData, sf=trace_sf.TracesData) -ScopeSpans = EncodeStrategy(pb2=trace_pb2.ScopeSpans, sf=trace_sf.ScopeSpans) -ResourceSpans = EncodeStrategy(pb2=trace_pb2.ResourceSpans, sf=trace_sf.ResourceSpans) -Span = EncodeStrategy(pb2=trace_pb2.Span, sf=trace_sf.Span) -Event = EncodeStrategy(pb2=trace_pb2.Span.Event, sf=trace_sf.Span_Event) -Link = EncodeStrategy(pb2=trace_pb2.Span.Link, sf=trace_sf.Span_Link) -Status = EncodeStrategy(pb2=trace_pb2.Status, sf=trace_sf.Status) - -Metric = EncodeStrategy(pb2=metrics_pb2.Metric, sf=metrics_sf.Metric) -ScopeMetrics = EncodeStrategy(pb2=metrics_pb2.ScopeMetrics, sf=metrics_sf.ScopeMetrics) -ResourceMetrics = EncodeStrategy(pb2=metrics_pb2.ResourceMetrics, sf=metrics_sf.ResourceMetrics) -MetricsData = EncodeStrategy(pb2=metrics_pb2.MetricsData, sf=metrics_sf.MetricsData) -Gauge = EncodeStrategy(pb2=metrics_pb2.Gauge, sf=metrics_sf.Gauge) -Sum = EncodeStrategy(pb2=metrics_pb2.Sum, sf=metrics_sf.Sum) -Histogram = EncodeStrategy(pb2=metrics_pb2.Histogram, sf=metrics_sf.Histogram) -ExponentialHistogram = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogram, sf=metrics_sf.ExponentialHistogram) -Summary = EncodeStrategy(pb2=metrics_pb2.Summary, sf=metrics_sf.Summary) -NumberDataPoint = EncodeStrategy(pb2=metrics_pb2.NumberDataPoint, sf=metrics_sf.NumberDataPoint) -Exemplar = EncodeStrategy(pb2=metrics_pb2.Exemplar, sf=metrics_sf.Exemplar) -HistogramDataPoint = EncodeStrategy(pb2=metrics_pb2.HistogramDataPoint, sf=metrics_sf.HistogramDataPoint) -ExponentialHistogramDataPoint = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogramDataPoint, sf=metrics_sf.ExponentialHistogramDataPoint) -SummaryDataPoint = EncodeStrategy(pb2=metrics_pb2.SummaryDataPoint, sf=metrics_sf.SummaryDataPoint) -ValueAtQuantile = EncodeStrategy(pb2=metrics_pb2.SummaryDataPoint.ValueAtQuantile, sf=metrics_sf.SummaryDataPoint_ValueAtQuantile) -Buckets = EncodeStrategy(pb2=metrics_pb2.ExponentialHistogramDataPoint.Buckets, sf=metrics_sf.ExponentialHistogramDataPoint_Buckets) - - -# Package the protobuf type with its arguments for serialization -@dataclass -class EncodeWithArgs: - kwargs: Dict[str, Any] - cls: EncodeStrategy - -# Strategies for generating opentelemetry-proto types -@composite -def instrumentation_scope(draw): - return EncodeWithArgs({ - "name": draw(pb_string()), - "version": draw(pb_string()), - "attributes": draw(pb_repeated(key_value())), - "dropped_attributes_count": draw(pb_uint32()), - }, InstrumentationScope) - -@composite -def resource(draw): - return EncodeWithArgs({ - "attributes": draw(pb_repeated(key_value())), - "dropped_attributes_count": draw(pb_uint32()), - }, Resource) - -@composite -def any_value(draw): - # oneof field so only set one - oneof = draw(integers(min_value=0, max_value=6)) - if oneof == 0: - kwargs = {"string_value": draw(pb_string())} - elif oneof == 1: - kwargs = {"bool_value": draw(pb_bool())} - elif oneof == 2: - kwargs = {"int_value": draw(pb_int64())} - elif oneof == 3: - kwargs = {"double_value": draw_pb_double(draw)} - elif oneof == 4: - kwargs = {"array_value": draw(array_value())} - elif oneof == 5: - kwargs = {"kvlist_value": draw(key_value_list())} - elif oneof == 6: - kwargs = {"bytes_value": draw(pb_bytes())} - return EncodeWithArgs(kwargs, AnyValue) - -@composite -def array_value(draw): - return EncodeWithArgs({ - "values": draw(pb_repeated(any_value())), - }, ArrayValue) - -@composite -def key_value(draw): - return EncodeWithArgs({ - "key": draw(pb_string()), - "value": draw(any_value()), - }, KeyValue) - -@composite -def key_value_list(draw): - return EncodeWithArgs({ - "values": draw(pb_repeated(key_value())), - }, KeyValueList) - -@composite -def logs_data(draw): - @composite - def log_record(draw): - return EncodeWithArgs({ - "time_unix_nano": draw(pb_fixed64()), - "observed_time_unix_nano": draw(pb_fixed64()), - "severity_number": draw(pb_enum(logs_sf.SeverityNumber)), - "severity_text": draw(pb_string()), - "body": draw(any_value()), - "attributes": draw(pb_repeated(key_value())), - "dropped_attributes_count": draw(pb_uint32()), - "flags": draw(pb_fixed32()), - "span_id": draw(pb_span_id()), - "trace_id": draw(pb_trace_id()), - }, LogRecord) - - @composite - def scope_logs(draw): - return EncodeWithArgs({ - "scope": draw(instrumentation_scope()), - "log_records": draw(pb_repeated(log_record())), - "schema_url": draw(pb_string()), - }, ScopeLogs) - - @composite - def resource_logs(draw): - return EncodeWithArgs({ - "resource": draw(resource()), - "scope_logs": draw(pb_repeated(scope_logs())), - "schema_url": draw(pb_string()), - }, ResourceLogs) - - return EncodeWithArgs({ - "resource_logs": draw(pb_repeated(resource_logs())), - }, LogsData) - -@composite -def traces_data(draw): - @composite - def event(draw): - return EncodeWithArgs({ - "time_unix_nano": draw(pb_fixed64()), - "name": draw(pb_string()), - "attributes": draw(pb_repeated(key_value())), - "dropped_attributes_count": draw(pb_uint32()), - }, Event) - - @composite - def link(draw): - return EncodeWithArgs({ - "trace_id": draw(pb_trace_id()), - "span_id": draw(pb_span_id()), - "trace_state": draw(pb_string()), - "attributes": draw(pb_repeated(key_value())), - "dropped_attributes_count": draw(pb_uint32()), - "flags": draw(pb_fixed32()), - }, Link) - - @composite - def status(draw): - return EncodeWithArgs({ - "code": draw(pb_enum(trace_sf.Status_StatusCode)), - "message": draw(pb_string()), - }, Status) - - @composite - def span(draw): - return EncodeWithArgs({ - "trace_id": draw(pb_trace_id()), - "span_id": draw(pb_span_id()), - "trace_state": draw(pb_string()), - "parent_span_id": draw(pb_span_id()), - "name": draw(pb_string()), - "kind": draw(pb_enum(trace_sf.Span_SpanKind)), - "start_time_unix_nano": draw(pb_fixed64()), - "end_time_unix_nano": draw(pb_fixed64()), - "attributes": draw(pb_repeated(key_value())), - "events": draw(pb_repeated(event())), - "links": draw(pb_repeated(link())), - "status": draw(status()), - "dropped_attributes_count": draw(pb_uint32()), - "dropped_events_count": draw(pb_uint32()), - "dropped_links_count": draw(pb_uint32()), - "flags": draw(pb_fixed32()), - }, Span) - - @composite - def scope_spans(draw): - return EncodeWithArgs({ - "scope": draw(instrumentation_scope()), - "spans": draw(pb_repeated(span())), - "schema_url": draw(pb_string()), - }, ScopeSpans) - - @composite - def resource_spans(draw): - return EncodeWithArgs({ - "resource": draw(resource()), - "scope_spans": draw(pb_repeated(scope_spans())), - "schema_url": draw(pb_string()), - }, ResourceSpans) - - return EncodeWithArgs({ - "resource_spans": draw(pb_repeated(resource_spans())), - }, TracesData) - -@composite -def metrics_data(draw): - @composite - def exemplar(draw): - kwargs = {} - oneof = draw(integers(min_value=0, max_value=1)) - if oneof == 0: - kwargs["as_double"] = draw(pb_double()) - elif oneof == 1: - kwargs["as_int"] = draw(pb_sfixed64()) - - return EncodeWithArgs({ - **kwargs, - "time_unix_nano": draw(pb_fixed64()), - "trace_id": draw(pb_trace_id()), - "span_id": draw(pb_span_id()), - "filtered_attributes": draw(pb_repeated(key_value())), - }, Exemplar) - - @composite - def value_at_quantile(draw): - return EncodeWithArgs({ - "quantile": draw_pb_double(draw), - "value": draw_pb_double(draw), - }, ValueAtQuantile) - - @composite - def summary_data_point(draw): - return EncodeWithArgs({ - "start_time_unix_nano": draw(pb_fixed64()), - "time_unix_nano": draw(pb_fixed64()), - "count": draw(pb_fixed64()), - "sum": draw_pb_double(draw), - "quantile_values": draw(pb_repeated(value_at_quantile())), - "attributes": draw(pb_repeated(key_value())), - "flags": draw(pb_uint32()), - }, SummaryDataPoint) - - @composite - def buckets(draw): - return EncodeWithArgs({ - "offset": draw(pb_sint32()), - "bucket_counts": draw(pb_repeated(pb_uint64())), - }, Buckets) - - @composite - def exponential_histogram_data_point(draw): - return EncodeWithArgs({ - "start_time_unix_nano": draw(pb_fixed64()), - "time_unix_nano": draw(pb_fixed64()), - "count": draw(pb_fixed64()), - "sum": draw_pb_double(draw), - "positive": draw(buckets()), - "attributes": draw(pb_repeated(key_value())), - "flags": draw(pb_uint32()), - "exemplars": draw(pb_repeated(exemplar())), - "max": draw_pb_double(draw), - "zero_threshold": draw_pb_double(draw), - }, ExponentialHistogramDataPoint) - - @composite - def histogram_data_point(draw): - return EncodeWithArgs({ - "start_time_unix_nano": draw(pb_fixed64()), - "time_unix_nano": draw(pb_fixed64()), - "count": draw(pb_fixed64()), - "sum": draw_pb_double(draw), - "bucket_counts": draw(pb_repeated(pb_uint64())), - "attributes": draw(pb_repeated(key_value())), - "flags": draw(pb_uint32()), - "exemplars": draw(pb_repeated(exemplar())), - "explicit_bounds": draw(pb_repeated(pb_double())), - "max": draw_pb_double(draw), - }, HistogramDataPoint) - - @composite - def number_data_point(draw): - oneof = draw(integers(min_value=0, max_value=3)) - kwargs = {} - if oneof == 0: - kwargs["as_int"] = draw(pb_sfixed32()) - elif oneof == 1: - kwargs["as_double"] = draw(pb_double()) - - return EncodeWithArgs({ - "start_time_unix_nano": draw(pb_fixed64()), - "time_unix_nano": draw(pb_fixed64()), - **kwargs, - "exemplars": draw(pb_repeated(exemplar())), - "attributes": draw(pb_repeated(key_value())), - "flags": draw(pb_uint32()), - }, NumberDataPoint) - - @composite - def summary(draw): - return EncodeWithArgs({ - "data_points": draw(pb_repeated(summary_data_point())), - }, Summary) - - @composite - def exponential_histogram(draw): - return EncodeWithArgs({ - "data_points": draw(pb_repeated(exponential_histogram_data_point())), - "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), - }, ExponentialHistogram) - - @composite - def histogram(draw): - return EncodeWithArgs({ - "data_points": draw(pb_repeated(histogram_data_point())), - "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), - }, Histogram) - - @composite - def sum(draw): - return EncodeWithArgs({ - "data_points": draw(pb_repeated(number_data_point())), - "aggregation_temporality": draw(pb_enum(metrics_sf.AggregationTemporality)), - "is_monotonic": draw(pb_bool()), - }, Sum) - - @composite - def gauge(draw): - return EncodeWithArgs({ - "data_points": draw(pb_repeated(number_data_point())), - }, Gauge) - - @composite - def metric(draw): - oneof = draw(integers(min_value=0, max_value=3)) - kwargs = {} - if oneof == 0: - kwargs["gauge"] = draw(gauge()) - elif oneof == 1: - kwargs["sum"] = draw(sum()) - elif oneof == 2: - kwargs["histogram"] = draw(histogram()) - elif oneof == 3: - kwargs["exponential_histogram"] = draw(exponential_histogram()) - - return EncodeWithArgs({ - "name": draw(pb_string()), - "description": draw(pb_string()), - "unit": draw(pb_string()), - **kwargs, - "metadata": draw(pb_repeated(key_value())), - }, Metric) - - @composite - def scope_metrics(draw): - return EncodeWithArgs({ - "scope": draw(instrumentation_scope()), - "metrics": draw(pb_repeated(metric())), - "schema_url": draw(pb_string()), - }, ScopeMetrics) - - @composite - def resource_metrics(draw): - return EncodeWithArgs({ - "resource": draw(resource()), - "scope_metrics": draw(pb_repeated(scope_metrics())), - "schema_url": draw(pb_string()), - }, ResourceMetrics) - - return EncodeWithArgs({ - "resource_metrics": draw(pb_repeated(resource_metrics())), - }, MetricsData) - - -# Helper function to recursively encode protobuf types using the generated args -# and the given serialization strategy -def encode_recurse(obj: EncodeWithArgs, strategy: str) -> Any: - kwargs = {} - for key, value in obj.kwargs.items(): - if isinstance(value, EncodeWithArgs): - kwargs[key] = encode_recurse(value, strategy) - elif isinstance(value, list) and value and isinstance(value[0], EncodeWithArgs): - kwargs[key] = [encode_recurse(v, strategy) for v in value] - else: - kwargs[key] = value - if strategy == "pb2": - return obj.cls.pb2(**kwargs) - elif strategy == "sf": - return obj.cls.sf(**kwargs) - -class TestProtoSerialization(unittest.TestCase): - @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) - @hypothesis.given(logs_data()) - def test_log_data(self, logs_data): - self.assertEqual( - encode_recurse(logs_data, "pb2").SerializeToString(deterministic=True), - bytes(encode_recurse(logs_data, "sf")) - ) - - @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) - @hypothesis.given(traces_data()) - def test_trace_data(self, traces_data): - self.assertEqual( - encode_recurse(traces_data, "pb2").SerializeToString(deterministic=True), - bytes(encode_recurse(traces_data, "sf")) - ) - - @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) - @hypothesis.given(metrics_data()) - def test_metrics_data(self, metrics_data): - self.assertEqual( - encode_recurse(metrics_data, "pb2").SerializeToString(deterministic=True), - bytes(encode_recurse(metrics_data, "sf")) - ) diff --git a/tests/test_protoc_plugin.py b/tests/test_protoc_plugin.py deleted file mode 100644 index a312995..0000000 --- a/tests/test_protoc_plugin.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Test protoc code generator plugin for custom protoc message types -""" -import unittest -import tempfile -import subprocess -import os - -# Import into globals() so generated code string can be compiled -from snowflake.telemetry._internal.serialize import ProtoSerializer, Enum - -class TestProtocPlugin(unittest.TestCase): - def namespace_serialize_message(self, message_type: str, local_namespace: dict, **kwargs) -> bytes: - assert message_type in local_namespace, f"Message type {message_type} not found in local namespace" - return local_namespace[message_type](**kwargs) - - def test_protoc_plugin(self): - with tempfile.NamedTemporaryFile(suffix=".proto", mode="w", delete=False) as proto_file: - # Define a simple proto file - proto_file.write( - """syntax = "proto3"; -package opentelemetry.proto.common.v1; - -message AnyValue { - oneof value { - string string_value = 1; - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValue array_value = 5; - KeyValueList kvlist_value = 6; - bytes bytes_value = 7; - } -} - -message ArrayValue { - repeated AnyValue values = 1; -} - -message KeyValueList { - repeated KeyValue values = 1; -} - -message KeyValue { - string key = 1; - AnyValue value = 2; -} - -message InstrumentationScope { - string name = 1; - string version = 2; - repeated KeyValue attributes = 3; - uint32 dropped_attributes_count = 4; -} -""" - ) - proto_file.flush() - proto_file.close() - - proto_file_dir = os.path.dirname(proto_file.name) - proto_file_name = os.path.basename(proto_file.name) - - # Run protoc with custom plugin to generate serialization code for messages - result = subprocess.run([ - "python", - "-m", - "grpc_tools.protoc", - "-I", - proto_file_dir, - "--plugin=protoc-gen-custom-plugin=scripts/plugin.py", - f"--custom-plugin_out={tempfile.gettempdir()}", - proto_file_name, - ], capture_output=True) - - # Ensure protoc ran successfully - self.assertEqual(result.returncode, 0) - - generated_code_file_name = proto_file_name.replace(".proto", ".py") - generated_code_file_dir = tempfile.gettempdir() - generated_code_file = os.path.join(generated_code_file_dir, generated_code_file_name) - - # Ensure generated code file exists - self.assertTrue(os.path.exists(os.path.join(generated_code_file_dir, generated_code_file_name))) - - # Ensure code can be executed and serializes correctly - with open(generated_code_file, "r") as f: - generated_code = f.read() - local_namespace = {} - eval(compile(generated_code, generated_code_file, "exec"), globals(), local_namespace) - - self.assertEqual(b'\n\x04test', self.namespace_serialize_message("AnyValue", local_namespace, string_value="test")) From faf52d42f7739c0e360e7b8e5b60dace58eb6e24 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Thu, 21 Nov 2024 08:37:17 -0800 Subject: [PATCH 05/11] Add otel proto codegen using forward serialization (#42) --- .github/workflows/check-codegen.yml | 36 + README.md | 8 + scripts/gen-requirements.txt | 5 + scripts/plugin.py | 345 +++++ scripts/proto_codegen.sh | 67 + scripts/templates/template.py.jinja2 | 76 ++ .../logs/v1/logs_service_marshaler.py | 108 ++ .../metrics/v1/metrics_service_marshaler.py | 108 ++ .../trace/v1/trace_service_marshaler.py | 108 ++ .../proto/common/v1/common_marshaler.py | 281 ++++ .../proto/logs/v1/logs_marshaler.py | 339 +++++ .../proto/metrics/v1/metrics_marshaler.py | 1209 +++++++++++++++++ .../proto/resource/v1/resource_marshaler.py | 55 + .../proto/trace/v1/trace_marshaler.py | 597 ++++++++ .../telemetry/_internal/serialize/__init__.py | 291 ++++ tests/snowflake-telemetry-test-utils/setup.py | 5 + tests/test_proto_serialization.py | 501 +++++++ tests/test_protoc_plugin.py | 91 ++ 18 files changed, 4230 insertions(+) create mode 100644 .github/workflows/check-codegen.yml create mode 100644 scripts/gen-requirements.txt create mode 100755 scripts/plugin.py create mode 100755 scripts/proto_codegen.sh create mode 100644 scripts/templates/template.py.jinja2 create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py create mode 100644 src/snowflake/telemetry/_internal/serialize/__init__.py create mode 100644 tests/test_proto_serialization.py create mode 100644 tests/test_protoc_plugin.py diff --git a/.github/workflows/check-codegen.yml b/.github/workflows/check-codegen.yml new file mode 100644 index 0000000..17b4c6f --- /dev/null +++ b/.github/workflows/check-codegen.yml @@ -0,0 +1,36 @@ +# This workflow will delete and regenerate the opentelemetry marshaling code using scripts/proto_codegen.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check Codegen + +on: + push: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - ".github/workflows/check-codegen.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - ".github/workflows/check-codegen.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Run codegen script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/proto/ + ./scripts/proto_codegen.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/proto_codegen.sh"; exit 1; } diff --git a/README.md b/README.md index ef6e6ae..ca9d2b3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ pip install --upgrade pip pip install . ``` +## Development + To develop this package, run ```bash @@ -33,3 +35,9 @@ source .venv/bin/activate pip install --upgrade pip pip install . ./tests/snowflake-telemetry-test-utils ``` + +### Code generation + +To regenerate the code under `src/snowflake/_internal/opentelemetry/proto/`, execute the script `./scripts/proto_codegen.sh`. The script expects the `src/snowflake/_internal/opentelemetry/proto/` directory to exist, and will delete all .py files in it before regerating the code. + +The commit/branch/tag of [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-proto) that the code is generated from is pinned to PROTO_REPO_BRANCH_OR_COMMIT, which can be configured in the script. It is currently pinned to the same tag as [opentelemetry-python](https://github.com/open-telemetry/opentelemetry-python/blob/main/scripts/proto_codegen.sh#L15). diff --git a/scripts/gen-requirements.txt b/scripts/gen-requirements.txt new file mode 100644 index 0000000..bf8c682 --- /dev/null +++ b/scripts/gen-requirements.txt @@ -0,0 +1,5 @@ +Jinja2==3.1.4 +grpcio-tools==1.62.3 +protobuf==4.25.5 +black==24.10.0 +isort==5.13.2 diff --git a/scripts/plugin.py b/scripts/plugin.py new file mode 100755 index 0000000..d01cb00 --- /dev/null +++ b/scripts/plugin.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import re +import os +import sys +import inspect +from enum import IntEnum +from typing import List, Optional +from textwrap import dedent, indent +from dataclasses import dataclass, field +# Must be imported into globals for the inline functions to work +from snowflake.telemetry._internal.serialize import MessageMarshaler # noqa + +from google.protobuf.compiler import plugin_pb2 as plugin +from google.protobuf.descriptor_pb2 import ( + FileDescriptorProto, + FieldDescriptorProto, + EnumDescriptorProto, + EnumValueDescriptorProto, + DescriptorProto, +) +from jinja2 import Environment, FileSystemLoader +import black +import isort.api + +INLINE_OPTIMIZATION = True +FILE_PATH_PREFIX = "snowflake.telemetry._internal" +FILE_NAME_SUFFIX = "_marshaler" + +# Inline utility functions + +# Inline the size function for a given proto message field +def inline_size_function(proto_type: str, attr_name: str, field_tag: str) -> str: + """ + For example: + + class MessageMarshaler: + def size_uint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR) + + Becomes: + + size += len(b"\x10") + Varint.size_varint_u32(self.int_value) + """ + function_definition = inspect.getsource(globals()["MessageMarshaler"].__dict__[f"size_{proto_type}"]) + # Remove the function header and unindent the function body + function_definition = function_definition.splitlines()[1:] + function_definition = "\n".join(function_definition) + function_definition = dedent(function_definition) + # Replace the attribute name + function_definition = function_definition.replace("FIELD_ATTR", f"self.{attr_name}") + # Replace the TAG + function_definition = function_definition.replace("TAG", field_tag) + # Inline the return statement + function_definition = function_definition.replace("return ", "size += ") + return function_definition + +# Inline the serialization function for a given proto message field +def inline_serialize_function(proto_type: str, attr_name: str, field_tag: str) -> str: + """ + For example: + + class MessageMarshaler: + def serialize_uint32(self, out: BytesIO, TAG: bytes, FIELD_ATTR: int) -> None: + out.write(TAG) + Varint.serialize_varint_u32(out, FIELD_ATTR) + + Becomes: + + out.write(b"\x10") + Varint.serialize_varint_u32(out, self.int_value) + """ + function_definition = inspect.getsource(globals()["MessageMarshaler"].__dict__[f"serialize_{proto_type}"]) + # Remove the function header and unindent the function body + function_definition = function_definition.splitlines()[1:] + function_definition = "\n".join(function_definition) + function_definition = dedent(function_definition) + # Replace the attribute name + function_definition = function_definition.replace("FIELD_ATTR", f"self.{attr_name}") + # Replace the TAG + function_definition = function_definition.replace("TAG", field_tag) + return function_definition + +# Add a presence check to a function definition +# https://protobuf.dev/programming-guides/proto3/#default +def add_presence_check(proto_type: str, encode_presence: bool, attr_name: str, function_definition: str) -> str: + # oneof, optional (virtual oneof), and message fields are encoded if they are not None + function_definition = indent(function_definition, " ") + if encode_presence: + return f"if self.{attr_name} is not None:\n{function_definition}" + # Other fields are encoded if they are not the default value + # Which happens to align with the bool(x) check for all primitive types + # TODO: Except + # - double and float -0.0 should be encoded, even though bool(-0.0) is False + return f"if self.{attr_name}:\n{function_definition}" + +class WireType(IntEnum): + VARINT = 0 + I64 = 1 + LEN = 2 + I32 = 5 + +@dataclass +class ProtoTypeDescriptor: + name: str + wire_type: WireType + python_type: str + default_val: str + +proto_type_to_descriptor = { + FieldDescriptorProto.TYPE_BOOL: ProtoTypeDescriptor("bool", WireType.VARINT, "bool", "False"), + FieldDescriptorProto.TYPE_ENUM: ProtoTypeDescriptor("enum", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_INT32: ProtoTypeDescriptor("int32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_INT64: ProtoTypeDescriptor("int64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_UINT32: ProtoTypeDescriptor("uint32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_UINT64: ProtoTypeDescriptor("uint64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_SINT32: ProtoTypeDescriptor("sint32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_SINT64: ProtoTypeDescriptor("sint64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_FIXED32: ProtoTypeDescriptor("fixed32", WireType.I32, "int", "0"), + FieldDescriptorProto.TYPE_FIXED64: ProtoTypeDescriptor("fixed64", WireType.I64, "int", "0"), + FieldDescriptorProto.TYPE_SFIXED32: ProtoTypeDescriptor("sfixed32", WireType.I32, "int", "0"), + FieldDescriptorProto.TYPE_SFIXED64: ProtoTypeDescriptor("sfixed64", WireType.I64, "int", "0"), + FieldDescriptorProto.TYPE_FLOAT: ProtoTypeDescriptor("float", WireType.I32, "float", "0.0"), + FieldDescriptorProto.TYPE_DOUBLE: ProtoTypeDescriptor("double", WireType.I64, "float", "0.0"), + FieldDescriptorProto.TYPE_STRING: ProtoTypeDescriptor("string", WireType.LEN, "str", '""'), + FieldDescriptorProto.TYPE_BYTES: ProtoTypeDescriptor("bytes", WireType.LEN, "bytes", 'b""'), + FieldDescriptorProto.TYPE_MESSAGE: ProtoTypeDescriptor("message", WireType.LEN, "PLACEHOLDER", "None"), +} + +@dataclass +class EnumValueTemplate: + name: str + number: int + + @staticmethod + def from_descriptor(descriptor: EnumValueDescriptorProto) -> "EnumValueTemplate": + return EnumValueTemplate( + name=descriptor.name, + number=descriptor.number, + ) + +@dataclass +class EnumTemplate: + name: str + values: List[EnumValueTemplate] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: EnumDescriptorProto) -> "EnumTemplate": + return EnumTemplate( + name=descriptor.name, + values=[EnumValueTemplate.from_descriptor(value) for value in descriptor.value], + ) + +def tag_to_repr_varint(tag: int) -> str: + out = bytearray() + while tag >= 128: + out.append((tag & 0x7F) | 0x80) + tag >>= 7 + out.append(tag) + return repr(bytes(out)) + +@dataclass +class FieldTemplate: + name: str + attr_name: str + number: int + generator: str + python_type: str + proto_type: str + default_val: str + serialize_field_inline: str + size_field_inline: str + + @staticmethod + def from_descriptor(descriptor: FieldDescriptorProto, group: Optional[str] = None) -> "FieldTemplate": + type_descriptor = proto_type_to_descriptor[descriptor.type] + python_type = type_descriptor.python_type + proto_type = type_descriptor.name + default_val = type_descriptor.default_val + + if proto_type == "message" or proto_type == "enum": + # Extract the class name of message fields, to use as python type + python_type = re.sub(r"^[a-zA-Z0-9_\.]+\.v1\.", "", descriptor.type_name) + + repeated = descriptor.label == FieldDescriptorProto.LABEL_REPEATED + if repeated: + # Update type for repeated fields + python_type = f"List[{python_type}]" + proto_type = f"repeated_{proto_type}" + # Default value is None, since we can't use a mutable default value like [] + default_val = "None" + + # Calculate the tag for the field to save some computation at runtime + tag = (descriptor.number << 3) | type_descriptor.wire_type.value + if repeated and type_descriptor.wire_type != WireType.LEN: + # Special case: repeated primitive fields are packed, so need to use LEN wire type + # Note: packed fields can be disabled in proto files, but we don't handle that case + # https://protobuf.dev/programming-guides/encoding/#packed + tag = (descriptor.number << 3) | WireType.LEN.value + # Convert the tag to a varint representation to inline it in the generated code + tag = tag_to_repr_varint(tag) + + # For oneof and optional fields, we need to encode the presence of the field. + # Optional fields are treated as virtual oneof fields, with a single field in the oneof. + # For message fields, we need to encode the presence of the field if it is not None. + # https://protobuf.dev/programming-guides/field_presence/ + encode_presence = group is not None or proto_type == "message" + if group is not None: + # The default value for oneof fields must be None, so that the default value is not encoded + default_val = "None" + + field_name = descriptor.name + attr_name = field_name + generator = None + if proto_type == "message" or repeated: + # For message and repeated fields, store as a private attribute that is + # initialized on access to match protobuf embedded message access pattern + if repeated: + # In python protobuf, repeated fields return an implementation of the list interface + # with a self.add() method to add and initialize elements + # This can be supported with a custom list implementation, but we use a simple list for now + # https://protobuf.dev/reference/python/python-generated/#repeated-message-fields + generator = "list()" + else: + # https://protobuf.dev/reference/python/python-generated/#embedded_message + generator = f"{python_type}()" + # the attribute name is prefixed with an underscore as message and repeated attributes + # are hidden behind a property that has the actual proto field name + attr_name = f"_{field_name}" + + # Inline the size and serialization functions for the field + if INLINE_OPTIMIZATION: + serialize_field_inline = inline_serialize_function(proto_type, attr_name, tag) + size_field_inline = inline_size_function(proto_type, attr_name, tag) + else: + serialize_field_inline = f"self.serialize_{proto_type}(out, {tag}, self.{attr_name})" + size_field_inline = f"size += self.size_{proto_type}({tag}, self.{attr_name})" + + serialize_field_inline = add_presence_check(proto_type, encode_presence, attr_name, serialize_field_inline) + size_field_inline = add_presence_check(proto_type, encode_presence, attr_name, size_field_inline) + + return FieldTemplate( + name=field_name, + attr_name=attr_name, + number=descriptor.number, + generator=generator, + python_type=python_type, + proto_type=proto_type, + default_val=default_val, + serialize_field_inline=serialize_field_inline, + size_field_inline=size_field_inline, + ) + +@dataclass +class MessageTemplate: + name: str + fields: List[FieldTemplate] = field(default_factory=list) + enums: List[EnumTemplate] = field(default_factory=list) + messages: List[MessageTemplate] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: DescriptorProto) -> "MessageTemplate": + # Helper function to extract the group name for a field, if it exists + def get_group(field: FieldDescriptorProto) -> str: + return descriptor.oneof_decl[field.oneof_index].name if field.HasField("oneof_index") else None + fields = [FieldTemplate.from_descriptor(field, get_group(field)) for field in descriptor.field] + fields.sort(key=lambda field: field.number) + + name = descriptor.name + return MessageTemplate( + name=name, + fields=fields, + enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], + messages=[MessageTemplate.from_descriptor(message) for message in descriptor.nested_type], + ) + +@dataclass +class FileTemplate: + messages: List[MessageTemplate] = field(default_factory=list) + enums: List[EnumTemplate] = field(default_factory=list) + imports: List[str] = field(default_factory=list) + name: str = "" + + @staticmethod + def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": + + # Extract the import paths for the proto file + imports = [] + for dependency in descriptor.dependency: + path = re.sub(r"\.proto$", "", dependency) + if descriptor.name.startswith(path): + continue + path = path.replace("/", ".") + path = f"{FILE_PATH_PREFIX}.{path}{FILE_NAME_SUFFIX}" + imports.append(path) + + return FileTemplate( + messages=[MessageTemplate.from_descriptor(message) for message in descriptor.message_type], + enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], + imports=imports, + name=descriptor.name, + ) + +def main(): + request = plugin.CodeGeneratorRequest() + request.ParseFromString(sys.stdin.buffer.read()) + + response = plugin.CodeGeneratorResponse() + # Needed since metrics.proto uses proto3 optional fields + # https://github.com/protocolbuffers/protobuf/blob/main/docs/implementing_proto3_presence.md + response.supported_features = plugin.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + + template_env = Environment(loader=FileSystemLoader(f"{os.path.dirname(os.path.realpath(__file__))}/templates")) + jinja_body_template = template_env.get_template("template.py.jinja2") + + for proto_file in request.proto_file: + file_name = re.sub(r"\.proto$", f"{FILE_NAME_SUFFIX}.py", proto_file.name) + file_descriptor_proto = proto_file + + file_template = FileTemplate.from_descriptor(file_descriptor_proto) + + code = jinja_body_template.render(file_template=file_template) + code = isort.api.sort_code_string( + code = code, + show_diff=False, + profile="black", + combine_as_imports=True, + lines_after_imports=2, + quiet=True, + force_grid_wrap=2, + ) + code = black.format_str( + src_contents=code, + mode=black.Mode(), + ) + + response_file = response.file.add() + response_file.name = file_name + response_file.content = code + + sys.stdout.buffer.write(response.SerializeToString()) + +if __name__ == '__main__': + main() diff --git a/scripts/proto_codegen.sh b/scripts/proto_codegen.sh new file mode 100755 index 0000000..9db288f --- /dev/null +++ b/scripts/proto_codegen.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Regenerate python code from OTLP protos in +# https://github.com/open-telemetry/opentelemetry-proto +# +# To use, update PROTO_REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-proto repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. +# +# Optional envars: +# PROTO_REPO_DIR - the path to an existing checkout of the opentelemetry-proto repo + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +PROTO_REPO_BRANCH_OR_COMMIT="v1.2.0" + +set -e + +PROTO_REPO_DIR=${PROTO_REPO_DIR:-"/tmp/opentelemetry-proto"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" +venv_dir="/tmp/proto_codegen_venv" + +# run on exit even if crash +cleanup() { + echo "Deleting $venv_dir" + rm -rf $venv_dir +} +trap cleanup EXIT + +echo "Creating temporary virtualenv at $venv_dir using $(python3 --version)" +python3 -m venv $venv_dir +source $venv_dir/bin/activate +python -m pip install \ + -c $repo_root/scripts/gen-requirements.txt \ + protobuf Jinja2 grpcio-tools black isort . + +echo 'python -m grpc_tools.protoc --version' +python -m grpc_tools.protoc --version + +# Clone the proto repo if it doesn't exist +if [ ! -d "$PROTO_REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-proto.git $PROTO_REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $PROTO_REPO_DIR + git fetch --all + git checkout $PROTO_REPO_BRANCH_OR_COMMIT + # pull if PROTO_REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# clean up old generated code +mkdir -p opentelemetry/proto +find opentelemetry/proto/ -regex ".*_marshaler\.py" -exec rm {} + + +# generate proto code for all protos +all_protos=$(find $PROTO_REPO_DIR/ -iname "*.proto") +python -m grpc_tools.protoc \ + -I $PROTO_REPO_DIR \ + --plugin=protoc-gen-custom-plugin=$repo_root/scripts/plugin.py \ + --custom-plugin_out=. \ + $all_protos diff --git a/scripts/templates/template.py.jinja2 b/scripts/templates/template.py.jinja2 new file mode 100644 index 0000000..9de445f --- /dev/null +++ b/scripts/templates/template.py.jinja2 @@ -0,0 +1,76 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: {{ file_template.name }} + +from __future__ import annotations + +import struct +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) +from typing import List + +{% for import in file_template.imports %} +from {{ import }} import * +{% endfor %} + +{% for enum in file_template.enums %} +class {{ enum.name }}(Enum): +{%- for value in enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% macro render_message(message) %} +class {{ message.name }}(MessageMarshaler): + +{%- for field in message.fields %} +{%- if field.generator %} + @property + def {{ field.name }}(self) -> {{ field.python_type }}: + if self.{{ field.attr_name }} is None: + self.{{ field.attr_name }} = {{ field.generator }} + return self.{{ field.attr_name }} +{%- else %} + {{ field.name }}: {{ field.python_type }} +{%- endif %} +{%- endfor %} + + def __init__( + self, +{%- for field in message.fields %} + {{ field.name }}: {{ field.python_type }} = {{ field.default_val }}, +{%- endfor %} + ): +{%- for field in message.fields %} + self.{{ field.attr_name }}: {{ field.python_type }} = {{ field.name }} +{%- endfor %} + + def calculate_size(self) -> int: + size = 0 +{%- for field in message.fields %} + {{ field.size_field_inline | indent(8) }} +{%- endfor %} + return size + + def write_to(self, out: bytearray) -> None: +{%- for field in message.fields %} + {{ field.serialize_field_inline | indent(8) }} +{%- endfor %} + +{% for nested_enum in message.enums %} + class {{ nested_enum.name }}(Enum): +{%- for value in nested_enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% for nested_message in message.messages %} +{{ render_message(nested_message) | indent(4) }} +{% endfor %} +{% endmacro %} + +{% for message in file_template.messages %} +{{ render_message(message) }} +{% endfor %} \ No newline at end of file diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py new file mode 100644 index 0000000..52711d2 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py @@ -0,0 +1,108 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/logs/v1/logs_service.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportLogsServiceRequest(MessageMarshaler): + @property + def resource_logs(self) -> List[ResourceLogs]: + if self._resource_logs is None: + self._resource_logs = list() + return self._resource_logs + + def __init__( + self, + resource_logs: List[ResourceLogs] = None, + ): + self._resource_logs: List[ResourceLogs] = resource_logs + + def calculate_size(self) -> int: + size = 0 + if self._resource_logs: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_logs + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_logs: + for v in self._resource_logs: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportLogsServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportLogsPartialSuccess: + if self._partial_success is None: + self._partial_success = ExportLogsPartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportLogsPartialSuccess = None, + ): + self._partial_success: ExportLogsPartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportLogsPartialSuccess(MessageMarshaler): + rejected_log_records: int + error_message: str + + def __init__( + self, + rejected_log_records: int = 0, + error_message: str = "", + ): + self.rejected_log_records: int = rejected_log_records + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_log_records: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_log_records) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_log_records: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_log_records) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py new file mode 100644 index 0000000..7701775 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py @@ -0,0 +1,108 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/metrics/v1/metrics_service.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportMetricsServiceRequest(MessageMarshaler): + @property + def resource_metrics(self) -> List[ResourceMetrics]: + if self._resource_metrics is None: + self._resource_metrics = list() + return self._resource_metrics + + def __init__( + self, + resource_metrics: List[ResourceMetrics] = None, + ): + self._resource_metrics: List[ResourceMetrics] = resource_metrics + + def calculate_size(self) -> int: + size = 0 + if self._resource_metrics: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_metrics + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_metrics: + for v in self._resource_metrics: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportMetricsServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportMetricsPartialSuccess: + if self._partial_success is None: + self._partial_success = ExportMetricsPartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportMetricsPartialSuccess = None, + ): + self._partial_success: ExportMetricsPartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportMetricsPartialSuccess(MessageMarshaler): + rejected_data_points: int + error_message: str + + def __init__( + self, + rejected_data_points: int = 0, + error_message: str = "", + ): + self.rejected_data_points: int = rejected_data_points + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_data_points: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_data_points) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_data_points: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_data_points) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py new file mode 100644 index 0000000..2488f6c --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py @@ -0,0 +1,108 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/trace/v1/trace_service.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportTraceServiceRequest(MessageMarshaler): + @property + def resource_spans(self) -> List[ResourceSpans]: + if self._resource_spans is None: + self._resource_spans = list() + return self._resource_spans + + def __init__( + self, + resource_spans: List[ResourceSpans] = None, + ): + self._resource_spans: List[ResourceSpans] = resource_spans + + def calculate_size(self) -> int: + size = 0 + if self._resource_spans: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_spans + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_spans: + for v in self._resource_spans: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportTraceServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportTracePartialSuccess: + if self._partial_success is None: + self._partial_success = ExportTracePartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportTracePartialSuccess = None, + ): + self._partial_success: ExportTracePartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportTracePartialSuccess(MessageMarshaler): + rejected_spans: int + error_message: str + + def __init__( + self, + rejected_spans: int = 0, + error_message: str = "", + ): + self.rejected_spans: int = rejected_spans + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_spans: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_spans) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_spans: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_spans) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py new file mode 100644 index 0000000..2d25bcc --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py @@ -0,0 +1,281 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/common/v1/common.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class AnyValue(MessageMarshaler): + string_value: str + bool_value: bool + int_value: int + double_value: float + + @property + def array_value(self) -> ArrayValue: + if self._array_value is None: + self._array_value = ArrayValue() + return self._array_value + + @property + def kvlist_value(self) -> KeyValueList: + if self._kvlist_value is None: + self._kvlist_value = KeyValueList() + return self._kvlist_value + + bytes_value: bytes + + def __init__( + self, + string_value: str = None, + bool_value: bool = None, + int_value: int = None, + double_value: float = None, + array_value: ArrayValue = None, + kvlist_value: KeyValueList = None, + bytes_value: bytes = None, + ): + self.string_value: str = string_value + self.bool_value: bool = bool_value + self.int_value: int = int_value + self.double_value: float = double_value + self._array_value: ArrayValue = array_value + self._kvlist_value: KeyValueList = kvlist_value + self.bytes_value: bytes = bytes_value + + def calculate_size(self) -> int: + size = 0 + if self.string_value is not None: + v = self.string_value.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.bool_value is not None: + size += len(b"\x10") + 1 + if self.int_value is not None: + size += len(b"\x18") + Varint.size_varint_i64(self.int_value) + if self.double_value is not None: + size += len(b"!") + 8 + if self._array_value is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._array_value._get_size()) + + self._array_value._get_size() + ) + if self._kvlist_value is not None: + size += ( + len(b"2") + + Varint.size_varint_u32(self._kvlist_value._get_size()) + + self._kvlist_value._get_size() + ) + if self.bytes_value is not None: + size += ( + len(b":") + + Varint.size_varint_u32(len(self.bytes_value)) + + len(self.bytes_value) + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.string_value is not None: + v = self.string_value.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.bool_value is not None: + out += b"\x10" + Varint.write_varint_u32(out, 1 if self.bool_value else 0) + if self.int_value is not None: + out += b"\x18" + Varint.write_varint_i64(out, self.int_value) + if self.double_value is not None: + out += b"!" + out += struct.pack(" List[AnyValue]: + if self._values is None: + self._values = list() + return self._values + + def __init__( + self, + values: List[AnyValue] = None, + ): + self._values: List[AnyValue] = values + + def calculate_size(self) -> int: + size = 0 + if self._values: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._values + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._values: + for v in self._values: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class KeyValueList(MessageMarshaler): + @property + def values(self) -> List[KeyValue]: + if self._values is None: + self._values = list() + return self._values + + def __init__( + self, + values: List[KeyValue] = None, + ): + self._values: List[KeyValue] = values + + def calculate_size(self) -> int: + size = 0 + if self._values: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._values + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._values: + for v in self._values: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class KeyValue(MessageMarshaler): + key: str + + @property + def value(self) -> AnyValue: + if self._value is None: + self._value = AnyValue() + return self._value + + def __init__( + self, + key: str = "", + value: AnyValue = None, + ): + self.key: str = key + self._value: AnyValue = value + + def calculate_size(self) -> int: + size = 0 + if self.key: + v = self.key.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self._value is not None: + size += ( + len(b"\x12") + + Varint.size_varint_u32(self._value._get_size()) + + self._value._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.key: + v = self.key.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self._value is not None: + out += b"\x12" + Varint.write_varint_u32(out, self._value._get_size()) + self._value.write_to(out) + + +class InstrumentationScope(MessageMarshaler): + name: str + version: str + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + name: str = "", + version: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self.name: str = name + self.version: str = version + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.version: + v = self.version.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b"\x1a") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b" ") + Varint.size_varint_u32(self.dropped_attributes_count) + return size + + def write_to(self, out: bytearray) -> None: + if self.name: + v = self.name.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.version: + v = self.version.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self._attributes: + for v in self._attributes: + out += b"\x1a" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b" " + Varint.write_varint_u32(out, self.dropped_attributes_count) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py new file mode 100644 index 0000000..18996f0 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py @@ -0,0 +1,339 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/logs/v1/logs.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import * +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class SeverityNumber(Enum): + SEVERITY_NUMBER_UNSPECIFIED = 0 + SEVERITY_NUMBER_TRACE = 1 + SEVERITY_NUMBER_TRACE2 = 2 + SEVERITY_NUMBER_TRACE3 = 3 + SEVERITY_NUMBER_TRACE4 = 4 + SEVERITY_NUMBER_DEBUG = 5 + SEVERITY_NUMBER_DEBUG2 = 6 + SEVERITY_NUMBER_DEBUG3 = 7 + SEVERITY_NUMBER_DEBUG4 = 8 + SEVERITY_NUMBER_INFO = 9 + SEVERITY_NUMBER_INFO2 = 10 + SEVERITY_NUMBER_INFO3 = 11 + SEVERITY_NUMBER_INFO4 = 12 + SEVERITY_NUMBER_WARN = 13 + SEVERITY_NUMBER_WARN2 = 14 + SEVERITY_NUMBER_WARN3 = 15 + SEVERITY_NUMBER_WARN4 = 16 + SEVERITY_NUMBER_ERROR = 17 + SEVERITY_NUMBER_ERROR2 = 18 + SEVERITY_NUMBER_ERROR3 = 19 + SEVERITY_NUMBER_ERROR4 = 20 + SEVERITY_NUMBER_FATAL = 21 + SEVERITY_NUMBER_FATAL2 = 22 + SEVERITY_NUMBER_FATAL3 = 23 + SEVERITY_NUMBER_FATAL4 = 24 + + +class LogRecordFlags(Enum): + LOG_RECORD_FLAGS_DO_NOT_USE = 0 + LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 255 + + +class LogsData(MessageMarshaler): + @property + def resource_logs(self) -> List[ResourceLogs]: + if self._resource_logs is None: + self._resource_logs = list() + return self._resource_logs + + def __init__( + self, + resource_logs: List[ResourceLogs] = None, + ): + self._resource_logs: List[ResourceLogs] = resource_logs + + def calculate_size(self) -> int: + size = 0 + if self._resource_logs: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_logs + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_logs: + for v in self._resource_logs: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceLogs(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_logs(self) -> List[ScopeLogs]: + if self._scope_logs is None: + self._scope_logs = list() + return self._scope_logs + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_logs: List[ScopeLogs] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_logs: List[ScopeLogs] = scope_logs + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_logs: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_logs + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_logs: + for v in self._scope_logs: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeLogs(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def log_records(self) -> List[LogRecord]: + if self._log_records is None: + self._log_records = list() + return self._log_records + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + log_records: List[LogRecord] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._log_records: List[LogRecord] = log_records + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._log_records: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._log_records + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._log_records: + for v in self._log_records: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class LogRecord(MessageMarshaler): + time_unix_nano: int + severity_number: SeverityNumber + severity_text: str + + @property + def body(self) -> AnyValue: + if self._body is None: + self._body = AnyValue() + return self._body + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + flags: int + trace_id: bytes + span_id: bytes + observed_time_unix_nano: int + + def __init__( + self, + time_unix_nano: int = 0, + severity_number: SeverityNumber = 0, + severity_text: str = "", + body: AnyValue = None, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + flags: int = 0, + trace_id: bytes = b"", + span_id: bytes = b"", + observed_time_unix_nano: int = 0, + ): + self.time_unix_nano: int = time_unix_nano + self.severity_number: SeverityNumber = severity_number + self.severity_text: str = severity_text + self._body: AnyValue = body + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self.flags: int = flags + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.observed_time_unix_nano: int = observed_time_unix_nano + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\t") + 8 + if self.severity_number: + v = self.severity_number + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + if self.severity_text: + v = self.severity_text.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._body is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._body._get_size()) + + self._body._get_size() + ) + if self._attributes: + size += sum( + message._get_size() + + len(b"2") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"8") + Varint.size_varint_u32(self.dropped_attributes_count) + if self.flags: + size += len(b"E") + 4 + if self.trace_id: + size += ( + len(b"J") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"R") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.observed_time_unix_nano: + size += len(b"Y") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\t" + out += struct.pack(" List[ResourceMetrics]: + if self._resource_metrics is None: + self._resource_metrics = list() + return self._resource_metrics + + def __init__( + self, + resource_metrics: List[ResourceMetrics] = None, + ): + self._resource_metrics: List[ResourceMetrics] = resource_metrics + + def calculate_size(self) -> int: + size = 0 + if self._resource_metrics: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_metrics + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_metrics: + for v in self._resource_metrics: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceMetrics(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_metrics(self) -> List[ScopeMetrics]: + if self._scope_metrics is None: + self._scope_metrics = list() + return self._scope_metrics + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_metrics: List[ScopeMetrics] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_metrics: List[ScopeMetrics] = scope_metrics + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_metrics: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_metrics + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_metrics: + for v in self._scope_metrics: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeMetrics(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def metrics(self) -> List[Metric]: + if self._metrics is None: + self._metrics = list() + return self._metrics + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + metrics: List[Metric] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._metrics: List[Metric] = metrics + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._metrics: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._metrics + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._metrics: + for v in self._metrics: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class Metric(MessageMarshaler): + name: str + description: str + unit: str + + @property + def gauge(self) -> Gauge: + if self._gauge is None: + self._gauge = Gauge() + return self._gauge + + @property + def sum(self) -> Sum: + if self._sum is None: + self._sum = Sum() + return self._sum + + @property + def histogram(self) -> Histogram: + if self._histogram is None: + self._histogram = Histogram() + return self._histogram + + @property + def exponential_histogram(self) -> ExponentialHistogram: + if self._exponential_histogram is None: + self._exponential_histogram = ExponentialHistogram() + return self._exponential_histogram + + @property + def summary(self) -> Summary: + if self._summary is None: + self._summary = Summary() + return self._summary + + @property + def metadata(self) -> List[KeyValue]: + if self._metadata is None: + self._metadata = list() + return self._metadata + + def __init__( + self, + name: str = "", + description: str = "", + unit: str = "", + gauge: Gauge = None, + sum: Sum = None, + histogram: Histogram = None, + exponential_histogram: ExponentialHistogram = None, + summary: Summary = None, + metadata: List[KeyValue] = None, + ): + self.name: str = name + self.description: str = description + self.unit: str = unit + self._gauge: Gauge = gauge + self._sum: Sum = sum + self._histogram: Histogram = histogram + self._exponential_histogram: ExponentialHistogram = exponential_histogram + self._summary: Summary = summary + self._metadata: List[KeyValue] = metadata + + def calculate_size(self) -> int: + size = 0 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.description: + v = self.description.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self.unit: + v = self.unit.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._gauge is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._gauge._get_size()) + + self._gauge._get_size() + ) + if self._sum is not None: + size += ( + len(b":") + + Varint.size_varint_u32(self._sum._get_size()) + + self._sum._get_size() + ) + if self._histogram is not None: + size += ( + len(b"J") + + Varint.size_varint_u32(self._histogram._get_size()) + + self._histogram._get_size() + ) + if self._exponential_histogram is not None: + size += ( + len(b"R") + + Varint.size_varint_u32(self._exponential_histogram._get_size()) + + self._exponential_histogram._get_size() + ) + if self._summary is not None: + size += ( + len(b"Z") + + Varint.size_varint_u32(self._summary._get_size()) + + self._summary._get_size() + ) + if self._metadata: + size += sum( + message._get_size() + + len(b"b") + + Varint.size_varint_u32(message._get_size()) + for message in self._metadata + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.name: + v = self.name.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.description: + v = self.description.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self.unit: + v = self.unit.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self._gauge is not None: + out += b"*" + Varint.write_varint_u32(out, self._gauge._get_size()) + self._gauge.write_to(out) + if self._sum is not None: + out += b":" + Varint.write_varint_u32(out, self._sum._get_size()) + self._sum.write_to(out) + if self._histogram is not None: + out += b"J" + Varint.write_varint_u32(out, self._histogram._get_size()) + self._histogram.write_to(out) + if self._exponential_histogram is not None: + out += b"R" + Varint.write_varint_u32(out, self._exponential_histogram._get_size()) + self._exponential_histogram.write_to(out) + if self._summary is not None: + out += b"Z" + Varint.write_varint_u32(out, self._summary._get_size()) + self._summary.write_to(out) + if self._metadata: + for v in self._metadata: + out += b"b" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class Gauge(MessageMarshaler): + @property + def data_points(self) -> List[NumberDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + def __init__( + self, + data_points: List[NumberDataPoint] = None, + ): + self._data_points: List[NumberDataPoint] = data_points + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class Sum(MessageMarshaler): + @property + def data_points(self) -> List[NumberDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + is_monotonic: bool + + def __init__( + self, + data_points: List[NumberDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + is_monotonic: bool = False, + ): + self._data_points: List[NumberDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + self.is_monotonic: bool = is_monotonic + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + if self.is_monotonic: + size += len(b"\x18") + 1 + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + if self.is_monotonic: + out += b"\x18" + Varint.write_varint_u32(out, 1 if self.is_monotonic else 0) + + +class Histogram(MessageMarshaler): + @property + def data_points(self) -> List[HistogramDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + + def __init__( + self, + data_points: List[HistogramDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + ): + self._data_points: List[HistogramDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + + +class ExponentialHistogram(MessageMarshaler): + @property + def data_points(self) -> List[ExponentialHistogramDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + + def __init__( + self, + data_points: List[ExponentialHistogramDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + ): + self._data_points: List[ExponentialHistogramDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + + +class Summary(MessageMarshaler): + @property + def data_points(self) -> List[SummaryDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + def __init__( + self, + data_points: List[SummaryDataPoint] = None, + ): + self._data_points: List[SummaryDataPoint] = data_points + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class NumberDataPoint(MessageMarshaler): + start_time_unix_nano: int + time_unix_nano: int + as_double: float + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + as_int: int + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + as_double: float = None, + exemplars: List[Exemplar] = None, + as_int: int = None, + attributes: List[KeyValue] = None, + flags: int = 0, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.as_double: float = as_double + self._exemplars: List[Exemplar] = exemplars + self.as_int: int = as_int + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.as_double is not None: + size += len(b"!") + 8 + if self._exemplars: + size += sum( + message._get_size() + + len(b"*") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self.as_int is not None: + size += len(b"1") + 8 + if self._attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"@") + Varint.size_varint_u32(self.flags) + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[int]: + if self._bucket_counts is None: + self._bucket_counts = list() + return self._bucket_counts + + @property + def explicit_bounds(self) -> List[float]: + if self._explicit_bounds is None: + self._explicit_bounds = list() + return self._explicit_bounds + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + min: float + max: float + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = None, + bucket_counts: List[int] = None, + explicit_bounds: List[float] = None, + exemplars: List[Exemplar] = None, + attributes: List[KeyValue] = None, + flags: int = 0, + min: float = None, + max: float = None, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self._bucket_counts: List[int] = bucket_counts + self._explicit_bounds: List[float] = explicit_bounds + self._exemplars: List[Exemplar] = exemplars + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + self.min: float = min + self.max: float = max + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum is not None: + size += len(b")") + 8 + if self._bucket_counts: + size += ( + len(b"2") + + len(self._bucket_counts) * 8 + + Varint.size_varint_u32(len(self._bucket_counts) * 8) + ) + if self._explicit_bounds: + size += ( + len(b":") + + len(self._explicit_bounds) * 8 + + Varint.size_varint_u32(len(self._explicit_bounds) * 8) + ) + if self._exemplars: + size += sum( + message._get_size() + + len(b"B") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self._attributes: + size += sum( + message._get_size() + + len(b"J") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"P") + Varint.size_varint_u32(self.flags) + if self.min is not None: + size += len(b"Y") + 8 + if self.max is not None: + size += len(b"a") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + start_time_unix_nano: int + time_unix_nano: int + count: int + sum: float + scale: int + zero_count: int + + @property + def positive(self) -> ExponentialHistogramDataPoint.Buckets: + if self._positive is None: + self._positive = ExponentialHistogramDataPoint.Buckets() + return self._positive + + @property + def negative(self) -> ExponentialHistogramDataPoint.Buckets: + if self._negative is None: + self._negative = ExponentialHistogramDataPoint.Buckets() + return self._negative + + flags: int + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + min: float + max: float + zero_threshold: float + + def __init__( + self, + attributes: List[KeyValue] = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = None, + scale: int = 0, + zero_count: int = 0, + positive: ExponentialHistogramDataPoint.Buckets = None, + negative: ExponentialHistogramDataPoint.Buckets = None, + flags: int = 0, + exemplars: List[Exemplar] = None, + min: float = None, + max: float = None, + zero_threshold: float = 0.0, + ): + self._attributes: List[KeyValue] = attributes + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self.scale: int = scale + self.zero_count: int = zero_count + self._positive: ExponentialHistogramDataPoint.Buckets = positive + self._negative: ExponentialHistogramDataPoint.Buckets = negative + self.flags: int = flags + self._exemplars: List[Exemplar] = exemplars + self.min: float = min + self.max: float = max + self.zero_threshold: float = zero_threshold + + def calculate_size(self) -> int: + size = 0 + if self._attributes: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum is not None: + size += len(b")") + 8 + if self.scale: + size += len(b"0") + Varint.size_varint_s32(self.scale) + if self.zero_count: + size += len(b"9") + 8 + if self._positive is not None: + size += ( + len(b"B") + + Varint.size_varint_u32(self._positive._get_size()) + + self._positive._get_size() + ) + if self._negative is not None: + size += ( + len(b"J") + + Varint.size_varint_u32(self._negative._get_size()) + + self._negative._get_size() + ) + if self.flags: + size += len(b"P") + Varint.size_varint_u32(self.flags) + if self._exemplars: + size += sum( + message._get_size() + + len(b"Z") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self.min is not None: + size += len(b"a") + 8 + if self.max is not None: + size += len(b"i") + 8 + if self.zero_threshold: + size += len(b"q") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self._attributes: + for v in self._attributes: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[int]: + if self._bucket_counts is None: + self._bucket_counts = list() + return self._bucket_counts + + def __init__( + self, + offset: int = 0, + bucket_counts: List[int] = None, + ): + self.offset: int = offset + self._bucket_counts: List[int] = bucket_counts + + def calculate_size(self) -> int: + size = 0 + if self.offset: + size += len(b"\x08") + Varint.size_varint_s32(self.offset) + if self._bucket_counts: + s = sum( + Varint.size_varint_u64(uint64) for uint64 in self._bucket_counts + ) + self.marshaler_cache[b"\x12"] = s + size += len(b"\x12") + s + Varint.size_varint_u32(s) + return size + + def write_to(self, out: bytearray) -> None: + if self.offset: + out += b"\x08" + Varint.write_varint_s32(out, self.offset) + if self._bucket_counts: + out += b"\x12" + Varint.write_varint_u32(out, self.marshaler_cache[b"\x12"]) + for v in self._bucket_counts: + Varint.write_varint_u64(out, v) + + +class SummaryDataPoint(MessageMarshaler): + start_time_unix_nano: int + time_unix_nano: int + count: int + sum: float + + @property + def quantile_values(self) -> List[SummaryDataPoint.ValueAtQuantile]: + if self._quantile_values is None: + self._quantile_values = list() + return self._quantile_values + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = 0.0, + quantile_values: List[SummaryDataPoint.ValueAtQuantile] = None, + attributes: List[KeyValue] = None, + flags: int = 0, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self._quantile_values: List[SummaryDataPoint.ValueAtQuantile] = quantile_values + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum: + size += len(b")") + 8 + if self._quantile_values: + size += sum( + message._get_size() + + len(b"2") + + Varint.size_varint_u32(message._get_size()) + for message in self._quantile_values + ) + if self._attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"@") + Varint.size_varint_u32(self.flags) + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" int: + size = 0 + if self.quantile: + size += len(b"\t") + 8 + if self.value: + size += len(b"\x11") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.quantile: + out += b"\t" + out += struct.pack(" List[KeyValue]: + if self._filtered_attributes is None: + self._filtered_attributes = list() + return self._filtered_attributes + + def __init__( + self, + time_unix_nano: int = 0, + as_double: float = None, + span_id: bytes = b"", + trace_id: bytes = b"", + as_int: int = None, + filtered_attributes: List[KeyValue] = None, + ): + self.time_unix_nano: int = time_unix_nano + self.as_double: float = as_double + self.span_id: bytes = span_id + self.trace_id: bytes = trace_id + self.as_int: int = as_int + self._filtered_attributes: List[KeyValue] = filtered_attributes + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\x11") + 8 + if self.as_double is not None: + size += len(b"\x19") + 8 + if self.span_id: + size += ( + len(b'"') + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_id: + size += ( + len(b"*") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.as_int is not None: + size += len(b"1") + 8 + if self._filtered_attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._filtered_attributes + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\x11" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self._attributes: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"\x10") + Varint.size_varint_u32(self.dropped_attributes_count) + return size + + def write_to(self, out: bytearray) -> None: + if self._attributes: + for v in self._attributes: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b"\x10" + Varint.write_varint_u32(out, self.dropped_attributes_count) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py new file mode 100644 index 0000000..ce957f6 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py @@ -0,0 +1,597 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/trace/v1/trace.proto + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import * +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class SpanFlags(Enum): + SPAN_FLAGS_DO_NOT_USE = 0 + SPAN_FLAGS_TRACE_FLAGS_MASK = 255 + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 256 + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 512 + + +class TracesData(MessageMarshaler): + @property + def resource_spans(self) -> List[ResourceSpans]: + if self._resource_spans is None: + self._resource_spans = list() + return self._resource_spans + + def __init__( + self, + resource_spans: List[ResourceSpans] = None, + ): + self._resource_spans: List[ResourceSpans] = resource_spans + + def calculate_size(self) -> int: + size = 0 + if self._resource_spans: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_spans + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_spans: + for v in self._resource_spans: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceSpans(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_spans(self) -> List[ScopeSpans]: + if self._scope_spans is None: + self._scope_spans = list() + return self._scope_spans + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_spans: List[ScopeSpans] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_spans: List[ScopeSpans] = scope_spans + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_spans: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_spans + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_spans: + for v in self._scope_spans: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeSpans(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def spans(self) -> List[Span]: + if self._spans is None: + self._spans = list() + return self._spans + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + spans: List[Span] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._spans: List[Span] = spans + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._spans: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._spans + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._spans: + for v in self._spans: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class Span(MessageMarshaler): + trace_id: bytes + span_id: bytes + trace_state: str + parent_span_id: bytes + name: str + kind: Span.SpanKind + start_time_unix_nano: int + end_time_unix_nano: int + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + @property + def events(self) -> List[Span.Event]: + if self._events is None: + self._events = list() + return self._events + + dropped_events_count: int + + @property + def links(self) -> List[Span.Link]: + if self._links is None: + self._links = list() + return self._links + + dropped_links_count: int + + @property + def status(self) -> Status: + if self._status is None: + self._status = Status() + return self._status + + flags: int + + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + parent_span_id: bytes = b"", + name: str = "", + kind: Span.SpanKind = 0, + start_time_unix_nano: int = 0, + end_time_unix_nano: int = 0, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + events: List[Span.Event] = None, + dropped_events_count: int = 0, + links: List[Span.Link] = None, + dropped_links_count: int = 0, + status: Status = None, + flags: int = 0, + ): + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.trace_state: str = trace_state + self.parent_span_id: bytes = parent_span_id + self.name: str = name + self.kind: Span.SpanKind = kind + self.start_time_unix_nano: int = start_time_unix_nano + self.end_time_unix_nano: int = end_time_unix_nano + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self._events: List[Span.Event] = events + self.dropped_events_count: int = dropped_events_count + self._links: List[Span.Link] = links + self.dropped_links_count: int = dropped_links_count + self._status: Status = status + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.trace_id: + size += ( + len(b"\n") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"\x12") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_state: + v = self.trace_state.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self.parent_span_id: + size += ( + len(b'"') + + Varint.size_varint_u32(len(self.parent_span_id)) + + len(self.parent_span_id) + ) + if self.name: + v = self.name.encode("utf-8") + size += len(b"*") + Varint.size_varint_u32(len(v)) + len(v) + if self.kind: + v = self.kind + if not isinstance(v, int): + v = v.value + size += len(b"0") + Varint.size_varint_u32(v) + if self.start_time_unix_nano: + size += len(b"9") + 8 + if self.end_time_unix_nano: + size += len(b"A") + 8 + if self._attributes: + size += sum( + message._get_size() + + len(b"J") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"P") + Varint.size_varint_u32(self.dropped_attributes_count) + if self._events: + size += sum( + message._get_size() + + len(b"Z") + + Varint.size_varint_u32(message._get_size()) + for message in self._events + ) + if self.dropped_events_count: + size += len(b"`") + Varint.size_varint_u32(self.dropped_events_count) + if self._links: + size += sum( + message._get_size() + + len(b"j") + + Varint.size_varint_u32(message._get_size()) + for message in self._links + ) + if self.dropped_links_count: + size += len(b"p") + Varint.size_varint_u32(self.dropped_links_count) + if self._status is not None: + size += ( + len(b"z") + + Varint.size_varint_u32(self._status._get_size()) + + self._status._get_size() + ) + if self.flags: + size += len(b"\x85\x01") + 4 + return size + + def write_to(self, out: bytearray) -> None: + if self.trace_id: + out += b"\n" + Varint.write_varint_u32(out, len(self.trace_id)) + out += self.trace_id + if self.span_id: + out += b"\x12" + Varint.write_varint_u32(out, len(self.span_id)) + out += self.span_id + if self.trace_state: + v = self.trace_state.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self.parent_span_id: + out += b'"' + Varint.write_varint_u32(out, len(self.parent_span_id)) + out += self.parent_span_id + if self.name: + v = self.name.encode("utf-8") + out += b"*" + Varint.write_varint_u32(out, len(v)) + out += v + if self.kind: + v = self.kind + if not isinstance(v, int): + v = v.value + out += b"0" + Varint.write_varint_u32(out, v) + if self.start_time_unix_nano: + out += b"9" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + time_unix_nano: int = 0, + name: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self.time_unix_nano: int = time_unix_nano + self.name: str = name + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\t") + 8 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b"\x1a") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b" ") + Varint.size_varint_u32( + self.dropped_attributes_count + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\t" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + flags: int + + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + flags: int = 0, + ): + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.trace_state: str = trace_state + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.trace_id: + size += ( + len(b"\n") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"\x12") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_state: + v = self.trace_state.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b'"') + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"(") + Varint.size_varint_u32( + self.dropped_attributes_count + ) + if self.flags: + size += len(b"5") + 4 + return size + + def write_to(self, out: bytearray) -> None: + if self.trace_id: + out += b"\n" + Varint.write_varint_u32(out, len(self.trace_id)) + out += self.trace_id + if self.span_id: + out += b"\x12" + Varint.write_varint_u32(out, len(self.span_id)) + out += self.span_id + if self.trace_state: + v = self.trace_state.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self._attributes: + for v in self._attributes: + out += b'"' + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b"(" + Varint.write_varint_u32(out, self.dropped_attributes_count) + if self.flags: + out += b"5" + out += struct.pack(" int: + size = 0 + if self.message: + v = self.message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self.code: + v = self.code + if not isinstance(v, int): + v = v.value + size += len(b"\x18") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.message: + v = self.message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self.code: + v = self.code + if not isinstance(v, int): + v = v.value + out += b"\x18" + Varint.write_varint_u32(out, v) + + class StatusCode(Enum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 diff --git a/src/snowflake/telemetry/_internal/serialize/__init__.py b/src/snowflake/telemetry/_internal/serialize/__init__.py new file mode 100644 index 0000000..bb4d92a --- /dev/null +++ b/src/snowflake/telemetry/_internal/serialize/__init__.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import struct +from enum import IntEnum +from typing import List, Union, Dict, Any + +# Alias Enum to IntEnum +Enum = IntEnum + +# Static class to handle varint encoding +# There is code duplication for performance reasons +# https://developers.google.com/protocol-buffers/docs/encoding#varints +class Varint: + @staticmethod + def size_varint_u32(value: int) -> int: + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + size_varint_u64 = size_varint_u32 + + @staticmethod + def size_varint_i32(value: int) -> int: + value = value + (1 << 32) if value < 0 else value + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + @staticmethod + def size_varint_i64(value: int) -> int: + value = value + (1 << 64) if value < 0 else value + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + @staticmethod + def size_varint_s32(value: int) -> int: + value = value << 1 if value >= 0 else (value << 1) ^ (~0) + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + size_varint_s64 = size_varint_s32 + + @staticmethod + def write_varint_u32(out: bytearray, value: int) -> None: + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + write_varint_u64 = write_varint_u32 + + @staticmethod + def write_varint_i32(out: bytearray, value: int) -> None: + value = value + (1 << 32) if value < 0 else value + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + @staticmethod + def write_varint_i64(out: bytearray, value: int) -> None: + value = value + (1 << 64) if value < 0 else value + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + @staticmethod + def write_varint_s32(out: bytearray, value: int) -> None: + value = value << 1 if value >= 0 else (value << 1) ^ (~0) + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + write_varint_s64 = write_varint_s32 + +# Base class for all custom messages +class MessageMarshaler: + # There is a high overhead for creating an empty dict + # For this reason, the cache dict is lazily initialized + @property + def marshaler_cache(self) -> Dict[bytes, Any]: + if not hasattr(self, "_marshaler_cache"): + self._marshaler_cache = {} + return self._marshaler_cache + + def write_to(self, out: bytearray) -> None: + ... + + def calculate_size(self) -> int: + ... + + def _get_size(self) -> int: + if not hasattr(self, "_size"): + self._size = self.calculate_size() + return self._size + + def SerializeToString(self) -> bytes: + # size MUST be calculated before serializing since some preprocessing is done + self._get_size() + stream = bytearray() + self.write_to(stream) + return bytes(stream) + + def __bytes__(self) -> bytes: + return self.SerializeToString() + + # The following size and serialize functions may be inlined by the code generator + # The following strings are replaced by the code generator for inlining: + # - TAG + # - FIELD_ATTR + + def size_bool(self, TAG: bytes, _) -> int: + return len(TAG) + 1 + + def size_enum(self, TAG: bytes, FIELD_ATTR: Union[Enum, int]) -> int: + v = FIELD_ATTR + if not isinstance(v, int): + v = v.value + return len(TAG) + Varint.size_varint_u32(v) + + def size_uint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR) + + def size_uint64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u64(FIELD_ATTR) + + def size_sint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_s32(FIELD_ATTR) + + def size_sint64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_s64(FIELD_ATTR) + + def size_int32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_i32(FIELD_ATTR) + + def size_int64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_i64(FIELD_ATTR) + + def size_float(self, TAG: bytes, FIELD_ATTR: float) -> int: + return len(TAG) + 4 + + def size_double(self, TAG: bytes, FIELD_ATTR: float) -> int: + return len(TAG) + 8 + + def size_fixed32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 4 + + def size_fixed64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 8 + + def size_sfixed32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 4 + + def size_sfixed64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 8 + + def size_bytes(self, TAG: bytes, FIELD_ATTR: bytes) -> int: + return len(TAG) + Varint.size_varint_u32(len(FIELD_ATTR)) + len(FIELD_ATTR) + + def size_string(self, TAG: bytes, FIELD_ATTR: str) -> int: + v = FIELD_ATTR.encode("utf-8") + return len(TAG) + Varint.size_varint_u32(len(v)) + len(v) + + def size_message(self, TAG: bytes, FIELD_ATTR: MessageMarshaler) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR._get_size()) + FIELD_ATTR._get_size() + + def size_repeated_message(self, TAG: bytes, FIELD_ATTR: List[MessageMarshaler]) -> int: + return sum(message._get_size() + len(TAG) + Varint.size_varint_u32(message._get_size()) for message in FIELD_ATTR) + + def size_repeated_double(self, TAG: bytes, FIELD_ATTR: List[float]): + return len(TAG) + len(FIELD_ATTR) * 8 + Varint.size_varint_u32(len(FIELD_ATTR) * 8) + + def size_repeated_fixed64(self, TAG: bytes, FIELD_ATTR: List[int]): + return len(TAG) + len(FIELD_ATTR) * 8 + Varint.size_varint_u32(len(FIELD_ATTR) * 8) + + def size_repeated_uint64(self, TAG: bytes, FIELD_ATTR: List[int]): + s = sum(Varint.size_varint_u64(uint64) for uint64 in FIELD_ATTR) + self.marshaler_cache[TAG] = s + return len(TAG) + s + Varint.size_varint_u32(s) + + def serialize_bool(self, out: bytearray, TAG: bytes, FIELD_ATTR: bool) -> None: + out += TAG + Varint.write_varint_u32(out, 1 if FIELD_ATTR else 0) + + def serialize_enum(self, out: bytearray, TAG: bytes, FIELD_ATTR: Union[Enum, int]) -> None: + v = FIELD_ATTR + if not isinstance(v, int): + v = v.value + out += TAG + Varint.write_varint_u32(out, v) + + def serialize_uint32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_u32(out, FIELD_ATTR) + + def serialize_uint64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_u64(out, FIELD_ATTR) + + def serialize_sint32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_s32(out, FIELD_ATTR) + + def serialize_sint64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_s64(out, FIELD_ATTR) + + def serialize_int32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_i32(out, FIELD_ATTR) + + def serialize_int64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_i64(out, FIELD_ATTR) + + def serialize_fixed32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR)) + out += FIELD_ATTR + + def serialize_string(self, out: bytearray, TAG: bytes, FIELD_ATTR: str) -> None: + v = FIELD_ATTR.encode("utf-8") + out += TAG + Varint.write_varint_u32(out, len(v)) + out += v + + def serialize_message(self, out: bytearray, TAG: bytes, FIELD_ATTR: MessageMarshaler) -> None: + out += TAG + Varint.write_varint_u32(out, FIELD_ATTR._get_size()) + FIELD_ATTR.write_to(out) + + def serialize_repeated_message(self, out: bytearray, TAG: bytes, FIELD_ATTR: List[MessageMarshaler]) -> None: + for v in FIELD_ATTR: + out += TAG + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + def serialize_repeated_double(self, out: bytearray, TAG: bytes, FIELD_ATTR: List[float]) -> None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR) * 8) + for v in FIELD_ATTR: + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR) * 8) + for v in FIELD_ATTR: + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, self.marshaler_cache[TAG]) + for v in FIELD_ATTR: + Varint.write_varint_u64(out, v) + diff --git a/tests/snowflake-telemetry-test-utils/setup.py b/tests/snowflake-telemetry-test-utils/setup.py index 4f78b56..0511b8e 100644 --- a/tests/snowflake-telemetry-test-utils/setup.py +++ b/tests/snowflake-telemetry-test-utils/setup.py @@ -17,6 +17,11 @@ install_requires=[ "pytest >= 7.0.0", "snowflake-telemetry-python == 0.6.0.dev", + "Jinja2 == 3.1.4", + "grpcio-tools >= 1.62.3", + "black >= 24.1.0", + "isort >= 5.12.0", + "hypothesis >= 6.0.0", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_proto_serialization.py b/tests/test_proto_serialization.py new file mode 100644 index 0000000..5e3458f --- /dev/null +++ b/tests/test_proto_serialization.py @@ -0,0 +1,501 @@ +from __future__ import annotations + +from typing import ( + Any, + Dict, + List, + Mapping, +) +import unittest +import hypothesis +import hypothesis.control as hc +import hypothesis.strategies as st + +import opentelemetry.proto.logs.v1.logs_pb2 as logs_pb2 +import opentelemetry.proto.trace.v1.trace_pb2 as trace_pb2 +import opentelemetry.proto.common.v1.common_pb2 as common_pb2 +import opentelemetry.proto.metrics.v1.metrics_pb2 as metrics_pb2 +import opentelemetry.proto.resource.v1.resource_pb2 as resource_pb2 + +import snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler as logs_sf +import snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler as trace_sf +import snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler as common_sf +import snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler as metrics_sf +import snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler as resource_sf + +# Strategy for generating protobuf types +def nullable(type): return st.one_of(st.none(), type) +def pb_uint32(): return nullable(st.integers(min_value=0, max_value=2**32-1)) +def pb_uint64(): return nullable(st.integers(min_value=0, max_value=2**64-1)) +def pb_int32(): return nullable(st.integers(min_value=-2**31, max_value=2**31-1)) +def pb_int64(): return nullable(st.integers(min_value=-2**63, max_value=2**63-1)) +def pb_sint32(): return nullable(st.integers(min_value=-2**31, max_value=2**31-1)) +def pb_sint64(): return nullable(st.integers(min_value=-2**63, max_value=2**63-1)) +def pb_float(): return nullable(st.floats(allow_nan=False, allow_infinity=False, width=32)) +def pb_double(): return nullable(st.floats(allow_nan=False, allow_infinity=False, width=64)) +def draw_pb_double(draw): + # -0.0 is an edge case that is not handled by the custom serialization library + double = draw(pb_double()) + hc.assume(str(double) != "-0.0") + return double +def pb_fixed64(): return pb_uint64() +def pb_fixed32(): return pb_uint32() +def pb_sfixed64(): return pb_int64() +def pb_sfixed32(): return pb_int32() +def pb_bool(): return nullable(st.booleans()) +def pb_string(): return nullable(st.text(max_size=20)) +def pb_bytes(): return nullable(st.binary(max_size=20)) +def draw_pb_enum(draw, enum): + # Sample int val of enum, will be converted to member in encode_recurse + # Sample from pb2 values as it is the source of truth + return draw(nullable(st.sampled_from([member for member in enum.values()]))) +def pb_repeated(type): return nullable(st.lists(type, max_size=3)) # limit the size of the repeated field to speed up testing +def pb_span_id(): return nullable(st.binary(min_size=8, max_size=8)) +def pb_trace_id(): return nullable(st.binary(min_size=16, max_size=16)) +# For drawing oneof fields +# call with pb_oneof(draw, field1=pb_type1_callable, field2=pb_type2_callable, ...) +def pb_oneof(draw, **kwargs): + n = len(kwargs) + r = draw(st.integers(min_value=0, max_value=n-1)) + k, v = list(kwargs.items())[r] + return {k: draw(v())} +def pb_message(type): + return nullable(type) + +SF = "_sf" +PB = "_pb2" + +# Strategies for generating opentelemetry-proto types +@st.composite +def instrumentation_scope(draw): + return { + SF: common_sf.InstrumentationScope, + PB: common_pb2.InstrumentationScope, + "name": draw(pb_string()), + "version": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + +@st.composite +def resource(draw): + return { + SF: resource_sf.Resource, + PB: resource_pb2.Resource, + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + +@st.composite +def any_value(draw): + return { + SF: common_sf.AnyValue, + PB: common_pb2.AnyValue, + **pb_oneof( + draw, + string_value=pb_string, + bool_value=pb_bool, + int_value=pb_int64, + double_value=pb_double, + array_value=array_value, + kvlist_value=key_value_list, + bytes_value=pb_bytes, + ), + } + +@st.composite +def array_value(draw): + return { + SF: common_sf.ArrayValue, + PB: common_pb2.ArrayValue, + "values": draw(pb_repeated(any_value())), + } + +@st.composite +def key_value(draw): + return { + SF: common_sf.KeyValue, + PB: common_pb2.KeyValue, + "key": draw(pb_string()), + "value": draw(any_value()), + } + +@st.composite +def key_value_list(draw): + return { + SF: common_sf.KeyValueList, + PB: common_pb2.KeyValueList, + "values": draw(pb_repeated(key_value())), + } + +@st.composite +def logs_data(draw): + @st.composite + def log_record(draw): + return { + SF: logs_sf.LogRecord, + PB: logs_pb2.LogRecord, + "time_unix_nano": draw(pb_fixed64()), + "observed_time_unix_nano": draw(pb_fixed64()), + "severity_number": draw_pb_enum(draw, logs_pb2.SeverityNumber), + "severity_text": draw(pb_string()), + "body": draw(pb_message(any_value())), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + "span_id": draw(pb_span_id()), + "trace_id": draw(pb_trace_id()), + } + + @st.composite + def scope_logs(draw): + return { + SF: logs_sf.ScopeLogs, + PB: logs_pb2.ScopeLogs, + "scope": draw(pb_message(instrumentation_scope())), + "log_records": draw(pb_repeated(log_record())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_logs(draw): + return { + SF: logs_sf.ResourceLogs, + PB: logs_pb2.ResourceLogs, + "resource": draw(pb_message(resource())), + "scope_logs": draw(pb_repeated(scope_logs())), + "schema_url": draw(pb_string()), + } + + return { + SF: logs_sf.LogsData, + PB: logs_pb2.LogsData, + "resource_logs": draw(pb_repeated(resource_logs())), + } + +@st.composite +def traces_data(draw): + @st.composite + def event(draw): + return { + SF: trace_sf.Span.Event, + PB: trace_pb2.Span.Event, + "time_unix_nano": draw(pb_fixed64()), + "name": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + + @st.composite + def link(draw): + return { + SF: trace_sf.Span.Link, + PB: trace_pb2.Span.Link, + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + } + + @st.composite + def status(draw): + return { + SF: trace_sf.Status, + PB: trace_pb2.Status, + "code": draw_pb_enum(draw, trace_pb2.Status.StatusCode), + "message": draw(pb_string()), + } + + @st.composite + def span(draw): + return { + SF: trace_sf.Span, + PB: trace_pb2.Span, + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "parent_span_id": draw(pb_span_id()), + "name": draw(pb_string()), + "kind": draw_pb_enum(draw, trace_pb2.Span.SpanKind), + "start_time_unix_nano": draw(pb_fixed64()), + "end_time_unix_nano": draw(pb_fixed64()), + "attributes": draw(pb_repeated(key_value())), + "events": draw(pb_repeated(event())), + "links": draw(pb_repeated(link())), + "status": draw(pb_message(status())), + "dropped_attributes_count": draw(pb_uint32()), + "dropped_events_count": draw(pb_uint32()), + "dropped_links_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + } + + @st.composite + def scope_spans(draw): + return { + SF: trace_sf.ScopeSpans, + PB: trace_pb2.ScopeSpans, + "scope": draw(pb_message(instrumentation_scope())), + "spans": draw(pb_repeated(span())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_spans(draw): + return { + SF: trace_sf.ResourceSpans, + PB: trace_pb2.ResourceSpans, + "resource": draw(pb_message(resource())), + "scope_spans": draw(pb_repeated(scope_spans())), + "schema_url": draw(pb_string()), + } + + return { + SF: trace_sf.TracesData, + PB: trace_pb2.TracesData, + "resource_spans": draw(pb_repeated(resource_spans())), + } + +@st.composite +def metrics_data(draw): + @st.composite + def exemplar(draw): + return { + SF: metrics_sf.Exemplar, + PB: metrics_pb2.Exemplar, + **pb_oneof( + draw, + as_double=pb_double, + as_int=pb_sfixed64, + ), + "time_unix_nano": draw(pb_fixed64()), + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "filtered_attributes": draw(pb_repeated(key_value())), + } + + @st.composite + def value_at_quantile(draw): + return { + SF: metrics_sf.SummaryDataPoint.ValueAtQuantile, + PB: metrics_pb2.SummaryDataPoint.ValueAtQuantile, + "quantile": draw_pb_double(draw), + "value": draw_pb_double(draw), + } + + @st.composite + def summary_data_point(draw): + return { + SF: metrics_sf.SummaryDataPoint, + PB: metrics_pb2.SummaryDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "quantile_values": draw(pb_repeated(value_at_quantile())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + } + + @st.composite + def buckets(draw): + return { + SF: metrics_sf.ExponentialHistogramDataPoint.Buckets, + PB: metrics_pb2.ExponentialHistogramDataPoint.Buckets, + "offset": draw(pb_sint32()), + "bucket_counts": draw(pb_repeated(pb_uint64())), + } + + @st.composite + def exponential_histogram_data_point(draw): + return { + SF: metrics_sf.ExponentialHistogramDataPoint, + PB: metrics_pb2.ExponentialHistogramDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + **pb_oneof( + draw, + positive=buckets, + negative=buckets, + ), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "max": draw_pb_double(draw), + "zero_threshold": draw_pb_double(draw), + } + + @st.composite + def histogram_data_point(draw): + return { + SF: metrics_sf.HistogramDataPoint, + PB: metrics_pb2.HistogramDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "bucket_counts": draw(pb_repeated(pb_uint64())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "explicit_bounds": draw(pb_repeated(pb_double())), + **pb_oneof( + draw, + max=pb_double, + min=pb_double, + ), + } + + @st.composite + def number_data_point(draw): + return { + SF: metrics_sf.NumberDataPoint, + PB: metrics_pb2.NumberDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + **pb_oneof( + draw, + as_int=pb_sfixed64, + as_double=pb_double, + ), + "exemplars": draw(pb_repeated(exemplar())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + } + + @st.composite + def summary(draw): + return { + SF: metrics_sf.Summary, + PB: metrics_pb2.Summary, + "data_points": draw(pb_repeated(summary_data_point())), + } + + @st.composite + def exponential_histogram(draw): + return { + SF: metrics_sf.ExponentialHistogram, + PB: metrics_pb2.ExponentialHistogram, + "data_points": draw(pb_repeated(exponential_histogram_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + } + + @st.composite + def histogram(draw): + return { + SF: metrics_sf.Histogram, + PB: metrics_pb2.Histogram, + "data_points": draw(pb_repeated(histogram_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + } + + @st.composite + def sum(draw): + return { + SF: metrics_sf.Sum, + PB: metrics_pb2.Sum, + "data_points": draw(pb_repeated(number_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + "is_monotonic": draw(pb_bool()), + } + + @st.composite + def gauge(draw): + return { + SF: metrics_sf.Gauge, + PB: metrics_pb2.Gauge, + "data_points": draw(pb_repeated(number_data_point())), + } + + @st.composite + def metric(draw): + return { + SF: metrics_sf.Metric, + PB: metrics_pb2.Metric, + "name": draw(pb_string()), + "description": draw(pb_string()), + "unit": draw(pb_string()), + **pb_oneof( + draw, + gauge=gauge, + sum=sum, + summary=summary, + histogram=histogram, + exponential_histogram=exponential_histogram, + ), + "metadata": draw(pb_repeated(key_value())), + } + + @st.composite + def scope_metrics(draw): + return { + SF: metrics_sf.ScopeMetrics, + PB: metrics_pb2.ScopeMetrics, + "scope": draw(pb_message(instrumentation_scope())), + "metrics": draw(pb_repeated(metric())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_metrics(draw): + return { + SF: metrics_sf.ResourceMetrics, + PB: metrics_pb2.ResourceMetrics, + "resource": draw(pb_message(resource())), + "scope_metrics": draw(pb_repeated(scope_metrics())), + "schema_url": draw(pb_string()), + } + + return { + SF: metrics_sf.MetricsData, + PB: metrics_pb2.MetricsData, + "resource_metrics": draw(pb_repeated(resource_metrics())), + } + + +# Helper functions to recursively encode protobuf types using the generated args +# and the given serialization strategy +def encode_recurse(obj: Dict[str, Any], strategy: str) -> Any: + kwargs = {} + for key, value in obj.items(): + if key in [SF, PB]: + continue + elif value is None: + continue + elif isinstance(value, Mapping): + kwargs[key] = encode_recurse(value, strategy) + elif isinstance(value, List) and value and isinstance(value[0], Mapping): + kwargs[key] = [encode_recurse(v, strategy) for v in value if v is not None] + elif isinstance(value, List): + kwargs[key] = [v for v in value if v is not None] + else: + kwargs[key] = value + return obj[strategy](**kwargs) + +class TestProtoSerialization(unittest.TestCase): + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(logs_data()) + def test_log_data(self, logs_data): + self.assertEqual( + encode_recurse(logs_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(logs_data, SF)) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(traces_data()) + def test_trace_data(self, traces_data): + self.assertEqual( + encode_recurse(traces_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(traces_data, SF)) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(metrics_data()) + def test_metrics_data(self, metrics_data): + self.assertEqual( + encode_recurse(metrics_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(metrics_data, SF)) + ) diff --git a/tests/test_protoc_plugin.py b/tests/test_protoc_plugin.py new file mode 100644 index 0000000..b0508d3 --- /dev/null +++ b/tests/test_protoc_plugin.py @@ -0,0 +1,91 @@ +""" +Test protoc code generator plugin for custom protoc message types +""" +import unittest +import tempfile +import subprocess +import os + +# Import into globals() so generated code string can be compiled +from snowflake.telemetry._internal.serialize import * + +class TestProtocPlugin(unittest.TestCase): + def namespace_serialize_message(self, message_type: str, local_namespace: dict, **kwargs) -> bytes: + assert message_type in local_namespace, f"Message type {message_type} not found in local namespace" + return local_namespace[message_type](**kwargs).SerializeToString() + + def test_protoc_plugin(self): + with tempfile.NamedTemporaryFile(suffix=".proto", mode="w", delete=False) as proto_file: + # Define a simple proto file + proto_file.write( + """syntax = "proto3"; +package opentelemetry.proto.common.v1; + +message AnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +message ArrayValue { + repeated AnyValue values = 1; +} + +message KeyValueList { + repeated KeyValue values = 1; +} + +message KeyValue { + string key = 1; + AnyValue value = 2; +} + +message InstrumentationScope { + string name = 1; + string version = 2; + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} +""" + ) + proto_file.flush() + proto_file.close() + + proto_file_dir = os.path.dirname(proto_file.name) + proto_file_name = os.path.basename(proto_file.name) + + # Run protoc with custom plugin to generate serialization code for messages + result = subprocess.run([ + "python", + "-m", + "grpc_tools.protoc", + "-I", + proto_file_dir, + "--plugin=protoc-gen-custom-plugin=scripts/plugin.py", + f"--custom-plugin_out={tempfile.gettempdir()}", + proto_file_name, + ], capture_output=True) + + # Ensure protoc ran successfully + self.assertEqual(result.returncode, 0) + + generated_code_file_dir = tempfile.gettempdir() + generated_code_file_name = proto_file_name.replace(".proto", "_marshaler.py") + generated_code_file = os.path.join(generated_code_file_dir, generated_code_file_name) + + # Ensure generated code file exists + self.assertTrue(os.path.exists(generated_code_file)) + + # Ensure code can be executed and serializes correctly + with open(generated_code_file, "r") as f: + generated_code = f.read() + local_namespace = {} + eval(compile(generated_code, generated_code_file, "exec"), globals(), local_namespace) + + self.assertEqual(b'\n\x04test', self.namespace_serialize_message("AnyValue", local_namespace, string_value="test")) From 1f5f4a83ec74744a6e98a0abad58554d74a50b96 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Tue, 10 Dec 2024 19:51:31 -0800 Subject: [PATCH 06/11] Update copyright notices on proto serialization files (#45) --- scripts/plugin.py | 13 +++++++++++ scripts/proto_codegen.sh | 2 +- scripts/templates/template.py.jinja2 | 10 +++++++++ .../logs/v1/logs_service_marshaler.py | 22 +++++++++++++++++++ .../metrics/v1/metrics_service_marshaler.py | 22 +++++++++++++++++++ .../trace/v1/trace_service_marshaler.py | 22 +++++++++++++++++++ .../proto/common/v1/common_marshaler.py | 22 +++++++++++++++++++ .../proto/logs/v1/logs_marshaler.py | 22 +++++++++++++++++++ .../proto/metrics/v1/metrics_marshaler.py | 22 +++++++++++++++++++ .../proto/resource/v1/resource_marshaler.py | 22 +++++++++++++++++++ .../proto/trace/v1/trace_marshaler.py | 22 +++++++++++++++++++ .../telemetry/_internal/serialize/__init__.py | 5 ++++- tests/test_protoc_plugin.py | 2 ++ 13 files changed, 206 insertions(+), 2 deletions(-) diff --git a/scripts/plugin.py b/scripts/plugin.py index d01cb00..d3d0c7f 100755 --- a/scripts/plugin.py +++ b/scripts/plugin.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 + from __future__ import annotations import re @@ -27,6 +28,7 @@ INLINE_OPTIMIZATION = True FILE_PATH_PREFIX = "snowflake.telemetry._internal" FILE_NAME_SUFFIX = "_marshaler" +OPENTELEMETRY_PROTO_DIR = os.environ["OPENTELEMETRY_PROTO_DIR"] # Inline utility functions @@ -281,9 +283,19 @@ class FileTemplate: enums: List[EnumTemplate] = field(default_factory=list) imports: List[str] = field(default_factory=list) name: str = "" + preamble: str = "" @staticmethod def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": + # Extract the preamble comment from the proto file + # In the case of the opentelemetry-proto files, the preamble is the license header + # Each line of the preamble is prefixed with "//" + preamble = "" + with open(f'{OPENTELEMETRY_PROTO_DIR}/{descriptor.name}', "r") as f: + line = f.readline() + while line and line.startswith("//"): + preamble += line.replace("//", "#", 1) + line = f.readline() # Extract the import paths for the proto file imports = [] @@ -300,6 +312,7 @@ def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], imports=imports, name=descriptor.name, + preamble=preamble, ) def main(): diff --git a/scripts/proto_codegen.sh b/scripts/proto_codegen.sh index 9db288f..4c7b08b 100755 --- a/scripts/proto_codegen.sh +++ b/scripts/proto_codegen.sh @@ -60,7 +60,7 @@ find opentelemetry/proto/ -regex ".*_marshaler\.py" -exec rm {} + # generate proto code for all protos all_protos=$(find $PROTO_REPO_DIR/ -iname "*.proto") -python -m grpc_tools.protoc \ +OPENTELEMETRY_PROTO_DIR=$PROTO_REPO_DIR python -m grpc_tools.protoc \ -I $PROTO_REPO_DIR \ --plugin=protoc-gen-custom-plugin=$repo_root/scripts/plugin.py \ --custom-plugin_out=. \ diff --git a/scripts/templates/template.py.jinja2 b/scripts/templates/template.py.jinja2 index 9de445f..b85408a 100644 --- a/scripts/templates/template.py.jinja2 +++ b/scripts/templates/template.py.jinja2 @@ -1,5 +1,15 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: {{ file_template.name }} +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +{{ file_template.preamble -}} +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py index 52711d2..e023504 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/collector/logs/v1/logs_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py index 7701775..5357be0 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/collector/metrics/v1/metrics_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py index 2488f6c..d3b9dce 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/collector/trace/v1/trace_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py index 2d25bcc..a1b909e 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/common/v1/common.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py index 18996f0..9900914 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/logs/v1/logs.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics_marshaler.py index ded8193..37fab8e 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/metrics/v1/metrics_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/metrics/v1/metrics.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource_marshaler.py index 7b43f3c..f4c6664 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/resource/v1/resource_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/resource/v1/resource.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py index ce957f6..748f670 100644 --- a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py @@ -1,5 +1,27 @@ # Generated by the protoc compiler with a custom plugin. DO NOT EDIT! # sources: opentelemetry/proto/trace/v1/trace.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. from __future__ import annotations diff --git a/src/snowflake/telemetry/_internal/serialize/__init__.py b/src/snowflake/telemetry/_internal/serialize/__init__.py index bb4d92a..89e2da8 100644 --- a/src/snowflake/telemetry/_internal/serialize/__init__.py +++ b/src/snowflake/telemetry/_internal/serialize/__init__.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. +# + from __future__ import annotations import struct @@ -288,4 +292,3 @@ def serialize_repeated_uint64(self, out: bytearray, TAG: bytes, FIELD_ATTR: List Varint.write_varint_u32(out, self.marshaler_cache[TAG]) for v in FIELD_ATTR: Varint.write_varint_u64(out, v) - diff --git a/tests/test_protoc_plugin.py b/tests/test_protoc_plugin.py index b0508d3..c27d923 100644 --- a/tests/test_protoc_plugin.py +++ b/tests/test_protoc_plugin.py @@ -60,6 +60,8 @@ def test_protoc_plugin(self): proto_file_dir = os.path.dirname(proto_file.name) proto_file_name = os.path.basename(proto_file.name) + os.environ["OPENTELEMETRY_PROTO_DIR"] = proto_file_dir + # Run protoc with custom plugin to generate serialization code for messages result = subprocess.run([ "python", From 196413dfc65b62e45c89e4ff48c7be4ac9ee43c3 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Wed, 11 Dec 2024 08:26:18 -0800 Subject: [PATCH 07/11] Internalize opentelemetry-exporter-otlp-proto-common adapter code (#43) --- .github/workflows/check-vendor.yml | 32 ++ CHANGELOG.md | 1 + anaconda/meta.yaml | 1 - benchmark/benchmark_serialize.py | 86 +++++ benchmark/util.py | 339 +++++++++++++++++ scripts/vendor_otlp_proto_common.sh | 58 +++ setup.py | 1 - .../exporter/otlp/proto/logs/__init__.py | 4 +- .../exporter/otlp/proto/metrics/__init__.py | 4 +- .../exporter/otlp/proto/traces/__init__.py | 4 +- .../exporter/otlp/proto/common/__init__.py | 24 ++ .../otlp/proto/common/_internal/__init__.py | 182 +++++++++ .../common/_internal/_log_encoder/__init__.py | 101 +++++ .../_internal/metrics_encoder/__init__.py | 344 ++++++++++++++++++ .../_internal/trace_encoder/__init__.py | 195 ++++++++++ .../otlp/proto/common/_log_encoder.py | 26 ++ .../otlp/proto/common/metrics_encoder.py | 26 ++ .../exporter/otlp/proto/common/py.typed | 0 .../otlp/proto/common/trace_encoder.py | 26 ++ .../exporter/otlp/proto/common/version.py | 21 ++ tests/snowflake-telemetry-test-utils/setup.py | 2 + tests/test_log_encoder.py | 10 +- tests/test_metrics_encoder.py | 30 +- tests/test_trace_encoder.py | 4 +- tests/test_vendored_exporter_version.py | 11 + 25 files changed, 1503 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/check-vendor.yml create mode 100644 benchmark/benchmark_serialize.py create mode 100644 benchmark/util.py create mode 100755 scripts/vendor_otlp_proto_common.sh create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py create mode 100644 src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py create mode 100644 tests/test_vendored_exporter_version.py diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml new file mode 100644 index 0000000..dc521fb --- /dev/null +++ b/.github/workflows/check-vendor.yml @@ -0,0 +1,32 @@ +# This workflow will delete and regenerate the opentelemetry-exporter-otlp-proto-common code using scripts/vendor_otlp_proto_common.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check OTLP Proto Common Vendored Code + +on: + push: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run vendor script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/exporter/ + ./scripts/vendor_otlp_proto_common.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/vendor_otlp_proto_common.sh"; exit 1; } diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b0510..ae5c320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased * Upgrade OpenTelemetry Python dependencies to version 1.26.0 +* Vendored in adapter code from package opentelemetry-exporter-otlp-proto-common and replaced protobuf dependency with custom vanilla python serialization ## 0.5.0 (2024-07-23) diff --git a/anaconda/meta.yaml b/anaconda/meta.yaml index cd27e14..620a7ee 100644 --- a/anaconda/meta.yaml +++ b/anaconda/meta.yaml @@ -12,7 +12,6 @@ requirements: run: - python - opentelemetry-api ==1.26.0 - - opentelemetry-exporter-otlp-proto-common ==1.26.0 - opentelemetry-sdk ==1.26.0 about: diff --git a/benchmark/benchmark_serialize.py b/benchmark/benchmark_serialize.py new file mode 100644 index 0000000..e931ed6 --- /dev/null +++ b/benchmark/benchmark_serialize.py @@ -0,0 +1,86 @@ +import google_benchmark as benchmark + +from util import get_logs_data, get_metrics_data, get_traces_data, get_logs_data_4MB + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs as pb2_encode_logs +from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics as pb2_encode_metrics +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans as pb2_encode_spans + +""" +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations +------------------------------------------------------------------------------ +test_bm_serialize_logs_data_4MB 730591536 ns 730562298 ns 1 +test_bm_pb2_serialize_logs_data_4MB 702522039 ns 702490893 ns 1 +test_bm_serialize_logs_data 100882 ns 100878 ns 6930 +test_bm_pb2_serialize_logs_data 97112 ns 97109 ns 7195 +test_bm_serialize_metrics_data 114938 ns 114934 ns 6096 +test_bm_pb2_serialize_metrics_data 161849 ns 161845 ns 4324 +test_bm_serialize_traces_data 123977 ns 123973 ns 5633 +test_bm_pb2_serialize_traces_data 131016 ns 131011 ns 5314 +""" + +def sanity_check(): + logs_data = get_logs_data() + metrics_data = get_metrics_data() + traces_data = get_traces_data() + + assert encode_logs(logs_data).SerializeToString() == pb2_encode_logs(logs_data).SerializeToString() + assert encode_metrics(metrics_data).SerializeToString() == pb2_encode_metrics(metrics_data).SerializeToString() + assert encode_spans(traces_data).SerializeToString() == pb2_encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + pb2_encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + pb2_encode_spans(traces_data).SerializeToString() + +if __name__ == "__main__": + sanity_check() + benchmark.main() \ No newline at end of file diff --git a/benchmark/util.py b/benchmark/util.py new file mode 100644 index 0000000..e73bc56 --- /dev/null +++ b/benchmark/util.py @@ -0,0 +1,339 @@ +from typing import Sequence + +from snowflake.telemetry.test.metrictestutil import _generate_gauge, _generate_sum + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + +from opentelemetry._logs import SeverityNumber +from opentelemetry.sdk._logs import LogData, LogLimits, LogRecord + +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Buckets, + ExponentialHistogram, + Histogram, + ExponentialHistogramDataPoint, + HistogramDataPoint, + Metric, + MetricsData, + ResourceMetrics, + ScopeMetrics, +) + +from opentelemetry.sdk.trace import Event, SpanContext, _Span +from opentelemetry.trace import SpanKind, Link, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + + + +def get_logs_data() -> Sequence[LogData]: + log1 = LogData( + log_record=LogRecord( + timestamp=1644650195189786880, + observed_timestamp=1644660000000000000, + trace_id=89564621134313219400156819398935297684, + span_id=1312458408527513268, + trace_flags=TraceFlags(0x01), + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Do not go gentle into that good night. Rage, rage against the dying of the light", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=InstrumentationScope( + "first_name", "first_version" + ), + ) + + log2 = LogData( + log_record=LogRecord( + timestamp=1644650249738562048, + observed_timestamp=1644660000000000000, + trace_id=0, + span_id=0, + trace_flags=TraceFlags.DEFAULT, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Cooper, this is no time for caution!", + resource=Resource({"second_resource": "CASE"}), + attributes={}, + ), + instrumentation_scope=InstrumentationScope( + "second_name", "second_version" + ), + ) + + log3 = LogData( + log_record=LogRecord( + timestamp=1644650427658989056, + observed_timestamp=1644660000000000000, + trace_id=271615924622795969659406376515024083555, + span_id=4242561578944770265, + trace_flags=TraceFlags(0x01), + severity_text="DEBUG", + severity_number=SeverityNumber.DEBUG, + body="To our galaxy", + resource=Resource({"second_resource": "CASE"}), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=None, + ) + + log4 = LogData( + log_record=LogRecord( + timestamp=1644650584292683008, + observed_timestamp=1644660000000000000, + trace_id=212592107417388365804938480559624925555, + span_id=6077757853989569223, + trace_flags=TraceFlags(0x01), + severity_text="INFO", + severity_number=SeverityNumber.INFO, + body="Love is the one thing that transcends time and space", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"filename": "model.py", "func_name": "run_method"}, + ), + instrumentation_scope=InstrumentationScope( + "another_name", "another_version" + ), + ) + + return [log1, log2, log3, log4] + +def get_logs_data_4MB() -> Sequence[LogData]: + out = [] + for _ in range(8000): + out.extend(get_logs_data()) + return out + +HISTOGRAM = Metric( + name="histogram", + description="foo", + unit="s", + data=Histogram( + data_points=[ + HistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=1641946016139533244, + time_unix_nano=1641946016139533244, + count=5, + sum=67, + bucket_counts=[1, 4], + explicit_bounds=[10.0, 20.0], + min=8, + max=18, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) + +EXPONENTIAL_HISTOGRAM = Metric( + name="exponential_histogram", + description="description", + unit="unit", + data=ExponentialHistogram( + data_points=[ + ExponentialHistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=0, + time_unix_nano=1, + count=2, + sum=3, + scale=4, + zero_count=5, + positive=Buckets(offset=6, bucket_counts=[7, 8]), + negative=Buckets(offset=9, bucket_counts=[10, 11]), + flags=12, + min=13.0, + max=14.0, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) +def get_metrics_data() -> MetricsData: + + metrics = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource( + attributes={"a": 1, "b": False}, + schema_url="resource_schema_url", + ), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM, HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="second_name", + version="second_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="third_name", + version="third_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_int", 33)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_double", 2.98)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_int", 9000)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_double", 52.028)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[EXPONENTIAL_HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ) + ], + schema_url="resource_schema_url", + ) + ] + ) + + return metrics + +def get_traces_data() -> Sequence[_Span]: + trace_id = 0x3E0C63257DE34C926F9EFCD03927272E + + base_time = 683647322 * 10**9 # in ns + start_times = ( + base_time, + base_time + 150 * 10**6, + base_time + 300 * 10**6, + base_time + 400 * 10**6, + ) + end_times = ( + start_times[0] + (50 * 10**6), + start_times[1] + (100 * 10**6), + start_times[2] + (200 * 10**6), + start_times[3] + (300 * 10**6), + ) + + parent_span_context = SpanContext( + trace_id, 0x1111111111111111, is_remote=True + ) + + other_context = SpanContext( + trace_id, 0x2222222222222222, is_remote=False + ) + + span1 = _Span( + name="test-span-1", + context=SpanContext( + trace_id, + 0x34BF92DEEFC58C92, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ), + parent=parent_span_context, + events=( + Event( + name="event0", + timestamp=base_time + 50 * 10**6, + attributes={ + "annotation_bool": True, + "annotation_string": "annotation_test", + "key_float": 0.3, + }, + ), + ), + links=( + Link(context=other_context, attributes={"key_bool": True}), + ), + resource=Resource({}, "resource_schema_url"), + ) + span1.start(start_time=start_times[0]) + span1.set_attribute("key_bool", False) + span1.set_attribute("key_string", "hello_world") + span1.set_attribute("key_float", 111.22) + span1.set_status(Status(StatusCode.ERROR, "Example description")) + span1.end(end_time=end_times[0]) + + span2 = _Span( + name="test-span-2", + context=parent_span_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span2.start(start_time=start_times[1]) + span2.end(end_time=end_times[1]) + + span3 = _Span( + name="test-span-3", + context=other_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span3.start(start_time=start_times[2]) + span3.set_attribute("key_string", "hello_world") + span3.end(end_time=end_times[2]) + + span4 = _Span( + name="test-span-4", + context=other_context, + parent=None, + resource=Resource({}, "resource_schema_url"), + instrumentation_scope=InstrumentationScope( + name="name", version="version" + ), + ) + span4.start(start_time=start_times[3]) + span4.end(end_time=end_times[3]) + + return [span1, span2, span3, span4] \ No newline at end of file diff --git a/scripts/vendor_otlp_proto_common.sh b/scripts/vendor_otlp_proto_common.sh new file mode 100755 index 0000000..eae16a2 --- /dev/null +++ b/scripts/vendor_otlp_proto_common.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Vendor in the python code in +# https://github.com/open-telemetry/opentelemetry-python/tree/main/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common +# +# To use, update REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-python repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +REPO_BRANCH_OR_COMMIT="v1.26.0" + +set -e + +REPO_DIR=${REPO_DIR:-"/tmp/opentelemetry-python"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" + +# Clone the proto repo if it doesn't exist +if [ ! -d "$REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-python.git $REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $REPO_DIR + git fetch --all + git checkout $REPO_BRANCH_OR_COMMIT + # pull if REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# Copy the entire file tree from exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/ +# to src/snowflake/telemetry/_internal/opentelemetry/ +cp -r $REPO_DIR/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter opentelemetry/ + +# If MacOS need '' after -i +# Detect the OS (macOS or Linux) +if [[ "$OSTYPE" == "darwin"* ]]; then + SED_CMD="sed -i ''" +else + SED_CMD="sed -i" +fi + +# Replace all the imports strings in the copied python files +# opentelemetry.exporter to snowflake.telemetry._internal.opentelemetry.exporter +# opentelemetry.proto.*_pb2 to snowflake.telemetry._internal.opentelemetry.proto.*_marshaler + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry.exporter/snowflake.telemetry._internal.opentelemetry.exporter/g' {} + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry\.proto\(.*\)_pb2/snowflake.telemetry._internal.opentelemetry.proto\1_marshaler/g' {} + + + +# Add a notice to the top of every file in compliance with Apache 2.0 to indicate that the file has been modified +# https://www.apache.org/licenses/LICENSE-2.0 +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD '14s|^|#\n# This file has been modified from the original source code at\n#\n# https://github.com/open-telemetry/opentelemetry-python/tree/'"$REPO_BRANCH_OR_COMMIT"'\n#\n# by Snowflake Inc.\n|' {} + diff --git a/setup.py b/setup.py index 06b813c..5e2f0e7 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ long_description=LONG_DESCRIPTION, install_requires=[ "opentelemetry-api == 1.26.0", - "opentelemetry-exporter-otlp-proto-common == 1.26.0", "opentelemetry-sdk == 1.26.0", ], packages=find_namespace_packages( diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py index 3214e3e..9361c20 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py @@ -21,10 +21,10 @@ import opentelemetry.sdk.util.instrumentation as otel_instrumentation import opentelemetry.sdk._logs._internal as _logs_internal -from opentelemetry.exporter.otlp.proto.common._log_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import ( encode_logs, ) -from opentelemetry.proto.logs.v1.logs_pb2 import LogsData +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogsData from opentelemetry.sdk.resources import Resource from opentelemetry.sdk._logs import export from opentelemetry.sdk import _logs diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py index 46291d1..91ff137 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py @@ -17,10 +17,10 @@ from typing import Dict import opentelemetry -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) -from opentelemetry.proto.metrics.v1.metrics_pb2 import MetricsData as PB2MetricsData +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler import MetricsData as PB2MetricsData from opentelemetry.sdk.metrics.export import ( AggregationTemporality, MetricExportResult, diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py index 7c877aa..41aa617 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py @@ -16,10 +16,10 @@ import abc import typing -from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import ( encode_spans, ) -from opentelemetry.proto.trace.v1.trace_pb2 import TracesData +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import TracesData from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import ( SpanExportResult, diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py new file mode 100644 index 0000000..c1250a0 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py @@ -0,0 +1,24 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ + +__all__ = ["__version__"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py new file mode 100644 index 0000000..530528f --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -0,0 +1,182 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +import logging +from collections.abc import Sequence +from itertools import count +from typing import ( + Any, + Mapping, + Optional, + List, + Callable, + TypeVar, + Dict, + Iterator, +) + +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + InstrumentationScope as PB2InstrumentationScope, +) +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import AnyValue as PB2AnyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import KeyValue as PB2KeyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + KeyValueList as PB2KeyValueList, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + ArrayValue as PB2ArrayValue, +) +from opentelemetry.sdk.trace import Resource +from opentelemetry.util.types import Attributes + +_logger = logging.getLogger(__name__) + +_TypingResourceT = TypeVar("_TypingResourceT") +_ResourceDataT = TypeVar("_ResourceDataT") + + +def _encode_instrumentation_scope( + instrumentation_scope: InstrumentationScope, +) -> PB2InstrumentationScope: + if instrumentation_scope is None: + return PB2InstrumentationScope() + return PB2InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + + +def _encode_resource(resource: Resource) -> PB2Resource: + return PB2Resource(attributes=_encode_attributes(resource.attributes)) + + +def _encode_value(value: Any) -> PB2AnyValue: + if isinstance(value, bool): + return PB2AnyValue(bool_value=value) + if isinstance(value, str): + return PB2AnyValue(string_value=value) + if isinstance(value, int): + return PB2AnyValue(int_value=value) + if isinstance(value, float): + return PB2AnyValue(double_value=value) + if isinstance(value, Sequence): + return PB2AnyValue( + array_value=PB2ArrayValue(values=[_encode_value(v) for v in value]) + ) + elif isinstance(value, Mapping): + return PB2AnyValue( + kvlist_value=PB2KeyValueList( + values=[_encode_key_value(str(k), v) for k, v in value.items()] + ) + ) + raise Exception(f"Invalid type {type(value)} of value {value}") + + +def _encode_key_value(key: str, value: Any) -> PB2KeyValue: + return PB2KeyValue(key=key, value=_encode_value(value)) + + +def _encode_span_id(span_id: int) -> bytes: + return span_id.to_bytes(length=8, byteorder="big", signed=False) + + +def _encode_trace_id(trace_id: int) -> bytes: + return trace_id.to_bytes(length=16, byteorder="big", signed=False) + + +def _encode_attributes( + attributes: Attributes, +) -> Optional[List[PB2KeyValue]]: + if attributes: + pb2_attributes = [] + for key, value in attributes.items(): + # pylint: disable=broad-exception-caught + try: + pb2_attributes.append(_encode_key_value(key, value)) + except Exception as error: + _logger.exception("Failed to encode key %s: %s", key, error) + else: + pb2_attributes = None + return pb2_attributes + + +def _get_resource_data( + sdk_resource_scope_data: Dict[Resource, _ResourceDataT], + resource_class: Callable[..., _TypingResourceT], + name: str, +) -> List[_TypingResourceT]: + resource_data = [] + + for ( + sdk_resource, + scope_data, + ) in sdk_resource_scope_data.items(): + collector_resource = PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ) + resource_data.append( + resource_class( + **{ + "resource": collector_resource, + "scope_{}".format(name): scope_data.values(), + } + ) + ) + return resource_data + + +def _create_exp_backoff_generator(max_value: int = 0) -> Iterator[int]: + """ + Generates an infinite sequence of exponential backoff values. The sequence starts + from 1 (2^0) and doubles each time (2^1, 2^2, 2^3, ...). If a max_value is specified + and non-zero, the generated values will not exceed this maximum, capping at max_value + instead of growing indefinitely. + + Parameters: + - max_value (int, optional): The maximum value to yield. If 0 or not provided, the + sequence grows without bound. + + Returns: + Iterator[int]: An iterator that yields the exponential backoff values, either uncapped or + capped at max_value. + + Example: + ``` + gen = _create_exp_backoff_generator(max_value=10) + for _ in range(5): + print(next(gen)) + ``` + This will print: + 1 + 2 + 4 + 8 + 10 + + Note: this functionality used to be handled by the 'backoff' package. + """ + for i in count(0): + out = 2**i + yield min(out, max_value) if max_value else out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py new file mode 100644 index 0000000..2f71a17 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py @@ -0,0 +1,101 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +from collections import defaultdict +from typing import Sequence, List + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, + _encode_value, + _encode_attributes, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.logs.v1.logs_service_marshaler import ( + ExportLogsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import ( + ScopeLogs, + ResourceLogs, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogRecord as PB2LogRecord + +from opentelemetry.sdk._logs import LogData + + +def encode_logs(batch: Sequence[LogData]) -> ExportLogsServiceRequest: + return ExportLogsServiceRequest(resource_logs=_encode_resource_logs(batch)) + + +def _encode_log(log_data: LogData) -> PB2LogRecord: + span_id = ( + None + if log_data.log_record.span_id == 0 + else _encode_span_id(log_data.log_record.span_id) + ) + trace_id = ( + None + if log_data.log_record.trace_id == 0 + else _encode_trace_id(log_data.log_record.trace_id) + ) + return PB2LogRecord( + time_unix_nano=log_data.log_record.timestamp, + observed_time_unix_nano=log_data.log_record.observed_timestamp, + span_id=span_id, + trace_id=trace_id, + flags=int(log_data.log_record.trace_flags), + body=_encode_value(log_data.log_record.body), + severity_text=log_data.log_record.severity_text, + attributes=_encode_attributes(log_data.log_record.attributes), + dropped_attributes_count=log_data.log_record.dropped_attributes, + severity_number=log_data.log_record.severity_number.value, + ) + + +def _encode_resource_logs(batch: Sequence[LogData]) -> List[ResourceLogs]: + sdk_resource_logs = defaultdict(lambda: defaultdict(list)) + + for sdk_log in batch: + sdk_resource = sdk_log.log_record.resource + sdk_instrumentation = sdk_log.instrumentation_scope or None + pb2_log = _encode_log(sdk_log) + + sdk_resource_logs[sdk_resource][sdk_instrumentation].append(pb2_log) + + pb2_resource_logs = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_logs.items(): + scope_logs = [] + for sdk_instrumentation, pb2_logs in sdk_instrumentations.items(): + scope_logs.append( + ScopeLogs( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + log_records=pb2_logs, + ) + ) + pb2_resource_logs.append( + ResourceLogs( + resource=_encode_resource(sdk_resource), + scope_logs=scope_logs, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_logs diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py new file mode 100644 index 0000000..9d729cd --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py @@ -0,0 +1,344 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +import logging + +from opentelemetry.sdk.metrics.export import ( + MetricExporter, +) +from opentelemetry.sdk.metrics.view import Aggregation +from os import environ +from opentelemetry.sdk.metrics import ( + Counter, + Histogram, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, +) +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, +) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.metrics.v1.metrics_service_marshaler import ( + ExportMetricsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1 import metrics_marshaler as pb2 +from opentelemetry.sdk.metrics.export import ( + MetricsData, + Gauge, + Histogram as HistogramType, + Sum, + ExponentialHistogram as ExponentialHistogramType, +) +from typing import Dict +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, +) +from opentelemetry.sdk.metrics.view import ( + ExponentialBucketHistogramAggregation, + ExplicitBucketHistogramAggregation, +) + +_logger = logging.getLogger(__name__) + + +class OTLPMetricExporterMixin: + def _common_configuration( + self, + preferred_temporality: Dict[type, AggregationTemporality] = None, + preferred_aggregation: Dict[type, Aggregation] = None, + ) -> None: + + MetricExporter.__init__( + self, + preferred_temporality=self._get_temporality(preferred_temporality), + preferred_aggregation=self._get_aggregation(preferred_aggregation), + ) + + def _get_temporality( + self, preferred_temporality: Dict[type, AggregationTemporality] + ) -> Dict[type, AggregationTemporality]: + + otel_exporter_otlp_metrics_temporality_preference = ( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, + "CUMULATIVE", + ) + .upper() + .strip() + ) + + if otel_exporter_otlp_metrics_temporality_preference == "DELTA": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.DELTA, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + elif otel_exporter_otlp_metrics_temporality_preference == "LOWMEMORY": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + else: + if otel_exporter_otlp_metrics_temporality_preference != ( + "CUMULATIVE" + ): + _logger.warning( + "Unrecognized OTEL_EXPORTER_METRICS_TEMPORALITY_PREFERENCE" + " value found: " + f"{otel_exporter_otlp_metrics_temporality_preference}, " + "using CUMULATIVE" + ) + instrument_class_temporality = { + Counter: AggregationTemporality.CUMULATIVE, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.CUMULATIVE, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + instrument_class_temporality.update(preferred_temporality or {}) + + return instrument_class_temporality + + def _get_aggregation( + self, + preferred_aggregation: Dict[type, Aggregation], + ) -> Dict[type, Aggregation]: + + otel_exporter_otlp_metrics_default_histogram_aggregation = environ.get( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + "explicit_bucket_histogram", + ) + + if otel_exporter_otlp_metrics_default_histogram_aggregation == ( + "base2_exponential_bucket_histogram" + ): + + instrument_class_aggregation = { + Histogram: ExponentialBucketHistogramAggregation(), + } + + else: + + if otel_exporter_otlp_metrics_default_histogram_aggregation != ( + "explicit_bucket_histogram" + ): + + _logger.warning( + ( + "Invalid value for %s: %s, using explicit bucket " + "histogram aggregation" + ), + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + otel_exporter_otlp_metrics_default_histogram_aggregation, + ) + + instrument_class_aggregation = { + Histogram: ExplicitBucketHistogramAggregation(), + } + + instrument_class_aggregation.update(preferred_aggregation or {}) + + return instrument_class_aggregation + + +def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: + resource_metrics_dict = {} + + for resource_metrics in data.resource_metrics: + + resource = resource_metrics.resource + + # It is safe to assume that each entry in data.resource_metrics is + # associated with an unique resource. + scope_metrics_dict = {} + + resource_metrics_dict[resource] = scope_metrics_dict + + for scope_metrics in resource_metrics.scope_metrics: + + instrumentation_scope = scope_metrics.scope + + # The SDK groups metrics in instrumentation scopes already so + # there is no need to check for existing instrumentation scopes + # here. + pb2_scope_metrics = pb2.ScopeMetrics( + scope=InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + ) + + scope_metrics_dict[instrumentation_scope] = pb2_scope_metrics + + for metric in scope_metrics.metrics: + pb2_metric = pb2.Metric( + name=metric.name, + description=metric.description, + unit=metric.unit, + ) + + if isinstance(metric.data, Gauge): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + pb2_metric.gauge.data_points.append(pt) + + elif isinstance(metric.data, HistogramType): + for data_point in metric.data.data_points: + pt = pb2.HistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + bucket_counts=data_point.bucket_counts, + explicit_bounds=data_point.explicit_bounds, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.histogram.data_points.append(pt) + + elif isinstance(metric.data, Sum): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + # note that because sum is a message type, the + # fields must be set individually rather than + # instantiating a pb2.Sum and setting it once + pb2_metric.sum.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.sum.is_monotonic = metric.data.is_monotonic + pb2_metric.sum.data_points.append(pt) + + elif isinstance(metric.data, ExponentialHistogramType): + for data_point in metric.data.data_points: + + if data_point.positive.bucket_counts: + positive = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.positive.offset, + bucket_counts=data_point.positive.bucket_counts, + ) + else: + positive = None + + if data_point.negative.bucket_counts: + negative = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.negative.offset, + bucket_counts=data_point.negative.bucket_counts, + ) + else: + negative = None + + pt = pb2.ExponentialHistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + scale=data_point.scale, + zero_count=data_point.zero_count, + positive=positive, + negative=negative, + flags=data_point.flags, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.exponential_histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.exponential_histogram.data_points.append(pt) + + else: + _logger.warning( + "unsupported data type %s", + metric.data.__class__.__name__, + ) + continue + + pb2_scope_metrics.metrics.append(pb2_metric) + + resource_data = [] + for ( + sdk_resource, + scope_data, + ) in resource_metrics_dict.items(): + resource_data.append( + pb2.ResourceMetrics( + resource=PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ), + scope_metrics=scope_data.values(), + schema_url=sdk_resource.schema_url, + ) + ) + resource_metrics = resource_data + return ExportMetricsServiceRequest(resource_metrics=resource_metrics) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py new file mode 100644 index 0000000..e269641 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py @@ -0,0 +1,195 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +import logging +from collections import defaultdict +from typing import List, Optional, Sequence + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.trace.v1.trace_service_marshaler import ( + ExportTraceServiceRequest as PB2ExportTraceServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ( + ResourceSpans as PB2ResourceSpans, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ScopeSpans as PB2ScopeSpans +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Span as PB2SPan +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import SpanFlags as PB2SpanFlags +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Status as PB2Status +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import Link, SpanKind +from opentelemetry.trace.span import SpanContext, Status, TraceState + +# pylint: disable=E1101 +_SPAN_KIND_MAP = { + SpanKind.INTERNAL: PB2SPan.SpanKind.SPAN_KIND_INTERNAL, + SpanKind.SERVER: PB2SPan.SpanKind.SPAN_KIND_SERVER, + SpanKind.CLIENT: PB2SPan.SpanKind.SPAN_KIND_CLIENT, + SpanKind.PRODUCER: PB2SPan.SpanKind.SPAN_KIND_PRODUCER, + SpanKind.CONSUMER: PB2SPan.SpanKind.SPAN_KIND_CONSUMER, +} + +_logger = logging.getLogger(__name__) + + +def encode_spans( + sdk_spans: Sequence[ReadableSpan], +) -> PB2ExportTraceServiceRequest: + return PB2ExportTraceServiceRequest( + resource_spans=_encode_resource_spans(sdk_spans) + ) + + +def _encode_resource_spans( + sdk_spans: Sequence[ReadableSpan], +) -> List[PB2ResourceSpans]: + # We need to inspect the spans and group + structure them as: + # + # Resource + # Instrumentation Library + # Spans + # + # First loop organizes the SDK spans in this structure. Protobuf messages + # are not hashable so we stick with SDK data in this phase. + # + # Second loop encodes the data into Protobuf format. + # + sdk_resource_spans = defaultdict(lambda: defaultdict(list)) + + for sdk_span in sdk_spans: + sdk_resource = sdk_span.resource + sdk_instrumentation = sdk_span.instrumentation_scope or None + pb2_span = _encode_span(sdk_span) + + sdk_resource_spans[sdk_resource][sdk_instrumentation].append(pb2_span) + + pb2_resource_spans = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_spans.items(): + scope_spans = [] + for sdk_instrumentation, pb2_spans in sdk_instrumentations.items(): + scope_spans.append( + PB2ScopeSpans( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + spans=pb2_spans, + ) + ) + pb2_resource_spans.append( + PB2ResourceSpans( + resource=_encode_resource(sdk_resource), + scope_spans=scope_spans, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_spans + + +def _span_flags(parent_span_context: Optional[SpanContext]) -> int: + flags = PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK + if parent_span_context and parent_span_context.is_remote: + flags |= PB2SpanFlags.SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK + return flags + + +def _encode_span(sdk_span: ReadableSpan) -> PB2SPan: + span_context = sdk_span.get_span_context() + return PB2SPan( + trace_id=_encode_trace_id(span_context.trace_id), + span_id=_encode_span_id(span_context.span_id), + trace_state=_encode_trace_state(span_context.trace_state), + parent_span_id=_encode_parent_id(sdk_span.parent), + name=sdk_span.name, + kind=_SPAN_KIND_MAP[sdk_span.kind], + start_time_unix_nano=sdk_span.start_time, + end_time_unix_nano=sdk_span.end_time, + attributes=_encode_attributes(sdk_span.attributes), + events=_encode_events(sdk_span.events), + links=_encode_links(sdk_span.links), + status=_encode_status(sdk_span.status), + dropped_attributes_count=sdk_span.dropped_attributes, + dropped_events_count=sdk_span.dropped_events, + dropped_links_count=sdk_span.dropped_links, + flags=_span_flags(sdk_span.parent), + ) + + +def _encode_events( + events: Sequence[Event], +) -> Optional[List[PB2SPan.Event]]: + pb2_events = None + if events: + pb2_events = [] + for event in events: + encoded_event = PB2SPan.Event( + name=event.name, + time_unix_nano=event.timestamp, + attributes=_encode_attributes(event.attributes), + dropped_attributes_count=event.dropped_attributes, + ) + pb2_events.append(encoded_event) + return pb2_events + + +def _encode_links(links: Sequence[Link]) -> Sequence[PB2SPan.Link]: + pb2_links = None + if links: + pb2_links = [] + for link in links: + encoded_link = PB2SPan.Link( + trace_id=_encode_trace_id(link.context.trace_id), + span_id=_encode_span_id(link.context.span_id), + attributes=_encode_attributes(link.attributes), + dropped_attributes_count=link.attributes.dropped, + flags=_span_flags(link.context), + ) + pb2_links.append(encoded_link) + return pb2_links + + +def _encode_status(status: Status) -> Optional[PB2Status]: + pb2_status = None + if status is not None: + pb2_status = PB2Status( + code=status.status_code.value, + message=status.description, + ) + return pb2_status + + +def _encode_trace_state(trace_state: TraceState) -> Optional[str]: + pb2_trace_state = None + if trace_state is not None: + pb2_trace_state = ",".join( + [f"{key}={value}" for key, value in (trace_state.items())] + ) + return pb2_trace_state + + +def _encode_parent_id(context: Optional[SpanContext]) -> Optional[bytes]: + if context: + return _encode_span_id(context.span_id) + return None diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py new file mode 100644 index 0000000..481a853 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal._log_encoder import ( + encode_logs, +) + +__all__ = ["encode_logs"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py new file mode 100644 index 0000000..4d82926 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder import ( + encode_metrics, +) + +__all__ = ["encode_metrics"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py new file mode 100644 index 0000000..bc37212 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( + encode_spans, +) + +__all__ = ["encode_spans"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py new file mode 100644 index 0000000..8ddc9b6 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py @@ -0,0 +1,21 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +__version__ = "1.26.0" diff --git a/tests/snowflake-telemetry-test-utils/setup.py b/tests/snowflake-telemetry-test-utils/setup.py index 0511b8e..6254955 100644 --- a/tests/snowflake-telemetry-test-utils/setup.py +++ b/tests/snowflake-telemetry-test-utils/setup.py @@ -15,6 +15,7 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, install_requires=[ + "opentelemetry-exporter-otlp-proto-common == 1.26.0", "pytest >= 7.0.0", "snowflake-telemetry-python == 0.6.0.dev", "Jinja2 == 3.1.4", @@ -22,6 +23,7 @@ "black >= 24.1.0", "isort >= 5.12.0", "hypothesis >= 6.0.0", + "google-benchmark", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_log_encoder.py b/tests/test_log_encoder.py index a4cfb95..dcf0e4e 100644 --- a/tests/test_log_encoder.py +++ b/tests/test_log_encoder.py @@ -22,7 +22,7 @@ _encode_trace_id, _encode_value, ) -from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( ExportLogsServiceRequest, ) @@ -55,7 +55,7 @@ class TestOTLPLogEncoder(unittest.TestCase): def test_encode(self): sdk_logs, expected_encoding = self.get_test_logs() - self.assertEqual(encode_logs(sdk_logs), expected_encoding) + self.assertEqual(encode_logs(sdk_logs).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_log_exporter(self): sdk_logs, expected_encoding = self.get_test_logs() @@ -69,11 +69,13 @@ def test_proto_log_exporter(self): def test_dropped_attributes_count(self): sdk_logs = self._get_test_logs_dropped_attributes() - encoded_logs = encode_logs(sdk_logs) + encoded_logs = bytes(encode_logs(sdk_logs)) + decoded_logs = PB2LogsData() + decoded_logs.ParseFromString(encoded_logs) self.assertTrue(hasattr(sdk_logs[0].log_record, "dropped_attributes")) self.assertEqual( # pylint:disable=no-member - encoded_logs.resource_logs[0] + decoded_logs.resource_logs[0] .scope_logs[0] .log_records[0] .dropped_attributes_count, diff --git a/tests/test_metrics_encoder.py b/tests/test_metrics_encoder.py index 76464a6..7a473ed 100644 --- a/tests/test_metrics_encoder.py +++ b/tests/test_metrics_encoder.py @@ -15,7 +15,7 @@ # pylint: disable=protected-access import unittest -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( @@ -159,8 +159,8 @@ def test_encode_sum_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -245,8 +245,8 @@ def test_encode_sum_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -328,8 +328,8 @@ def test_encode_gauge_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -411,8 +411,8 @@ def test_encode_gauge_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -502,8 +502,8 @@ def test_encode_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -730,8 +730,8 @@ def test_encode_multiple_scope_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -856,8 +856,8 @@ def test_encode_exponential_histogram(self): ] ) # pylint: disable=protected-access - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() diff --git a/tests/test_trace_encoder.py b/tests/test_trace_encoder.py index 322a521..d8b326e 100644 --- a/tests/test_trace_encoder.py +++ b/tests/test_trace_encoder.py @@ -25,7 +25,7 @@ _SPAN_KIND_MAP, _encode_status, ) -from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( ExportTraceServiceRequest as PB2ExportTraceServiceRequest, ) @@ -67,7 +67,7 @@ class TestOTLPTraceEncoder(unittest.TestCase): def test_encode_spans(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() - self.assertEqual(encode_spans(otel_spans), expected_encoding) + self.assertEqual(encode_spans(otel_spans).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_span_exporter(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() diff --git a/tests/test_vendored_exporter_version.py b/tests/test_vendored_exporter_version.py new file mode 100644 index 0000000..d6509b1 --- /dev/null +++ b/tests/test_vendored_exporter_version.py @@ -0,0 +1,11 @@ +import unittest + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ as vendored_version + +from opentelemetry.sdk.version import __version__ as sdk_version +from opentelemetry.exporter.otlp.proto.common.version import __version__ as exporter_version + +class TestVendoredExporterVersion(unittest.TestCase): + def test_version(self): + self.assertEqual(sdk_version, vendored_version, "SDK version should match vendored version") + self.assertEqual(exporter_version, vendored_version, "Exporter version should match vendored version") From d8f0ac8072f036eb8a8c2ff804a4dfaf0c582544 Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Thu, 12 Dec 2024 10:47:48 -0800 Subject: [PATCH 08/11] Fix check-codegen workflow trigger (#44) --- .github/workflows/check-codegen.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check-codegen.yml b/.github/workflows/check-codegen.yml index 17b4c6f..5f79026 100644 --- a/.github/workflows/check-codegen.yml +++ b/.github/workflows/check-codegen.yml @@ -10,12 +10,14 @@ on: paths: - "scripts/**" - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - "src/snowflake/telemetry/serialize/**" - ".github/workflows/check-codegen.yml" pull_request: branches: [ "main" ] paths: - "scripts/**" - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - "src/snowflake/telemetry/serialize/**" - ".github/workflows/check-codegen.yml" jobs: From d89ed8d336f862a6fdc0ed14d2b5326de5d3a67d Mon Sep 17 00:00:00 2001 From: Jeevan Opel Date: Thu, 12 Dec 2024 15:37:04 -0800 Subject: [PATCH 09/11] Add NOTICE.txt (#46) --- NOTICE.txt | 420 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 NOTICE.txt diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..3382561 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,420 @@ +Snowflake Telemetry Python Library +Copyright (c) 2024 Snowflake Inc. + +The following notices are required by licensors of software used in the Snowflake Telemetry Python Library. + +-------------------------------------------------------------------------------- + +This library includes software which is copied from or derived from the OpenTelemetry Python API and SDK. +OpenTelemetry Python v1.26.0 +https://github.com/open-telemetry/opentelemetry-python + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This library includes software which is derived from the OpenTelemetry Protobuf definitions. +OpenTelemetry Protocol Specification v1.2.0 +https://github.com/open-telemetry/opentelemetry-proto + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 0c5b4faf024997d993f7cd1d00e6ae0cb0bb7d08 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Thu, 12 Dec 2024 16:08:39 -0800 Subject: [PATCH 10/11] Update build test to run on release branches too (#49) --- .github/workflows/build-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 33d6449..daf809f 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -5,9 +5,9 @@ name: Build and Test on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] permissions: contents: read @@ -24,6 +24,7 @@ jobs: - "3.9" - "3.10" - "3.11" + - "3.12" steps: - uses: actions/checkout@v3 From 2cafd147f5d981136d66ff075727acd0d8486aa7 Mon Sep 17 00:00:00 2001 From: Sindy Li Date: Thu, 12 Dec 2024 15:58:40 -0800 Subject: [PATCH 11/11] Add support for Python 3.12 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5e2f0e7..b554ae0 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database", "Topic :: Software Development", "Topic :: Software Development :: Libraries",