diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 33d6449..daf809f 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -5,9 +5,9 @@ name: Build and Test on: push: - branches: [ "main" ] + branches: [ "main", "release/*" ] pull_request: - branches: [ "main" ] + branches: [ "main", "release/*" ] permissions: contents: read @@ -24,6 +24,7 @@ jobs: - "3.9" - "3.10" - "3.11" + - "3.12" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/check-codegen.yml b/.github/workflows/check-codegen.yml new file mode 100644 index 0000000..5f79026 --- /dev/null +++ b/.github/workflows/check-codegen.yml @@ -0,0 +1,38 @@ +# This workflow will delete and regenerate the opentelemetry marshaling code using scripts/proto_codegen.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check Codegen + +on: + push: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - "src/snowflake/telemetry/serialize/**" + - ".github/workflows/check-codegen.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/**" + - "src/snowflake/telemetry/_internal/opentelemetry/proto/**" + - "src/snowflake/telemetry/serialize/**" + - ".github/workflows/check-codegen.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Run codegen script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/proto/ + ./scripts/proto_codegen.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/proto_codegen.sh"; exit 1; } diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml new file mode 100644 index 0000000..dc521fb --- /dev/null +++ b/.github/workflows/check-vendor.yml @@ -0,0 +1,32 @@ +# This workflow will delete and regenerate the opentelemetry-exporter-otlp-proto-common code using scripts/vendor_otlp_proto_common.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check OTLP Proto Common Vendored Code + +on: + push: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run vendor script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/exporter/ + ./scripts/vendor_otlp_proto_common.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/vendor_otlp_proto_common.sh"; exit 1; } diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1a3c6..ae5c320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release History +## Unreleased + +* Upgrade OpenTelemetry Python dependencies to version 1.26.0 +* Vendored in adapter code from package opentelemetry-exporter-otlp-proto-common and replaced protobuf dependency with custom vanilla python serialization + ## 0.5.0 (2024-07-23) * Set empty resource for Python OpenTelemetry config. diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..3382561 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,420 @@ +Snowflake Telemetry Python Library +Copyright (c) 2024 Snowflake Inc. + +The following notices are required by licensors of software used in the Snowflake Telemetry Python Library. + +-------------------------------------------------------------------------------- + +This library includes software which is copied from or derived from the OpenTelemetry Python API and SDK. +OpenTelemetry Python v1.26.0 +https://github.com/open-telemetry/opentelemetry-python + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This library includes software which is derived from the OpenTelemetry Protobuf definitions. +OpenTelemetry Protocol Specification v1.2.0 +https://github.com/open-telemetry/opentelemetry-proto + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index ef6e6ae..ca9d2b3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ pip install --upgrade pip pip install . ``` +## Development + To develop this package, run ```bash @@ -33,3 +35,9 @@ source .venv/bin/activate pip install --upgrade pip pip install . ./tests/snowflake-telemetry-test-utils ``` + +### Code generation + +To regenerate the code under `src/snowflake/_internal/opentelemetry/proto/`, execute the script `./scripts/proto_codegen.sh`. The script expects the `src/snowflake/_internal/opentelemetry/proto/` directory to exist, and will delete all .py files in it before regerating the code. + +The commit/branch/tag of [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-proto) that the code is generated from is pinned to PROTO_REPO_BRANCH_OR_COMMIT, which can be configured in the script. It is currently pinned to the same tag as [opentelemetry-python](https://github.com/open-telemetry/opentelemetry-python/blob/main/scripts/proto_codegen.sh#L15). diff --git a/anaconda/meta.yaml b/anaconda/meta.yaml index 8bfdb31..620a7ee 100644 --- a/anaconda/meta.yaml +++ b/anaconda/meta.yaml @@ -1,6 +1,6 @@ package: name: snowflake_telemetry_python - version: "0.5.0" + version: "0.6.0.dev" source: path: {{ environ.get('SNOWFLAKE_TELEMETRY_DIR') }} @@ -11,9 +11,8 @@ requirements: - setuptools >=40.0.0 run: - python - - opentelemetry-api ==1.23.0 - - opentelemetry-exporter-otlp-proto-common ==1.23.0 - - opentelemetry-sdk ==1.23.0 + - opentelemetry-api ==1.26.0 + - opentelemetry-sdk ==1.26.0 about: home: https://www.snowflake.com/ diff --git a/benchmark/benchmark_serialize.py b/benchmark/benchmark_serialize.py new file mode 100644 index 0000000..e931ed6 --- /dev/null +++ b/benchmark/benchmark_serialize.py @@ -0,0 +1,86 @@ +import google_benchmark as benchmark + +from util import get_logs_data, get_metrics_data, get_traces_data, get_logs_data_4MB + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs as pb2_encode_logs +from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics as pb2_encode_metrics +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans as pb2_encode_spans + +""" +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations +------------------------------------------------------------------------------ +test_bm_serialize_logs_data_4MB 730591536 ns 730562298 ns 1 +test_bm_pb2_serialize_logs_data_4MB 702522039 ns 702490893 ns 1 +test_bm_serialize_logs_data 100882 ns 100878 ns 6930 +test_bm_pb2_serialize_logs_data 97112 ns 97109 ns 7195 +test_bm_serialize_metrics_data 114938 ns 114934 ns 6096 +test_bm_pb2_serialize_metrics_data 161849 ns 161845 ns 4324 +test_bm_serialize_traces_data 123977 ns 123973 ns 5633 +test_bm_pb2_serialize_traces_data 131016 ns 131011 ns 5314 +""" + +def sanity_check(): + logs_data = get_logs_data() + metrics_data = get_metrics_data() + traces_data = get_traces_data() + + assert encode_logs(logs_data).SerializeToString() == pb2_encode_logs(logs_data).SerializeToString() + assert encode_metrics(metrics_data).SerializeToString() == pb2_encode_metrics(metrics_data).SerializeToString() + assert encode_spans(traces_data).SerializeToString() == pb2_encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + pb2_encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + pb2_encode_spans(traces_data).SerializeToString() + +if __name__ == "__main__": + sanity_check() + benchmark.main() \ No newline at end of file diff --git a/benchmark/util.py b/benchmark/util.py new file mode 100644 index 0000000..e73bc56 --- /dev/null +++ b/benchmark/util.py @@ -0,0 +1,339 @@ +from typing import Sequence + +from snowflake.telemetry.test.metrictestutil import _generate_gauge, _generate_sum + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + +from opentelemetry._logs import SeverityNumber +from opentelemetry.sdk._logs import LogData, LogLimits, LogRecord + +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Buckets, + ExponentialHistogram, + Histogram, + ExponentialHistogramDataPoint, + HistogramDataPoint, + Metric, + MetricsData, + ResourceMetrics, + ScopeMetrics, +) + +from opentelemetry.sdk.trace import Event, SpanContext, _Span +from opentelemetry.trace import SpanKind, Link, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + + + +def get_logs_data() -> Sequence[LogData]: + log1 = LogData( + log_record=LogRecord( + timestamp=1644650195189786880, + observed_timestamp=1644660000000000000, + trace_id=89564621134313219400156819398935297684, + span_id=1312458408527513268, + trace_flags=TraceFlags(0x01), + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Do not go gentle into that good night. Rage, rage against the dying of the light", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=InstrumentationScope( + "first_name", "first_version" + ), + ) + + log2 = LogData( + log_record=LogRecord( + timestamp=1644650249738562048, + observed_timestamp=1644660000000000000, + trace_id=0, + span_id=0, + trace_flags=TraceFlags.DEFAULT, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Cooper, this is no time for caution!", + resource=Resource({"second_resource": "CASE"}), + attributes={}, + ), + instrumentation_scope=InstrumentationScope( + "second_name", "second_version" + ), + ) + + log3 = LogData( + log_record=LogRecord( + timestamp=1644650427658989056, + observed_timestamp=1644660000000000000, + trace_id=271615924622795969659406376515024083555, + span_id=4242561578944770265, + trace_flags=TraceFlags(0x01), + severity_text="DEBUG", + severity_number=SeverityNumber.DEBUG, + body="To our galaxy", + resource=Resource({"second_resource": "CASE"}), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=None, + ) + + log4 = LogData( + log_record=LogRecord( + timestamp=1644650584292683008, + observed_timestamp=1644660000000000000, + trace_id=212592107417388365804938480559624925555, + span_id=6077757853989569223, + trace_flags=TraceFlags(0x01), + severity_text="INFO", + severity_number=SeverityNumber.INFO, + body="Love is the one thing that transcends time and space", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"filename": "model.py", "func_name": "run_method"}, + ), + instrumentation_scope=InstrumentationScope( + "another_name", "another_version" + ), + ) + + return [log1, log2, log3, log4] + +def get_logs_data_4MB() -> Sequence[LogData]: + out = [] + for _ in range(8000): + out.extend(get_logs_data()) + return out + +HISTOGRAM = Metric( + name="histogram", + description="foo", + unit="s", + data=Histogram( + data_points=[ + HistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=1641946016139533244, + time_unix_nano=1641946016139533244, + count=5, + sum=67, + bucket_counts=[1, 4], + explicit_bounds=[10.0, 20.0], + min=8, + max=18, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) + +EXPONENTIAL_HISTOGRAM = Metric( + name="exponential_histogram", + description="description", + unit="unit", + data=ExponentialHistogram( + data_points=[ + ExponentialHistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=0, + time_unix_nano=1, + count=2, + sum=3, + scale=4, + zero_count=5, + positive=Buckets(offset=6, bucket_counts=[7, 8]), + negative=Buckets(offset=9, bucket_counts=[10, 11]), + flags=12, + min=13.0, + max=14.0, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) +def get_metrics_data() -> MetricsData: + + metrics = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource( + attributes={"a": 1, "b": False}, + schema_url="resource_schema_url", + ), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM, HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="second_name", + version="second_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="third_name", + version="third_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_int", 33)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_double", 2.98)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_int", 9000)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_double", 52.028)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[EXPONENTIAL_HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ) + ], + schema_url="resource_schema_url", + ) + ] + ) + + return metrics + +def get_traces_data() -> Sequence[_Span]: + trace_id = 0x3E0C63257DE34C926F9EFCD03927272E + + base_time = 683647322 * 10**9 # in ns + start_times = ( + base_time, + base_time + 150 * 10**6, + base_time + 300 * 10**6, + base_time + 400 * 10**6, + ) + end_times = ( + start_times[0] + (50 * 10**6), + start_times[1] + (100 * 10**6), + start_times[2] + (200 * 10**6), + start_times[3] + (300 * 10**6), + ) + + parent_span_context = SpanContext( + trace_id, 0x1111111111111111, is_remote=True + ) + + other_context = SpanContext( + trace_id, 0x2222222222222222, is_remote=False + ) + + span1 = _Span( + name="test-span-1", + context=SpanContext( + trace_id, + 0x34BF92DEEFC58C92, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ), + parent=parent_span_context, + events=( + Event( + name="event0", + timestamp=base_time + 50 * 10**6, + attributes={ + "annotation_bool": True, + "annotation_string": "annotation_test", + "key_float": 0.3, + }, + ), + ), + links=( + Link(context=other_context, attributes={"key_bool": True}), + ), + resource=Resource({}, "resource_schema_url"), + ) + span1.start(start_time=start_times[0]) + span1.set_attribute("key_bool", False) + span1.set_attribute("key_string", "hello_world") + span1.set_attribute("key_float", 111.22) + span1.set_status(Status(StatusCode.ERROR, "Example description")) + span1.end(end_time=end_times[0]) + + span2 = _Span( + name="test-span-2", + context=parent_span_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span2.start(start_time=start_times[1]) + span2.end(end_time=end_times[1]) + + span3 = _Span( + name="test-span-3", + context=other_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span3.start(start_time=start_times[2]) + span3.set_attribute("key_string", "hello_world") + span3.end(end_time=end_times[2]) + + span4 = _Span( + name="test-span-4", + context=other_context, + parent=None, + resource=Resource({}, "resource_schema_url"), + instrumentation_scope=InstrumentationScope( + name="name", version="version" + ), + ) + span4.start(start_time=start_times[3]) + span4.end(end_time=end_times[3]) + + return [span1, span2, span3, span4] \ No newline at end of file diff --git a/scripts/gen-requirements.txt b/scripts/gen-requirements.txt new file mode 100644 index 0000000..bf8c682 --- /dev/null +++ b/scripts/gen-requirements.txt @@ -0,0 +1,5 @@ +Jinja2==3.1.4 +grpcio-tools==1.62.3 +protobuf==4.25.5 +black==24.10.0 +isort==5.13.2 diff --git a/scripts/plugin.py b/scripts/plugin.py new file mode 100755 index 0000000..d3d0c7f --- /dev/null +++ b/scripts/plugin.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import re +import os +import sys +import inspect +from enum import IntEnum +from typing import List, Optional +from textwrap import dedent, indent +from dataclasses import dataclass, field +# Must be imported into globals for the inline functions to work +from snowflake.telemetry._internal.serialize import MessageMarshaler # noqa + +from google.protobuf.compiler import plugin_pb2 as plugin +from google.protobuf.descriptor_pb2 import ( + FileDescriptorProto, + FieldDescriptorProto, + EnumDescriptorProto, + EnumValueDescriptorProto, + DescriptorProto, +) +from jinja2 import Environment, FileSystemLoader +import black +import isort.api + +INLINE_OPTIMIZATION = True +FILE_PATH_PREFIX = "snowflake.telemetry._internal" +FILE_NAME_SUFFIX = "_marshaler" +OPENTELEMETRY_PROTO_DIR = os.environ["OPENTELEMETRY_PROTO_DIR"] + +# Inline utility functions + +# Inline the size function for a given proto message field +def inline_size_function(proto_type: str, attr_name: str, field_tag: str) -> str: + """ + For example: + + class MessageMarshaler: + def size_uint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR) + + Becomes: + + size += len(b"\x10") + Varint.size_varint_u32(self.int_value) + """ + function_definition = inspect.getsource(globals()["MessageMarshaler"].__dict__[f"size_{proto_type}"]) + # Remove the function header and unindent the function body + function_definition = function_definition.splitlines()[1:] + function_definition = "\n".join(function_definition) + function_definition = dedent(function_definition) + # Replace the attribute name + function_definition = function_definition.replace("FIELD_ATTR", f"self.{attr_name}") + # Replace the TAG + function_definition = function_definition.replace("TAG", field_tag) + # Inline the return statement + function_definition = function_definition.replace("return ", "size += ") + return function_definition + +# Inline the serialization function for a given proto message field +def inline_serialize_function(proto_type: str, attr_name: str, field_tag: str) -> str: + """ + For example: + + class MessageMarshaler: + def serialize_uint32(self, out: BytesIO, TAG: bytes, FIELD_ATTR: int) -> None: + out.write(TAG) + Varint.serialize_varint_u32(out, FIELD_ATTR) + + Becomes: + + out.write(b"\x10") + Varint.serialize_varint_u32(out, self.int_value) + """ + function_definition = inspect.getsource(globals()["MessageMarshaler"].__dict__[f"serialize_{proto_type}"]) + # Remove the function header and unindent the function body + function_definition = function_definition.splitlines()[1:] + function_definition = "\n".join(function_definition) + function_definition = dedent(function_definition) + # Replace the attribute name + function_definition = function_definition.replace("FIELD_ATTR", f"self.{attr_name}") + # Replace the TAG + function_definition = function_definition.replace("TAG", field_tag) + return function_definition + +# Add a presence check to a function definition +# https://protobuf.dev/programming-guides/proto3/#default +def add_presence_check(proto_type: str, encode_presence: bool, attr_name: str, function_definition: str) -> str: + # oneof, optional (virtual oneof), and message fields are encoded if they are not None + function_definition = indent(function_definition, " ") + if encode_presence: + return f"if self.{attr_name} is not None:\n{function_definition}" + # Other fields are encoded if they are not the default value + # Which happens to align with the bool(x) check for all primitive types + # TODO: Except + # - double and float -0.0 should be encoded, even though bool(-0.0) is False + return f"if self.{attr_name}:\n{function_definition}" + +class WireType(IntEnum): + VARINT = 0 + I64 = 1 + LEN = 2 + I32 = 5 + +@dataclass +class ProtoTypeDescriptor: + name: str + wire_type: WireType + python_type: str + default_val: str + +proto_type_to_descriptor = { + FieldDescriptorProto.TYPE_BOOL: ProtoTypeDescriptor("bool", WireType.VARINT, "bool", "False"), + FieldDescriptorProto.TYPE_ENUM: ProtoTypeDescriptor("enum", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_INT32: ProtoTypeDescriptor("int32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_INT64: ProtoTypeDescriptor("int64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_UINT32: ProtoTypeDescriptor("uint32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_UINT64: ProtoTypeDescriptor("uint64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_SINT32: ProtoTypeDescriptor("sint32", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_SINT64: ProtoTypeDescriptor("sint64", WireType.VARINT, "int", "0"), + FieldDescriptorProto.TYPE_FIXED32: ProtoTypeDescriptor("fixed32", WireType.I32, "int", "0"), + FieldDescriptorProto.TYPE_FIXED64: ProtoTypeDescriptor("fixed64", WireType.I64, "int", "0"), + FieldDescriptorProto.TYPE_SFIXED32: ProtoTypeDescriptor("sfixed32", WireType.I32, "int", "0"), + FieldDescriptorProto.TYPE_SFIXED64: ProtoTypeDescriptor("sfixed64", WireType.I64, "int", "0"), + FieldDescriptorProto.TYPE_FLOAT: ProtoTypeDescriptor("float", WireType.I32, "float", "0.0"), + FieldDescriptorProto.TYPE_DOUBLE: ProtoTypeDescriptor("double", WireType.I64, "float", "0.0"), + FieldDescriptorProto.TYPE_STRING: ProtoTypeDescriptor("string", WireType.LEN, "str", '""'), + FieldDescriptorProto.TYPE_BYTES: ProtoTypeDescriptor("bytes", WireType.LEN, "bytes", 'b""'), + FieldDescriptorProto.TYPE_MESSAGE: ProtoTypeDescriptor("message", WireType.LEN, "PLACEHOLDER", "None"), +} + +@dataclass +class EnumValueTemplate: + name: str + number: int + + @staticmethod + def from_descriptor(descriptor: EnumValueDescriptorProto) -> "EnumValueTemplate": + return EnumValueTemplate( + name=descriptor.name, + number=descriptor.number, + ) + +@dataclass +class EnumTemplate: + name: str + values: List[EnumValueTemplate] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: EnumDescriptorProto) -> "EnumTemplate": + return EnumTemplate( + name=descriptor.name, + values=[EnumValueTemplate.from_descriptor(value) for value in descriptor.value], + ) + +def tag_to_repr_varint(tag: int) -> str: + out = bytearray() + while tag >= 128: + out.append((tag & 0x7F) | 0x80) + tag >>= 7 + out.append(tag) + return repr(bytes(out)) + +@dataclass +class FieldTemplate: + name: str + attr_name: str + number: int + generator: str + python_type: str + proto_type: str + default_val: str + serialize_field_inline: str + size_field_inline: str + + @staticmethod + def from_descriptor(descriptor: FieldDescriptorProto, group: Optional[str] = None) -> "FieldTemplate": + type_descriptor = proto_type_to_descriptor[descriptor.type] + python_type = type_descriptor.python_type + proto_type = type_descriptor.name + default_val = type_descriptor.default_val + + if proto_type == "message" or proto_type == "enum": + # Extract the class name of message fields, to use as python type + python_type = re.sub(r"^[a-zA-Z0-9_\.]+\.v1\.", "", descriptor.type_name) + + repeated = descriptor.label == FieldDescriptorProto.LABEL_REPEATED + if repeated: + # Update type for repeated fields + python_type = f"List[{python_type}]" + proto_type = f"repeated_{proto_type}" + # Default value is None, since we can't use a mutable default value like [] + default_val = "None" + + # Calculate the tag for the field to save some computation at runtime + tag = (descriptor.number << 3) | type_descriptor.wire_type.value + if repeated and type_descriptor.wire_type != WireType.LEN: + # Special case: repeated primitive fields are packed, so need to use LEN wire type + # Note: packed fields can be disabled in proto files, but we don't handle that case + # https://protobuf.dev/programming-guides/encoding/#packed + tag = (descriptor.number << 3) | WireType.LEN.value + # Convert the tag to a varint representation to inline it in the generated code + tag = tag_to_repr_varint(tag) + + # For oneof and optional fields, we need to encode the presence of the field. + # Optional fields are treated as virtual oneof fields, with a single field in the oneof. + # For message fields, we need to encode the presence of the field if it is not None. + # https://protobuf.dev/programming-guides/field_presence/ + encode_presence = group is not None or proto_type == "message" + if group is not None: + # The default value for oneof fields must be None, so that the default value is not encoded + default_val = "None" + + field_name = descriptor.name + attr_name = field_name + generator = None + if proto_type == "message" or repeated: + # For message and repeated fields, store as a private attribute that is + # initialized on access to match protobuf embedded message access pattern + if repeated: + # In python protobuf, repeated fields return an implementation of the list interface + # with a self.add() method to add and initialize elements + # This can be supported with a custom list implementation, but we use a simple list for now + # https://protobuf.dev/reference/python/python-generated/#repeated-message-fields + generator = "list()" + else: + # https://protobuf.dev/reference/python/python-generated/#embedded_message + generator = f"{python_type}()" + # the attribute name is prefixed with an underscore as message and repeated attributes + # are hidden behind a property that has the actual proto field name + attr_name = f"_{field_name}" + + # Inline the size and serialization functions for the field + if INLINE_OPTIMIZATION: + serialize_field_inline = inline_serialize_function(proto_type, attr_name, tag) + size_field_inline = inline_size_function(proto_type, attr_name, tag) + else: + serialize_field_inline = f"self.serialize_{proto_type}(out, {tag}, self.{attr_name})" + size_field_inline = f"size += self.size_{proto_type}({tag}, self.{attr_name})" + + serialize_field_inline = add_presence_check(proto_type, encode_presence, attr_name, serialize_field_inline) + size_field_inline = add_presence_check(proto_type, encode_presence, attr_name, size_field_inline) + + return FieldTemplate( + name=field_name, + attr_name=attr_name, + number=descriptor.number, + generator=generator, + python_type=python_type, + proto_type=proto_type, + default_val=default_val, + serialize_field_inline=serialize_field_inline, + size_field_inline=size_field_inline, + ) + +@dataclass +class MessageTemplate: + name: str + fields: List[FieldTemplate] = field(default_factory=list) + enums: List[EnumTemplate] = field(default_factory=list) + messages: List[MessageTemplate] = field(default_factory=list) + + @staticmethod + def from_descriptor(descriptor: DescriptorProto) -> "MessageTemplate": + # Helper function to extract the group name for a field, if it exists + def get_group(field: FieldDescriptorProto) -> str: + return descriptor.oneof_decl[field.oneof_index].name if field.HasField("oneof_index") else None + fields = [FieldTemplate.from_descriptor(field, get_group(field)) for field in descriptor.field] + fields.sort(key=lambda field: field.number) + + name = descriptor.name + return MessageTemplate( + name=name, + fields=fields, + enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], + messages=[MessageTemplate.from_descriptor(message) for message in descriptor.nested_type], + ) + +@dataclass +class FileTemplate: + messages: List[MessageTemplate] = field(default_factory=list) + enums: List[EnumTemplate] = field(default_factory=list) + imports: List[str] = field(default_factory=list) + name: str = "" + preamble: str = "" + + @staticmethod + def from_descriptor(descriptor: FileDescriptorProto) -> "FileTemplate": + # Extract the preamble comment from the proto file + # In the case of the opentelemetry-proto files, the preamble is the license header + # Each line of the preamble is prefixed with "//" + preamble = "" + with open(f'{OPENTELEMETRY_PROTO_DIR}/{descriptor.name}', "r") as f: + line = f.readline() + while line and line.startswith("//"): + preamble += line.replace("//", "#", 1) + line = f.readline() + + # Extract the import paths for the proto file + imports = [] + for dependency in descriptor.dependency: + path = re.sub(r"\.proto$", "", dependency) + if descriptor.name.startswith(path): + continue + path = path.replace("/", ".") + path = f"{FILE_PATH_PREFIX}.{path}{FILE_NAME_SUFFIX}" + imports.append(path) + + return FileTemplate( + messages=[MessageTemplate.from_descriptor(message) for message in descriptor.message_type], + enums=[EnumTemplate.from_descriptor(enum) for enum in descriptor.enum_type], + imports=imports, + name=descriptor.name, + preamble=preamble, + ) + +def main(): + request = plugin.CodeGeneratorRequest() + request.ParseFromString(sys.stdin.buffer.read()) + + response = plugin.CodeGeneratorResponse() + # Needed since metrics.proto uses proto3 optional fields + # https://github.com/protocolbuffers/protobuf/blob/main/docs/implementing_proto3_presence.md + response.supported_features = plugin.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + + template_env = Environment(loader=FileSystemLoader(f"{os.path.dirname(os.path.realpath(__file__))}/templates")) + jinja_body_template = template_env.get_template("template.py.jinja2") + + for proto_file in request.proto_file: + file_name = re.sub(r"\.proto$", f"{FILE_NAME_SUFFIX}.py", proto_file.name) + file_descriptor_proto = proto_file + + file_template = FileTemplate.from_descriptor(file_descriptor_proto) + + code = jinja_body_template.render(file_template=file_template) + code = isort.api.sort_code_string( + code = code, + show_diff=False, + profile="black", + combine_as_imports=True, + lines_after_imports=2, + quiet=True, + force_grid_wrap=2, + ) + code = black.format_str( + src_contents=code, + mode=black.Mode(), + ) + + response_file = response.file.add() + response_file.name = file_name + response_file.content = code + + sys.stdout.buffer.write(response.SerializeToString()) + +if __name__ == '__main__': + main() diff --git a/scripts/proto_codegen.sh b/scripts/proto_codegen.sh new file mode 100755 index 0000000..4c7b08b --- /dev/null +++ b/scripts/proto_codegen.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Regenerate python code from OTLP protos in +# https://github.com/open-telemetry/opentelemetry-proto +# +# To use, update PROTO_REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-proto repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. +# +# Optional envars: +# PROTO_REPO_DIR - the path to an existing checkout of the opentelemetry-proto repo + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +PROTO_REPO_BRANCH_OR_COMMIT="v1.2.0" + +set -e + +PROTO_REPO_DIR=${PROTO_REPO_DIR:-"/tmp/opentelemetry-proto"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" +venv_dir="/tmp/proto_codegen_venv" + +# run on exit even if crash +cleanup() { + echo "Deleting $venv_dir" + rm -rf $venv_dir +} +trap cleanup EXIT + +echo "Creating temporary virtualenv at $venv_dir using $(python3 --version)" +python3 -m venv $venv_dir +source $venv_dir/bin/activate +python -m pip install \ + -c $repo_root/scripts/gen-requirements.txt \ + protobuf Jinja2 grpcio-tools black isort . + +echo 'python -m grpc_tools.protoc --version' +python -m grpc_tools.protoc --version + +# Clone the proto repo if it doesn't exist +if [ ! -d "$PROTO_REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-proto.git $PROTO_REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $PROTO_REPO_DIR + git fetch --all + git checkout $PROTO_REPO_BRANCH_OR_COMMIT + # pull if PROTO_REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# clean up old generated code +mkdir -p opentelemetry/proto +find opentelemetry/proto/ -regex ".*_marshaler\.py" -exec rm {} + + +# generate proto code for all protos +all_protos=$(find $PROTO_REPO_DIR/ -iname "*.proto") +OPENTELEMETRY_PROTO_DIR=$PROTO_REPO_DIR python -m grpc_tools.protoc \ + -I $PROTO_REPO_DIR \ + --plugin=protoc-gen-custom-plugin=$repo_root/scripts/plugin.py \ + --custom-plugin_out=. \ + $all_protos diff --git a/scripts/templates/template.py.jinja2 b/scripts/templates/template.py.jinja2 new file mode 100644 index 0000000..b85408a --- /dev/null +++ b/scripts/templates/template.py.jinja2 @@ -0,0 +1,86 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: {{ file_template.name }} +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +{{ file_template.preamble -}} +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) +from typing import List + +{% for import in file_template.imports %} +from {{ import }} import * +{% endfor %} + +{% for enum in file_template.enums %} +class {{ enum.name }}(Enum): +{%- for value in enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% macro render_message(message) %} +class {{ message.name }}(MessageMarshaler): + +{%- for field in message.fields %} +{%- if field.generator %} + @property + def {{ field.name }}(self) -> {{ field.python_type }}: + if self.{{ field.attr_name }} is None: + self.{{ field.attr_name }} = {{ field.generator }} + return self.{{ field.attr_name }} +{%- else %} + {{ field.name }}: {{ field.python_type }} +{%- endif %} +{%- endfor %} + + def __init__( + self, +{%- for field in message.fields %} + {{ field.name }}: {{ field.python_type }} = {{ field.default_val }}, +{%- endfor %} + ): +{%- for field in message.fields %} + self.{{ field.attr_name }}: {{ field.python_type }} = {{ field.name }} +{%- endfor %} + + def calculate_size(self) -> int: + size = 0 +{%- for field in message.fields %} + {{ field.size_field_inline | indent(8) }} +{%- endfor %} + return size + + def write_to(self, out: bytearray) -> None: +{%- for field in message.fields %} + {{ field.serialize_field_inline | indent(8) }} +{%- endfor %} + +{% for nested_enum in message.enums %} + class {{ nested_enum.name }}(Enum): +{%- for value in nested_enum.values %} + {{ value.name }} = {{ value.number }} +{%- endfor %} +{% endfor %} + +{% for nested_message in message.messages %} +{{ render_message(nested_message) | indent(4) }} +{% endfor %} +{% endmacro %} + +{% for message in file_template.messages %} +{{ render_message(message) }} +{% endfor %} \ No newline at end of file diff --git a/scripts/vendor_otlp_proto_common.sh b/scripts/vendor_otlp_proto_common.sh new file mode 100755 index 0000000..eae16a2 --- /dev/null +++ b/scripts/vendor_otlp_proto_common.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Vendor in the python code in +# https://github.com/open-telemetry/opentelemetry-python/tree/main/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common +# +# To use, update REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-python repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +REPO_BRANCH_OR_COMMIT="v1.26.0" + +set -e + +REPO_DIR=${REPO_DIR:-"/tmp/opentelemetry-python"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" + +# Clone the proto repo if it doesn't exist +if [ ! -d "$REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-python.git $REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $REPO_DIR + git fetch --all + git checkout $REPO_BRANCH_OR_COMMIT + # pull if REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# Copy the entire file tree from exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/ +# to src/snowflake/telemetry/_internal/opentelemetry/ +cp -r $REPO_DIR/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter opentelemetry/ + +# If MacOS need '' after -i +# Detect the OS (macOS or Linux) +if [[ "$OSTYPE" == "darwin"* ]]; then + SED_CMD="sed -i ''" +else + SED_CMD="sed -i" +fi + +# Replace all the imports strings in the copied python files +# opentelemetry.exporter to snowflake.telemetry._internal.opentelemetry.exporter +# opentelemetry.proto.*_pb2 to snowflake.telemetry._internal.opentelemetry.proto.*_marshaler + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry.exporter/snowflake.telemetry._internal.opentelemetry.exporter/g' {} + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry\.proto\(.*\)_pb2/snowflake.telemetry._internal.opentelemetry.proto\1_marshaler/g' {} + + + +# Add a notice to the top of every file in compliance with Apache 2.0 to indicate that the file has been modified +# https://www.apache.org/licenses/LICENSE-2.0 +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD '14s|^|#\n# This file has been modified from the original source code at\n#\n# https://github.com/open-telemetry/opentelemetry-python/tree/'"$REPO_BRANCH_OR_COMMIT"'\n#\n# by Snowflake Inc.\n|' {} + diff --git a/setup.py b/setup.py index 92c3682..b554ae0 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,8 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, install_requires=[ - "opentelemetry-api == 1.23.0", - "opentelemetry-exporter-otlp-proto-common == 1.23.0", - "opentelemetry-sdk == 1.23.0", + "opentelemetry-api == 1.26.0", + "opentelemetry-sdk == 1.26.0", ], packages=find_namespace_packages( where='src' @@ -53,6 +52,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database", "Topic :: Software Development", "Topic :: Software Development :: Libraries", diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py index a08f74c..9361c20 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py @@ -21,10 +21,10 @@ import opentelemetry.sdk.util.instrumentation as otel_instrumentation import opentelemetry.sdk._logs._internal as _logs_internal -from opentelemetry.exporter.otlp.proto.common._log_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import ( encode_logs, ) -from opentelemetry.proto.logs.v1.logs_pb2 import LogsData +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogsData from opentelemetry.sdk.resources import Resource from opentelemetry.sdk._logs import export from opentelemetry.sdk import _logs @@ -183,10 +183,10 @@ class _SnowflakeTelemetryLoggerProvider(_logs.LoggerProvider): """ def get_logger( - self, - name: str, - version: types.Optional[str] = None, - schema_url: types.Optional[str] = None, + self, name: str, + version: types.Optional[str] = None, + schema_url: types.Optional[str] = None, + attributes: types.Optional[types.Attributes] = None, ) -> _logs.Logger: return _SnowflakeTelemetryLogger( Resource.get_empty(), diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py index 46291d1..91ff137 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py @@ -17,10 +17,10 @@ from typing import Dict import opentelemetry -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) -from opentelemetry.proto.metrics.v1.metrics_pb2 import MetricsData as PB2MetricsData +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler import MetricsData as PB2MetricsData from opentelemetry.sdk.metrics.export import ( AggregationTemporality, MetricExportResult, diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py index 7c877aa..41aa617 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py @@ -16,10 +16,10 @@ import abc import typing -from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import ( encode_spans, ) -from opentelemetry.proto.trace.v1.trace_pb2 import TracesData +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import TracesData from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import ( SpanExportResult, diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py new file mode 100644 index 0000000..c1250a0 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py @@ -0,0 +1,24 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ + +__all__ = ["__version__"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py new file mode 100644 index 0000000..530528f --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -0,0 +1,182 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +import logging +from collections.abc import Sequence +from itertools import count +from typing import ( + Any, + Mapping, + Optional, + List, + Callable, + TypeVar, + Dict, + Iterator, +) + +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + InstrumentationScope as PB2InstrumentationScope, +) +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import AnyValue as PB2AnyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import KeyValue as PB2KeyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + KeyValueList as PB2KeyValueList, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + ArrayValue as PB2ArrayValue, +) +from opentelemetry.sdk.trace import Resource +from opentelemetry.util.types import Attributes + +_logger = logging.getLogger(__name__) + +_TypingResourceT = TypeVar("_TypingResourceT") +_ResourceDataT = TypeVar("_ResourceDataT") + + +def _encode_instrumentation_scope( + instrumentation_scope: InstrumentationScope, +) -> PB2InstrumentationScope: + if instrumentation_scope is None: + return PB2InstrumentationScope() + return PB2InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + + +def _encode_resource(resource: Resource) -> PB2Resource: + return PB2Resource(attributes=_encode_attributes(resource.attributes)) + + +def _encode_value(value: Any) -> PB2AnyValue: + if isinstance(value, bool): + return PB2AnyValue(bool_value=value) + if isinstance(value, str): + return PB2AnyValue(string_value=value) + if isinstance(value, int): + return PB2AnyValue(int_value=value) + if isinstance(value, float): + return PB2AnyValue(double_value=value) + if isinstance(value, Sequence): + return PB2AnyValue( + array_value=PB2ArrayValue(values=[_encode_value(v) for v in value]) + ) + elif isinstance(value, Mapping): + return PB2AnyValue( + kvlist_value=PB2KeyValueList( + values=[_encode_key_value(str(k), v) for k, v in value.items()] + ) + ) + raise Exception(f"Invalid type {type(value)} of value {value}") + + +def _encode_key_value(key: str, value: Any) -> PB2KeyValue: + return PB2KeyValue(key=key, value=_encode_value(value)) + + +def _encode_span_id(span_id: int) -> bytes: + return span_id.to_bytes(length=8, byteorder="big", signed=False) + + +def _encode_trace_id(trace_id: int) -> bytes: + return trace_id.to_bytes(length=16, byteorder="big", signed=False) + + +def _encode_attributes( + attributes: Attributes, +) -> Optional[List[PB2KeyValue]]: + if attributes: + pb2_attributes = [] + for key, value in attributes.items(): + # pylint: disable=broad-exception-caught + try: + pb2_attributes.append(_encode_key_value(key, value)) + except Exception as error: + _logger.exception("Failed to encode key %s: %s", key, error) + else: + pb2_attributes = None + return pb2_attributes + + +def _get_resource_data( + sdk_resource_scope_data: Dict[Resource, _ResourceDataT], + resource_class: Callable[..., _TypingResourceT], + name: str, +) -> List[_TypingResourceT]: + resource_data = [] + + for ( + sdk_resource, + scope_data, + ) in sdk_resource_scope_data.items(): + collector_resource = PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ) + resource_data.append( + resource_class( + **{ + "resource": collector_resource, + "scope_{}".format(name): scope_data.values(), + } + ) + ) + return resource_data + + +def _create_exp_backoff_generator(max_value: int = 0) -> Iterator[int]: + """ + Generates an infinite sequence of exponential backoff values. The sequence starts + from 1 (2^0) and doubles each time (2^1, 2^2, 2^3, ...). If a max_value is specified + and non-zero, the generated values will not exceed this maximum, capping at max_value + instead of growing indefinitely. + + Parameters: + - max_value (int, optional): The maximum value to yield. If 0 or not provided, the + sequence grows without bound. + + Returns: + Iterator[int]: An iterator that yields the exponential backoff values, either uncapped or + capped at max_value. + + Example: + ``` + gen = _create_exp_backoff_generator(max_value=10) + for _ in range(5): + print(next(gen)) + ``` + This will print: + 1 + 2 + 4 + 8 + 10 + + Note: this functionality used to be handled by the 'backoff' package. + """ + for i in count(0): + out = 2**i + yield min(out, max_value) if max_value else out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py new file mode 100644 index 0000000..2f71a17 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py @@ -0,0 +1,101 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +from collections import defaultdict +from typing import Sequence, List + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, + _encode_value, + _encode_attributes, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.logs.v1.logs_service_marshaler import ( + ExportLogsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import ( + ScopeLogs, + ResourceLogs, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogRecord as PB2LogRecord + +from opentelemetry.sdk._logs import LogData + + +def encode_logs(batch: Sequence[LogData]) -> ExportLogsServiceRequest: + return ExportLogsServiceRequest(resource_logs=_encode_resource_logs(batch)) + + +def _encode_log(log_data: LogData) -> PB2LogRecord: + span_id = ( + None + if log_data.log_record.span_id == 0 + else _encode_span_id(log_data.log_record.span_id) + ) + trace_id = ( + None + if log_data.log_record.trace_id == 0 + else _encode_trace_id(log_data.log_record.trace_id) + ) + return PB2LogRecord( + time_unix_nano=log_data.log_record.timestamp, + observed_time_unix_nano=log_data.log_record.observed_timestamp, + span_id=span_id, + trace_id=trace_id, + flags=int(log_data.log_record.trace_flags), + body=_encode_value(log_data.log_record.body), + severity_text=log_data.log_record.severity_text, + attributes=_encode_attributes(log_data.log_record.attributes), + dropped_attributes_count=log_data.log_record.dropped_attributes, + severity_number=log_data.log_record.severity_number.value, + ) + + +def _encode_resource_logs(batch: Sequence[LogData]) -> List[ResourceLogs]: + sdk_resource_logs = defaultdict(lambda: defaultdict(list)) + + for sdk_log in batch: + sdk_resource = sdk_log.log_record.resource + sdk_instrumentation = sdk_log.instrumentation_scope or None + pb2_log = _encode_log(sdk_log) + + sdk_resource_logs[sdk_resource][sdk_instrumentation].append(pb2_log) + + pb2_resource_logs = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_logs.items(): + scope_logs = [] + for sdk_instrumentation, pb2_logs in sdk_instrumentations.items(): + scope_logs.append( + ScopeLogs( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + log_records=pb2_logs, + ) + ) + pb2_resource_logs.append( + ResourceLogs( + resource=_encode_resource(sdk_resource), + scope_logs=scope_logs, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_logs diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py new file mode 100644 index 0000000..9d729cd --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py @@ -0,0 +1,344 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +import logging + +from opentelemetry.sdk.metrics.export import ( + MetricExporter, +) +from opentelemetry.sdk.metrics.view import Aggregation +from os import environ +from opentelemetry.sdk.metrics import ( + Counter, + Histogram, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, +) +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, +) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.metrics.v1.metrics_service_marshaler import ( + ExportMetricsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1 import metrics_marshaler as pb2 +from opentelemetry.sdk.metrics.export import ( + MetricsData, + Gauge, + Histogram as HistogramType, + Sum, + ExponentialHistogram as ExponentialHistogramType, +) +from typing import Dict +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, +) +from opentelemetry.sdk.metrics.view import ( + ExponentialBucketHistogramAggregation, + ExplicitBucketHistogramAggregation, +) + +_logger = logging.getLogger(__name__) + + +class OTLPMetricExporterMixin: + def _common_configuration( + self, + preferred_temporality: Dict[type, AggregationTemporality] = None, + preferred_aggregation: Dict[type, Aggregation] = None, + ) -> None: + + MetricExporter.__init__( + self, + preferred_temporality=self._get_temporality(preferred_temporality), + preferred_aggregation=self._get_aggregation(preferred_aggregation), + ) + + def _get_temporality( + self, preferred_temporality: Dict[type, AggregationTemporality] + ) -> Dict[type, AggregationTemporality]: + + otel_exporter_otlp_metrics_temporality_preference = ( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, + "CUMULATIVE", + ) + .upper() + .strip() + ) + + if otel_exporter_otlp_metrics_temporality_preference == "DELTA": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.DELTA, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + elif otel_exporter_otlp_metrics_temporality_preference == "LOWMEMORY": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + else: + if otel_exporter_otlp_metrics_temporality_preference != ( + "CUMULATIVE" + ): + _logger.warning( + "Unrecognized OTEL_EXPORTER_METRICS_TEMPORALITY_PREFERENCE" + " value found: " + f"{otel_exporter_otlp_metrics_temporality_preference}, " + "using CUMULATIVE" + ) + instrument_class_temporality = { + Counter: AggregationTemporality.CUMULATIVE, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.CUMULATIVE, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + instrument_class_temporality.update(preferred_temporality or {}) + + return instrument_class_temporality + + def _get_aggregation( + self, + preferred_aggregation: Dict[type, Aggregation], + ) -> Dict[type, Aggregation]: + + otel_exporter_otlp_metrics_default_histogram_aggregation = environ.get( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + "explicit_bucket_histogram", + ) + + if otel_exporter_otlp_metrics_default_histogram_aggregation == ( + "base2_exponential_bucket_histogram" + ): + + instrument_class_aggregation = { + Histogram: ExponentialBucketHistogramAggregation(), + } + + else: + + if otel_exporter_otlp_metrics_default_histogram_aggregation != ( + "explicit_bucket_histogram" + ): + + _logger.warning( + ( + "Invalid value for %s: %s, using explicit bucket " + "histogram aggregation" + ), + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + otel_exporter_otlp_metrics_default_histogram_aggregation, + ) + + instrument_class_aggregation = { + Histogram: ExplicitBucketHistogramAggregation(), + } + + instrument_class_aggregation.update(preferred_aggregation or {}) + + return instrument_class_aggregation + + +def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: + resource_metrics_dict = {} + + for resource_metrics in data.resource_metrics: + + resource = resource_metrics.resource + + # It is safe to assume that each entry in data.resource_metrics is + # associated with an unique resource. + scope_metrics_dict = {} + + resource_metrics_dict[resource] = scope_metrics_dict + + for scope_metrics in resource_metrics.scope_metrics: + + instrumentation_scope = scope_metrics.scope + + # The SDK groups metrics in instrumentation scopes already so + # there is no need to check for existing instrumentation scopes + # here. + pb2_scope_metrics = pb2.ScopeMetrics( + scope=InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + ) + + scope_metrics_dict[instrumentation_scope] = pb2_scope_metrics + + for metric in scope_metrics.metrics: + pb2_metric = pb2.Metric( + name=metric.name, + description=metric.description, + unit=metric.unit, + ) + + if isinstance(metric.data, Gauge): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + pb2_metric.gauge.data_points.append(pt) + + elif isinstance(metric.data, HistogramType): + for data_point in metric.data.data_points: + pt = pb2.HistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + bucket_counts=data_point.bucket_counts, + explicit_bounds=data_point.explicit_bounds, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.histogram.data_points.append(pt) + + elif isinstance(metric.data, Sum): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + # note that because sum is a message type, the + # fields must be set individually rather than + # instantiating a pb2.Sum and setting it once + pb2_metric.sum.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.sum.is_monotonic = metric.data.is_monotonic + pb2_metric.sum.data_points.append(pt) + + elif isinstance(metric.data, ExponentialHistogramType): + for data_point in metric.data.data_points: + + if data_point.positive.bucket_counts: + positive = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.positive.offset, + bucket_counts=data_point.positive.bucket_counts, + ) + else: + positive = None + + if data_point.negative.bucket_counts: + negative = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.negative.offset, + bucket_counts=data_point.negative.bucket_counts, + ) + else: + negative = None + + pt = pb2.ExponentialHistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + scale=data_point.scale, + zero_count=data_point.zero_count, + positive=positive, + negative=negative, + flags=data_point.flags, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.exponential_histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.exponential_histogram.data_points.append(pt) + + else: + _logger.warning( + "unsupported data type %s", + metric.data.__class__.__name__, + ) + continue + + pb2_scope_metrics.metrics.append(pb2_metric) + + resource_data = [] + for ( + sdk_resource, + scope_data, + ) in resource_metrics_dict.items(): + resource_data.append( + pb2.ResourceMetrics( + resource=PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ), + scope_metrics=scope_data.values(), + schema_url=sdk_resource.schema_url, + ) + ) + resource_metrics = resource_data + return ExportMetricsServiceRequest(resource_metrics=resource_metrics) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py new file mode 100644 index 0000000..e269641 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py @@ -0,0 +1,195 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +import logging +from collections import defaultdict +from typing import List, Optional, Sequence + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.trace.v1.trace_service_marshaler import ( + ExportTraceServiceRequest as PB2ExportTraceServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ( + ResourceSpans as PB2ResourceSpans, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ScopeSpans as PB2ScopeSpans +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Span as PB2SPan +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import SpanFlags as PB2SpanFlags +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Status as PB2Status +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import Link, SpanKind +from opentelemetry.trace.span import SpanContext, Status, TraceState + +# pylint: disable=E1101 +_SPAN_KIND_MAP = { + SpanKind.INTERNAL: PB2SPan.SpanKind.SPAN_KIND_INTERNAL, + SpanKind.SERVER: PB2SPan.SpanKind.SPAN_KIND_SERVER, + SpanKind.CLIENT: PB2SPan.SpanKind.SPAN_KIND_CLIENT, + SpanKind.PRODUCER: PB2SPan.SpanKind.SPAN_KIND_PRODUCER, + SpanKind.CONSUMER: PB2SPan.SpanKind.SPAN_KIND_CONSUMER, +} + +_logger = logging.getLogger(__name__) + + +def encode_spans( + sdk_spans: Sequence[ReadableSpan], +) -> PB2ExportTraceServiceRequest: + return PB2ExportTraceServiceRequest( + resource_spans=_encode_resource_spans(sdk_spans) + ) + + +def _encode_resource_spans( + sdk_spans: Sequence[ReadableSpan], +) -> List[PB2ResourceSpans]: + # We need to inspect the spans and group + structure them as: + # + # Resource + # Instrumentation Library + # Spans + # + # First loop organizes the SDK spans in this structure. Protobuf messages + # are not hashable so we stick with SDK data in this phase. + # + # Second loop encodes the data into Protobuf format. + # + sdk_resource_spans = defaultdict(lambda: defaultdict(list)) + + for sdk_span in sdk_spans: + sdk_resource = sdk_span.resource + sdk_instrumentation = sdk_span.instrumentation_scope or None + pb2_span = _encode_span(sdk_span) + + sdk_resource_spans[sdk_resource][sdk_instrumentation].append(pb2_span) + + pb2_resource_spans = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_spans.items(): + scope_spans = [] + for sdk_instrumentation, pb2_spans in sdk_instrumentations.items(): + scope_spans.append( + PB2ScopeSpans( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + spans=pb2_spans, + ) + ) + pb2_resource_spans.append( + PB2ResourceSpans( + resource=_encode_resource(sdk_resource), + scope_spans=scope_spans, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_spans + + +def _span_flags(parent_span_context: Optional[SpanContext]) -> int: + flags = PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK + if parent_span_context and parent_span_context.is_remote: + flags |= PB2SpanFlags.SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK + return flags + + +def _encode_span(sdk_span: ReadableSpan) -> PB2SPan: + span_context = sdk_span.get_span_context() + return PB2SPan( + trace_id=_encode_trace_id(span_context.trace_id), + span_id=_encode_span_id(span_context.span_id), + trace_state=_encode_trace_state(span_context.trace_state), + parent_span_id=_encode_parent_id(sdk_span.parent), + name=sdk_span.name, + kind=_SPAN_KIND_MAP[sdk_span.kind], + start_time_unix_nano=sdk_span.start_time, + end_time_unix_nano=sdk_span.end_time, + attributes=_encode_attributes(sdk_span.attributes), + events=_encode_events(sdk_span.events), + links=_encode_links(sdk_span.links), + status=_encode_status(sdk_span.status), + dropped_attributes_count=sdk_span.dropped_attributes, + dropped_events_count=sdk_span.dropped_events, + dropped_links_count=sdk_span.dropped_links, + flags=_span_flags(sdk_span.parent), + ) + + +def _encode_events( + events: Sequence[Event], +) -> Optional[List[PB2SPan.Event]]: + pb2_events = None + if events: + pb2_events = [] + for event in events: + encoded_event = PB2SPan.Event( + name=event.name, + time_unix_nano=event.timestamp, + attributes=_encode_attributes(event.attributes), + dropped_attributes_count=event.dropped_attributes, + ) + pb2_events.append(encoded_event) + return pb2_events + + +def _encode_links(links: Sequence[Link]) -> Sequence[PB2SPan.Link]: + pb2_links = None + if links: + pb2_links = [] + for link in links: + encoded_link = PB2SPan.Link( + trace_id=_encode_trace_id(link.context.trace_id), + span_id=_encode_span_id(link.context.span_id), + attributes=_encode_attributes(link.attributes), + dropped_attributes_count=link.attributes.dropped, + flags=_span_flags(link.context), + ) + pb2_links.append(encoded_link) + return pb2_links + + +def _encode_status(status: Status) -> Optional[PB2Status]: + pb2_status = None + if status is not None: + pb2_status = PB2Status( + code=status.status_code.value, + message=status.description, + ) + return pb2_status + + +def _encode_trace_state(trace_state: TraceState) -> Optional[str]: + pb2_trace_state = None + if trace_state is not None: + pb2_trace_state = ",".join( + [f"{key}={value}" for key, value in (trace_state.items())] + ) + return pb2_trace_state + + +def _encode_parent_id(context: Optional[SpanContext]) -> Optional[bytes]: + if context: + return _encode_span_id(context.span_id) + return None diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py new file mode 100644 index 0000000..481a853 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal._log_encoder import ( + encode_logs, +) + +__all__ = ["encode_logs"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py new file mode 100644 index 0000000..4d82926 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder import ( + encode_metrics, +) + +__all__ = ["encode_metrics"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py new file mode 100644 index 0000000..bc37212 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( + encode_spans, +) + +__all__ = ["encode_spans"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py new file mode 100644 index 0000000..8ddc9b6 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py @@ -0,0 +1,21 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +__version__ = "1.26.0" diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py new file mode 100644 index 0000000..e023504 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/logs/v1/logs_service_marshaler.py @@ -0,0 +1,130 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/logs/v1/logs_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportLogsServiceRequest(MessageMarshaler): + @property + def resource_logs(self) -> List[ResourceLogs]: + if self._resource_logs is None: + self._resource_logs = list() + return self._resource_logs + + def __init__( + self, + resource_logs: List[ResourceLogs] = None, + ): + self._resource_logs: List[ResourceLogs] = resource_logs + + def calculate_size(self) -> int: + size = 0 + if self._resource_logs: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_logs + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_logs: + for v in self._resource_logs: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportLogsServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportLogsPartialSuccess: + if self._partial_success is None: + self._partial_success = ExportLogsPartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportLogsPartialSuccess = None, + ): + self._partial_success: ExportLogsPartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportLogsPartialSuccess(MessageMarshaler): + rejected_log_records: int + error_message: str + + def __init__( + self, + rejected_log_records: int = 0, + error_message: str = "", + ): + self.rejected_log_records: int = rejected_log_records + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_log_records: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_log_records) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_log_records: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_log_records) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py new file mode 100644 index 0000000..5357be0 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/metrics/v1/metrics_service_marshaler.py @@ -0,0 +1,130 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/metrics/v1/metrics_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportMetricsServiceRequest(MessageMarshaler): + @property + def resource_metrics(self) -> List[ResourceMetrics]: + if self._resource_metrics is None: + self._resource_metrics = list() + return self._resource_metrics + + def __init__( + self, + resource_metrics: List[ResourceMetrics] = None, + ): + self._resource_metrics: List[ResourceMetrics] = resource_metrics + + def calculate_size(self) -> int: + size = 0 + if self._resource_metrics: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_metrics + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_metrics: + for v in self._resource_metrics: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportMetricsServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportMetricsPartialSuccess: + if self._partial_success is None: + self._partial_success = ExportMetricsPartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportMetricsPartialSuccess = None, + ): + self._partial_success: ExportMetricsPartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportMetricsPartialSuccess(MessageMarshaler): + rejected_data_points: int + error_message: str + + def __init__( + self, + rejected_data_points: int = 0, + error_message: str = "", + ): + self.rejected_data_points: int = rejected_data_points + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_data_points: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_data_points) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_data_points: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_data_points) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py new file mode 100644 index 0000000..d3b9dce --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/collector/trace/v1/trace_service_marshaler.py @@ -0,0 +1,130 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/collector/trace/v1/trace_service.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class ExportTraceServiceRequest(MessageMarshaler): + @property + def resource_spans(self) -> List[ResourceSpans]: + if self._resource_spans is None: + self._resource_spans = list() + return self._resource_spans + + def __init__( + self, + resource_spans: List[ResourceSpans] = None, + ): + self._resource_spans: List[ResourceSpans] = resource_spans + + def calculate_size(self) -> int: + size = 0 + if self._resource_spans: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_spans + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_spans: + for v in self._resource_spans: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ExportTraceServiceResponse(MessageMarshaler): + @property + def partial_success(self) -> ExportTracePartialSuccess: + if self._partial_success is None: + self._partial_success = ExportTracePartialSuccess() + return self._partial_success + + def __init__( + self, + partial_success: ExportTracePartialSuccess = None, + ): + self._partial_success: ExportTracePartialSuccess = partial_success + + def calculate_size(self) -> int: + size = 0 + if self._partial_success is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._partial_success._get_size()) + + self._partial_success._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._partial_success is not None: + out += b"\n" + Varint.write_varint_u32(out, self._partial_success._get_size()) + self._partial_success.write_to(out) + + +class ExportTracePartialSuccess(MessageMarshaler): + rejected_spans: int + error_message: str + + def __init__( + self, + rejected_spans: int = 0, + error_message: str = "", + ): + self.rejected_spans: int = rejected_spans + self.error_message: str = error_message + + def calculate_size(self) -> int: + size = 0 + if self.rejected_spans: + size += len(b"\x08") + Varint.size_varint_i64(self.rejected_spans) + if self.error_message: + v = self.error_message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.rejected_spans: + out += b"\x08" + Varint.write_varint_i64(out, self.rejected_spans) + if self.error_message: + v = self.error_message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py new file mode 100644 index 0000000..a1b909e --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/common/v1/common_marshaler.py @@ -0,0 +1,303 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/common/v1/common.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class AnyValue(MessageMarshaler): + string_value: str + bool_value: bool + int_value: int + double_value: float + + @property + def array_value(self) -> ArrayValue: + if self._array_value is None: + self._array_value = ArrayValue() + return self._array_value + + @property + def kvlist_value(self) -> KeyValueList: + if self._kvlist_value is None: + self._kvlist_value = KeyValueList() + return self._kvlist_value + + bytes_value: bytes + + def __init__( + self, + string_value: str = None, + bool_value: bool = None, + int_value: int = None, + double_value: float = None, + array_value: ArrayValue = None, + kvlist_value: KeyValueList = None, + bytes_value: bytes = None, + ): + self.string_value: str = string_value + self.bool_value: bool = bool_value + self.int_value: int = int_value + self.double_value: float = double_value + self._array_value: ArrayValue = array_value + self._kvlist_value: KeyValueList = kvlist_value + self.bytes_value: bytes = bytes_value + + def calculate_size(self) -> int: + size = 0 + if self.string_value is not None: + v = self.string_value.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.bool_value is not None: + size += len(b"\x10") + 1 + if self.int_value is not None: + size += len(b"\x18") + Varint.size_varint_i64(self.int_value) + if self.double_value is not None: + size += len(b"!") + 8 + if self._array_value is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._array_value._get_size()) + + self._array_value._get_size() + ) + if self._kvlist_value is not None: + size += ( + len(b"2") + + Varint.size_varint_u32(self._kvlist_value._get_size()) + + self._kvlist_value._get_size() + ) + if self.bytes_value is not None: + size += ( + len(b":") + + Varint.size_varint_u32(len(self.bytes_value)) + + len(self.bytes_value) + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.string_value is not None: + v = self.string_value.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.bool_value is not None: + out += b"\x10" + Varint.write_varint_u32(out, 1 if self.bool_value else 0) + if self.int_value is not None: + out += b"\x18" + Varint.write_varint_i64(out, self.int_value) + if self.double_value is not None: + out += b"!" + out += struct.pack(" List[AnyValue]: + if self._values is None: + self._values = list() + return self._values + + def __init__( + self, + values: List[AnyValue] = None, + ): + self._values: List[AnyValue] = values + + def calculate_size(self) -> int: + size = 0 + if self._values: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._values + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._values: + for v in self._values: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class KeyValueList(MessageMarshaler): + @property + def values(self) -> List[KeyValue]: + if self._values is None: + self._values = list() + return self._values + + def __init__( + self, + values: List[KeyValue] = None, + ): + self._values: List[KeyValue] = values + + def calculate_size(self) -> int: + size = 0 + if self._values: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._values + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._values: + for v in self._values: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class KeyValue(MessageMarshaler): + key: str + + @property + def value(self) -> AnyValue: + if self._value is None: + self._value = AnyValue() + return self._value + + def __init__( + self, + key: str = "", + value: AnyValue = None, + ): + self.key: str = key + self._value: AnyValue = value + + def calculate_size(self) -> int: + size = 0 + if self.key: + v = self.key.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self._value is not None: + size += ( + len(b"\x12") + + Varint.size_varint_u32(self._value._get_size()) + + self._value._get_size() + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.key: + v = self.key.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self._value is not None: + out += b"\x12" + Varint.write_varint_u32(out, self._value._get_size()) + self._value.write_to(out) + + +class InstrumentationScope(MessageMarshaler): + name: str + version: str + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + name: str = "", + version: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self.name: str = name + self.version: str = version + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.version: + v = self.version.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b"\x1a") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b" ") + Varint.size_varint_u32(self.dropped_attributes_count) + return size + + def write_to(self, out: bytearray) -> None: + if self.name: + v = self.name.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.version: + v = self.version.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self._attributes: + for v in self._attributes: + out += b"\x1a" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b" " + Varint.write_varint_u32(out, self.dropped_attributes_count) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py new file mode 100644 index 0000000..9900914 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/logs/v1/logs_marshaler.py @@ -0,0 +1,361 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/logs/v1/logs.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import * +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class SeverityNumber(Enum): + SEVERITY_NUMBER_UNSPECIFIED = 0 + SEVERITY_NUMBER_TRACE = 1 + SEVERITY_NUMBER_TRACE2 = 2 + SEVERITY_NUMBER_TRACE3 = 3 + SEVERITY_NUMBER_TRACE4 = 4 + SEVERITY_NUMBER_DEBUG = 5 + SEVERITY_NUMBER_DEBUG2 = 6 + SEVERITY_NUMBER_DEBUG3 = 7 + SEVERITY_NUMBER_DEBUG4 = 8 + SEVERITY_NUMBER_INFO = 9 + SEVERITY_NUMBER_INFO2 = 10 + SEVERITY_NUMBER_INFO3 = 11 + SEVERITY_NUMBER_INFO4 = 12 + SEVERITY_NUMBER_WARN = 13 + SEVERITY_NUMBER_WARN2 = 14 + SEVERITY_NUMBER_WARN3 = 15 + SEVERITY_NUMBER_WARN4 = 16 + SEVERITY_NUMBER_ERROR = 17 + SEVERITY_NUMBER_ERROR2 = 18 + SEVERITY_NUMBER_ERROR3 = 19 + SEVERITY_NUMBER_ERROR4 = 20 + SEVERITY_NUMBER_FATAL = 21 + SEVERITY_NUMBER_FATAL2 = 22 + SEVERITY_NUMBER_FATAL3 = 23 + SEVERITY_NUMBER_FATAL4 = 24 + + +class LogRecordFlags(Enum): + LOG_RECORD_FLAGS_DO_NOT_USE = 0 + LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 255 + + +class LogsData(MessageMarshaler): + @property + def resource_logs(self) -> List[ResourceLogs]: + if self._resource_logs is None: + self._resource_logs = list() + return self._resource_logs + + def __init__( + self, + resource_logs: List[ResourceLogs] = None, + ): + self._resource_logs: List[ResourceLogs] = resource_logs + + def calculate_size(self) -> int: + size = 0 + if self._resource_logs: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_logs + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_logs: + for v in self._resource_logs: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceLogs(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_logs(self) -> List[ScopeLogs]: + if self._scope_logs is None: + self._scope_logs = list() + return self._scope_logs + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_logs: List[ScopeLogs] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_logs: List[ScopeLogs] = scope_logs + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_logs: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_logs + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_logs: + for v in self._scope_logs: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeLogs(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def log_records(self) -> List[LogRecord]: + if self._log_records is None: + self._log_records = list() + return self._log_records + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + log_records: List[LogRecord] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._log_records: List[LogRecord] = log_records + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._log_records: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._log_records + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._log_records: + for v in self._log_records: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class LogRecord(MessageMarshaler): + time_unix_nano: int + severity_number: SeverityNumber + severity_text: str + + @property + def body(self) -> AnyValue: + if self._body is None: + self._body = AnyValue() + return self._body + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + flags: int + trace_id: bytes + span_id: bytes + observed_time_unix_nano: int + + def __init__( + self, + time_unix_nano: int = 0, + severity_number: SeverityNumber = 0, + severity_text: str = "", + body: AnyValue = None, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + flags: int = 0, + trace_id: bytes = b"", + span_id: bytes = b"", + observed_time_unix_nano: int = 0, + ): + self.time_unix_nano: int = time_unix_nano + self.severity_number: SeverityNumber = severity_number + self.severity_text: str = severity_text + self._body: AnyValue = body + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self.flags: int = flags + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.observed_time_unix_nano: int = observed_time_unix_nano + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\t") + 8 + if self.severity_number: + v = self.severity_number + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + if self.severity_text: + v = self.severity_text.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._body is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._body._get_size()) + + self._body._get_size() + ) + if self._attributes: + size += sum( + message._get_size() + + len(b"2") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"8") + Varint.size_varint_u32(self.dropped_attributes_count) + if self.flags: + size += len(b"E") + 4 + if self.trace_id: + size += ( + len(b"J") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"R") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.observed_time_unix_nano: + size += len(b"Y") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\t" + out += struct.pack(" List[ResourceMetrics]: + if self._resource_metrics is None: + self._resource_metrics = list() + return self._resource_metrics + + def __init__( + self, + resource_metrics: List[ResourceMetrics] = None, + ): + self._resource_metrics: List[ResourceMetrics] = resource_metrics + + def calculate_size(self) -> int: + size = 0 + if self._resource_metrics: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_metrics + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_metrics: + for v in self._resource_metrics: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceMetrics(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_metrics(self) -> List[ScopeMetrics]: + if self._scope_metrics is None: + self._scope_metrics = list() + return self._scope_metrics + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_metrics: List[ScopeMetrics] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_metrics: List[ScopeMetrics] = scope_metrics + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_metrics: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_metrics + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_metrics: + for v in self._scope_metrics: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeMetrics(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def metrics(self) -> List[Metric]: + if self._metrics is None: + self._metrics = list() + return self._metrics + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + metrics: List[Metric] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._metrics: List[Metric] = metrics + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._metrics: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._metrics + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._metrics: + for v in self._metrics: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class Metric(MessageMarshaler): + name: str + description: str + unit: str + + @property + def gauge(self) -> Gauge: + if self._gauge is None: + self._gauge = Gauge() + return self._gauge + + @property + def sum(self) -> Sum: + if self._sum is None: + self._sum = Sum() + return self._sum + + @property + def histogram(self) -> Histogram: + if self._histogram is None: + self._histogram = Histogram() + return self._histogram + + @property + def exponential_histogram(self) -> ExponentialHistogram: + if self._exponential_histogram is None: + self._exponential_histogram = ExponentialHistogram() + return self._exponential_histogram + + @property + def summary(self) -> Summary: + if self._summary is None: + self._summary = Summary() + return self._summary + + @property + def metadata(self) -> List[KeyValue]: + if self._metadata is None: + self._metadata = list() + return self._metadata + + def __init__( + self, + name: str = "", + description: str = "", + unit: str = "", + gauge: Gauge = None, + sum: Sum = None, + histogram: Histogram = None, + exponential_histogram: ExponentialHistogram = None, + summary: Summary = None, + metadata: List[KeyValue] = None, + ): + self.name: str = name + self.description: str = description + self.unit: str = unit + self._gauge: Gauge = gauge + self._sum: Sum = sum + self._histogram: Histogram = histogram + self._exponential_histogram: ExponentialHistogram = exponential_histogram + self._summary: Summary = summary + self._metadata: List[KeyValue] = metadata + + def calculate_size(self) -> int: + size = 0 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\n") + Varint.size_varint_u32(len(v)) + len(v) + if self.description: + v = self.description.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self.unit: + v = self.unit.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._gauge is not None: + size += ( + len(b"*") + + Varint.size_varint_u32(self._gauge._get_size()) + + self._gauge._get_size() + ) + if self._sum is not None: + size += ( + len(b":") + + Varint.size_varint_u32(self._sum._get_size()) + + self._sum._get_size() + ) + if self._histogram is not None: + size += ( + len(b"J") + + Varint.size_varint_u32(self._histogram._get_size()) + + self._histogram._get_size() + ) + if self._exponential_histogram is not None: + size += ( + len(b"R") + + Varint.size_varint_u32(self._exponential_histogram._get_size()) + + self._exponential_histogram._get_size() + ) + if self._summary is not None: + size += ( + len(b"Z") + + Varint.size_varint_u32(self._summary._get_size()) + + self._summary._get_size() + ) + if self._metadata: + size += sum( + message._get_size() + + len(b"b") + + Varint.size_varint_u32(message._get_size()) + for message in self._metadata + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.name: + v = self.name.encode("utf-8") + out += b"\n" + Varint.write_varint_u32(out, len(v)) + out += v + if self.description: + v = self.description.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self.unit: + v = self.unit.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self._gauge is not None: + out += b"*" + Varint.write_varint_u32(out, self._gauge._get_size()) + self._gauge.write_to(out) + if self._sum is not None: + out += b":" + Varint.write_varint_u32(out, self._sum._get_size()) + self._sum.write_to(out) + if self._histogram is not None: + out += b"J" + Varint.write_varint_u32(out, self._histogram._get_size()) + self._histogram.write_to(out) + if self._exponential_histogram is not None: + out += b"R" + Varint.write_varint_u32(out, self._exponential_histogram._get_size()) + self._exponential_histogram.write_to(out) + if self._summary is not None: + out += b"Z" + Varint.write_varint_u32(out, self._summary._get_size()) + self._summary.write_to(out) + if self._metadata: + for v in self._metadata: + out += b"b" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class Gauge(MessageMarshaler): + @property + def data_points(self) -> List[NumberDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + def __init__( + self, + data_points: List[NumberDataPoint] = None, + ): + self._data_points: List[NumberDataPoint] = data_points + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class Sum(MessageMarshaler): + @property + def data_points(self) -> List[NumberDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + is_monotonic: bool + + def __init__( + self, + data_points: List[NumberDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + is_monotonic: bool = False, + ): + self._data_points: List[NumberDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + self.is_monotonic: bool = is_monotonic + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + if self.is_monotonic: + size += len(b"\x18") + 1 + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + if self.is_monotonic: + out += b"\x18" + Varint.write_varint_u32(out, 1 if self.is_monotonic else 0) + + +class Histogram(MessageMarshaler): + @property + def data_points(self) -> List[HistogramDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + + def __init__( + self, + data_points: List[HistogramDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + ): + self._data_points: List[HistogramDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + + +class ExponentialHistogram(MessageMarshaler): + @property + def data_points(self) -> List[ExponentialHistogramDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + aggregation_temporality: AggregationTemporality + + def __init__( + self, + data_points: List[ExponentialHistogramDataPoint] = None, + aggregation_temporality: AggregationTemporality = 0, + ): + self._data_points: List[ExponentialHistogramDataPoint] = data_points + self.aggregation_temporality: AggregationTemporality = aggregation_temporality + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + size += len(b"\x10") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.aggregation_temporality: + v = self.aggregation_temporality + if not isinstance(v, int): + v = v.value + out += b"\x10" + Varint.write_varint_u32(out, v) + + +class Summary(MessageMarshaler): + @property + def data_points(self) -> List[SummaryDataPoint]: + if self._data_points is None: + self._data_points = list() + return self._data_points + + def __init__( + self, + data_points: List[SummaryDataPoint] = None, + ): + self._data_points: List[SummaryDataPoint] = data_points + + def calculate_size(self) -> int: + size = 0 + if self._data_points: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._data_points + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._data_points: + for v in self._data_points: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class NumberDataPoint(MessageMarshaler): + start_time_unix_nano: int + time_unix_nano: int + as_double: float + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + as_int: int + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + as_double: float = None, + exemplars: List[Exemplar] = None, + as_int: int = None, + attributes: List[KeyValue] = None, + flags: int = 0, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.as_double: float = as_double + self._exemplars: List[Exemplar] = exemplars + self.as_int: int = as_int + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.as_double is not None: + size += len(b"!") + 8 + if self._exemplars: + size += sum( + message._get_size() + + len(b"*") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self.as_int is not None: + size += len(b"1") + 8 + if self._attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"@") + Varint.size_varint_u32(self.flags) + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[int]: + if self._bucket_counts is None: + self._bucket_counts = list() + return self._bucket_counts + + @property + def explicit_bounds(self) -> List[float]: + if self._explicit_bounds is None: + self._explicit_bounds = list() + return self._explicit_bounds + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + min: float + max: float + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = None, + bucket_counts: List[int] = None, + explicit_bounds: List[float] = None, + exemplars: List[Exemplar] = None, + attributes: List[KeyValue] = None, + flags: int = 0, + min: float = None, + max: float = None, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self._bucket_counts: List[int] = bucket_counts + self._explicit_bounds: List[float] = explicit_bounds + self._exemplars: List[Exemplar] = exemplars + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + self.min: float = min + self.max: float = max + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum is not None: + size += len(b")") + 8 + if self._bucket_counts: + size += ( + len(b"2") + + len(self._bucket_counts) * 8 + + Varint.size_varint_u32(len(self._bucket_counts) * 8) + ) + if self._explicit_bounds: + size += ( + len(b":") + + len(self._explicit_bounds) * 8 + + Varint.size_varint_u32(len(self._explicit_bounds) * 8) + ) + if self._exemplars: + size += sum( + message._get_size() + + len(b"B") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self._attributes: + size += sum( + message._get_size() + + len(b"J") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"P") + Varint.size_varint_u32(self.flags) + if self.min is not None: + size += len(b"Y") + 8 + if self.max is not None: + size += len(b"a") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + start_time_unix_nano: int + time_unix_nano: int + count: int + sum: float + scale: int + zero_count: int + + @property + def positive(self) -> ExponentialHistogramDataPoint.Buckets: + if self._positive is None: + self._positive = ExponentialHistogramDataPoint.Buckets() + return self._positive + + @property + def negative(self) -> ExponentialHistogramDataPoint.Buckets: + if self._negative is None: + self._negative = ExponentialHistogramDataPoint.Buckets() + return self._negative + + flags: int + + @property + def exemplars(self) -> List[Exemplar]: + if self._exemplars is None: + self._exemplars = list() + return self._exemplars + + min: float + max: float + zero_threshold: float + + def __init__( + self, + attributes: List[KeyValue] = None, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = None, + scale: int = 0, + zero_count: int = 0, + positive: ExponentialHistogramDataPoint.Buckets = None, + negative: ExponentialHistogramDataPoint.Buckets = None, + flags: int = 0, + exemplars: List[Exemplar] = None, + min: float = None, + max: float = None, + zero_threshold: float = 0.0, + ): + self._attributes: List[KeyValue] = attributes + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self.scale: int = scale + self.zero_count: int = zero_count + self._positive: ExponentialHistogramDataPoint.Buckets = positive + self._negative: ExponentialHistogramDataPoint.Buckets = negative + self.flags: int = flags + self._exemplars: List[Exemplar] = exemplars + self.min: float = min + self.max: float = max + self.zero_threshold: float = zero_threshold + + def calculate_size(self) -> int: + size = 0 + if self._attributes: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum is not None: + size += len(b")") + 8 + if self.scale: + size += len(b"0") + Varint.size_varint_s32(self.scale) + if self.zero_count: + size += len(b"9") + 8 + if self._positive is not None: + size += ( + len(b"B") + + Varint.size_varint_u32(self._positive._get_size()) + + self._positive._get_size() + ) + if self._negative is not None: + size += ( + len(b"J") + + Varint.size_varint_u32(self._negative._get_size()) + + self._negative._get_size() + ) + if self.flags: + size += len(b"P") + Varint.size_varint_u32(self.flags) + if self._exemplars: + size += sum( + message._get_size() + + len(b"Z") + + Varint.size_varint_u32(message._get_size()) + for message in self._exemplars + ) + if self.min is not None: + size += len(b"a") + 8 + if self.max is not None: + size += len(b"i") + 8 + if self.zero_threshold: + size += len(b"q") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self._attributes: + for v in self._attributes: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" List[int]: + if self._bucket_counts is None: + self._bucket_counts = list() + return self._bucket_counts + + def __init__( + self, + offset: int = 0, + bucket_counts: List[int] = None, + ): + self.offset: int = offset + self._bucket_counts: List[int] = bucket_counts + + def calculate_size(self) -> int: + size = 0 + if self.offset: + size += len(b"\x08") + Varint.size_varint_s32(self.offset) + if self._bucket_counts: + s = sum( + Varint.size_varint_u64(uint64) for uint64 in self._bucket_counts + ) + self.marshaler_cache[b"\x12"] = s + size += len(b"\x12") + s + Varint.size_varint_u32(s) + return size + + def write_to(self, out: bytearray) -> None: + if self.offset: + out += b"\x08" + Varint.write_varint_s32(out, self.offset) + if self._bucket_counts: + out += b"\x12" + Varint.write_varint_u32(out, self.marshaler_cache[b"\x12"]) + for v in self._bucket_counts: + Varint.write_varint_u64(out, v) + + +class SummaryDataPoint(MessageMarshaler): + start_time_unix_nano: int + time_unix_nano: int + count: int + sum: float + + @property + def quantile_values(self) -> List[SummaryDataPoint.ValueAtQuantile]: + if self._quantile_values is None: + self._quantile_values = list() + return self._quantile_values + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + flags: int + + def __init__( + self, + start_time_unix_nano: int = 0, + time_unix_nano: int = 0, + count: int = 0, + sum: float = 0.0, + quantile_values: List[SummaryDataPoint.ValueAtQuantile] = None, + attributes: List[KeyValue] = None, + flags: int = 0, + ): + self.start_time_unix_nano: int = start_time_unix_nano + self.time_unix_nano: int = time_unix_nano + self.count: int = count + self.sum: float = sum + self._quantile_values: List[SummaryDataPoint.ValueAtQuantile] = quantile_values + self._attributes: List[KeyValue] = attributes + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.start_time_unix_nano: + size += len(b"\x11") + 8 + if self.time_unix_nano: + size += len(b"\x19") + 8 + if self.count: + size += len(b"!") + 8 + if self.sum: + size += len(b")") + 8 + if self._quantile_values: + size += sum( + message._get_size() + + len(b"2") + + Varint.size_varint_u32(message._get_size()) + for message in self._quantile_values + ) + if self._attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.flags: + size += len(b"@") + Varint.size_varint_u32(self.flags) + return size + + def write_to(self, out: bytearray) -> None: + if self.start_time_unix_nano: + out += b"\x11" + out += struct.pack(" int: + size = 0 + if self.quantile: + size += len(b"\t") + 8 + if self.value: + size += len(b"\x11") + 8 + return size + + def write_to(self, out: bytearray) -> None: + if self.quantile: + out += b"\t" + out += struct.pack(" List[KeyValue]: + if self._filtered_attributes is None: + self._filtered_attributes = list() + return self._filtered_attributes + + def __init__( + self, + time_unix_nano: int = 0, + as_double: float = None, + span_id: bytes = b"", + trace_id: bytes = b"", + as_int: int = None, + filtered_attributes: List[KeyValue] = None, + ): + self.time_unix_nano: int = time_unix_nano + self.as_double: float = as_double + self.span_id: bytes = span_id + self.trace_id: bytes = trace_id + self.as_int: int = as_int + self._filtered_attributes: List[KeyValue] = filtered_attributes + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\x11") + 8 + if self.as_double is not None: + size += len(b"\x19") + 8 + if self.span_id: + size += ( + len(b'"') + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_id: + size += ( + len(b"*") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.as_int is not None: + size += len(b"1") + 8 + if self._filtered_attributes: + size += sum( + message._get_size() + + len(b":") + + Varint.size_varint_u32(message._get_size()) + for message in self._filtered_attributes + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\x11" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self._attributes: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"\x10") + Varint.size_varint_u32(self.dropped_attributes_count) + return size + + def write_to(self, out: bytearray) -> None: + if self._attributes: + for v in self._attributes: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b"\x10" + Varint.write_varint_u32(out, self.dropped_attributes_count) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py new file mode 100644 index 0000000..748f670 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/proto/trace/v1/trace_marshaler.py @@ -0,0 +1,619 @@ +# Generated by the protoc compiler with a custom plugin. DO NOT EDIT! +# sources: opentelemetry/proto/trace/v1/trace.proto +# +# Copyright (c) 2012-2024 Snowflake Inc. All rights reserved. +# +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been generated from the original proto schema at +# +# https://github.com/open-telemetry/opentelemetry-proto +# +# using a custom protoc compiler plugin by Snowflake Inc. + +from __future__ import annotations + +import struct +from typing import List + +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import * +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import * +from snowflake.telemetry._internal.serialize import ( + Enum, + MessageMarshaler, + Varint, +) + + +class SpanFlags(Enum): + SPAN_FLAGS_DO_NOT_USE = 0 + SPAN_FLAGS_TRACE_FLAGS_MASK = 255 + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 256 + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 512 + + +class TracesData(MessageMarshaler): + @property + def resource_spans(self) -> List[ResourceSpans]: + if self._resource_spans is None: + self._resource_spans = list() + return self._resource_spans + + def __init__( + self, + resource_spans: List[ResourceSpans] = None, + ): + self._resource_spans: List[ResourceSpans] = resource_spans + + def calculate_size(self) -> int: + size = 0 + if self._resource_spans: + size += sum( + message._get_size() + + len(b"\n") + + Varint.size_varint_u32(message._get_size()) + for message in self._resource_spans + ) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource_spans: + for v in self._resource_spans: + out += b"\n" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + +class ResourceSpans(MessageMarshaler): + @property + def resource(self) -> Resource: + if self._resource is None: + self._resource = Resource() + return self._resource + + @property + def scope_spans(self) -> List[ScopeSpans]: + if self._scope_spans is None: + self._scope_spans = list() + return self._scope_spans + + schema_url: str + + def __init__( + self, + resource: Resource = None, + scope_spans: List[ScopeSpans] = None, + schema_url: str = "", + ): + self._resource: Resource = resource + self._scope_spans: List[ScopeSpans] = scope_spans + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._resource is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._resource._get_size()) + + self._resource._get_size() + ) + if self._scope_spans: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._scope_spans + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._resource is not None: + out += b"\n" + Varint.write_varint_u32(out, self._resource._get_size()) + self._resource.write_to(out) + if self._scope_spans: + for v in self._scope_spans: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class ScopeSpans(MessageMarshaler): + @property + def scope(self) -> InstrumentationScope: + if self._scope is None: + self._scope = InstrumentationScope() + return self._scope + + @property + def spans(self) -> List[Span]: + if self._spans is None: + self._spans = list() + return self._spans + + schema_url: str + + def __init__( + self, + scope: InstrumentationScope = None, + spans: List[Span] = None, + schema_url: str = "", + ): + self._scope: InstrumentationScope = scope + self._spans: List[Span] = spans + self.schema_url: str = schema_url + + def calculate_size(self) -> int: + size = 0 + if self._scope is not None: + size += ( + len(b"\n") + + Varint.size_varint_u32(self._scope._get_size()) + + self._scope._get_size() + ) + if self._spans: + size += sum( + message._get_size() + + len(b"\x12") + + Varint.size_varint_u32(message._get_size()) + for message in self._spans + ) + if self.schema_url: + v = self.schema_url.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + return size + + def write_to(self, out: bytearray) -> None: + if self._scope is not None: + out += b"\n" + Varint.write_varint_u32(out, self._scope._get_size()) + self._scope.write_to(out) + if self._spans: + for v in self._spans: + out += b"\x12" + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.schema_url: + v = self.schema_url.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + + +class Span(MessageMarshaler): + trace_id: bytes + span_id: bytes + trace_state: str + parent_span_id: bytes + name: str + kind: Span.SpanKind + start_time_unix_nano: int + end_time_unix_nano: int + + @property + def attributes(self) -> List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + @property + def events(self) -> List[Span.Event]: + if self._events is None: + self._events = list() + return self._events + + dropped_events_count: int + + @property + def links(self) -> List[Span.Link]: + if self._links is None: + self._links = list() + return self._links + + dropped_links_count: int + + @property + def status(self) -> Status: + if self._status is None: + self._status = Status() + return self._status + + flags: int + + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + parent_span_id: bytes = b"", + name: str = "", + kind: Span.SpanKind = 0, + start_time_unix_nano: int = 0, + end_time_unix_nano: int = 0, + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + events: List[Span.Event] = None, + dropped_events_count: int = 0, + links: List[Span.Link] = None, + dropped_links_count: int = 0, + status: Status = None, + flags: int = 0, + ): + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.trace_state: str = trace_state + self.parent_span_id: bytes = parent_span_id + self.name: str = name + self.kind: Span.SpanKind = kind + self.start_time_unix_nano: int = start_time_unix_nano + self.end_time_unix_nano: int = end_time_unix_nano + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self._events: List[Span.Event] = events + self.dropped_events_count: int = dropped_events_count + self._links: List[Span.Link] = links + self.dropped_links_count: int = dropped_links_count + self._status: Status = status + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.trace_id: + size += ( + len(b"\n") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"\x12") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_state: + v = self.trace_state.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self.parent_span_id: + size += ( + len(b'"') + + Varint.size_varint_u32(len(self.parent_span_id)) + + len(self.parent_span_id) + ) + if self.name: + v = self.name.encode("utf-8") + size += len(b"*") + Varint.size_varint_u32(len(v)) + len(v) + if self.kind: + v = self.kind + if not isinstance(v, int): + v = v.value + size += len(b"0") + Varint.size_varint_u32(v) + if self.start_time_unix_nano: + size += len(b"9") + 8 + if self.end_time_unix_nano: + size += len(b"A") + 8 + if self._attributes: + size += sum( + message._get_size() + + len(b"J") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"P") + Varint.size_varint_u32(self.dropped_attributes_count) + if self._events: + size += sum( + message._get_size() + + len(b"Z") + + Varint.size_varint_u32(message._get_size()) + for message in self._events + ) + if self.dropped_events_count: + size += len(b"`") + Varint.size_varint_u32(self.dropped_events_count) + if self._links: + size += sum( + message._get_size() + + len(b"j") + + Varint.size_varint_u32(message._get_size()) + for message in self._links + ) + if self.dropped_links_count: + size += len(b"p") + Varint.size_varint_u32(self.dropped_links_count) + if self._status is not None: + size += ( + len(b"z") + + Varint.size_varint_u32(self._status._get_size()) + + self._status._get_size() + ) + if self.flags: + size += len(b"\x85\x01") + 4 + return size + + def write_to(self, out: bytearray) -> None: + if self.trace_id: + out += b"\n" + Varint.write_varint_u32(out, len(self.trace_id)) + out += self.trace_id + if self.span_id: + out += b"\x12" + Varint.write_varint_u32(out, len(self.span_id)) + out += self.span_id + if self.trace_state: + v = self.trace_state.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self.parent_span_id: + out += b'"' + Varint.write_varint_u32(out, len(self.parent_span_id)) + out += self.parent_span_id + if self.name: + v = self.name.encode("utf-8") + out += b"*" + Varint.write_varint_u32(out, len(v)) + out += v + if self.kind: + v = self.kind + if not isinstance(v, int): + v = v.value + out += b"0" + Varint.write_varint_u32(out, v) + if self.start_time_unix_nano: + out += b"9" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + + def __init__( + self, + time_unix_nano: int = 0, + name: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + ): + self.time_unix_nano: int = time_unix_nano + self.name: str = name + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + + def calculate_size(self) -> int: + size = 0 + if self.time_unix_nano: + size += len(b"\t") + 8 + if self.name: + v = self.name.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b"\x1a") + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b" ") + Varint.size_varint_u32( + self.dropped_attributes_count + ) + return size + + def write_to(self, out: bytearray) -> None: + if self.time_unix_nano: + out += b"\t" + out += struct.pack(" List[KeyValue]: + if self._attributes is None: + self._attributes = list() + return self._attributes + + dropped_attributes_count: int + flags: int + + def __init__( + self, + trace_id: bytes = b"", + span_id: bytes = b"", + trace_state: str = "", + attributes: List[KeyValue] = None, + dropped_attributes_count: int = 0, + flags: int = 0, + ): + self.trace_id: bytes = trace_id + self.span_id: bytes = span_id + self.trace_state: str = trace_state + self._attributes: List[KeyValue] = attributes + self.dropped_attributes_count: int = dropped_attributes_count + self.flags: int = flags + + def calculate_size(self) -> int: + size = 0 + if self.trace_id: + size += ( + len(b"\n") + + Varint.size_varint_u32(len(self.trace_id)) + + len(self.trace_id) + ) + if self.span_id: + size += ( + len(b"\x12") + + Varint.size_varint_u32(len(self.span_id)) + + len(self.span_id) + ) + if self.trace_state: + v = self.trace_state.encode("utf-8") + size += len(b"\x1a") + Varint.size_varint_u32(len(v)) + len(v) + if self._attributes: + size += sum( + message._get_size() + + len(b'"') + + Varint.size_varint_u32(message._get_size()) + for message in self._attributes + ) + if self.dropped_attributes_count: + size += len(b"(") + Varint.size_varint_u32( + self.dropped_attributes_count + ) + if self.flags: + size += len(b"5") + 4 + return size + + def write_to(self, out: bytearray) -> None: + if self.trace_id: + out += b"\n" + Varint.write_varint_u32(out, len(self.trace_id)) + out += self.trace_id + if self.span_id: + out += b"\x12" + Varint.write_varint_u32(out, len(self.span_id)) + out += self.span_id + if self.trace_state: + v = self.trace_state.encode("utf-8") + out += b"\x1a" + Varint.write_varint_u32(out, len(v)) + out += v + if self._attributes: + for v in self._attributes: + out += b'"' + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + if self.dropped_attributes_count: + out += b"(" + Varint.write_varint_u32(out, self.dropped_attributes_count) + if self.flags: + out += b"5" + out += struct.pack(" int: + size = 0 + if self.message: + v = self.message.encode("utf-8") + size += len(b"\x12") + Varint.size_varint_u32(len(v)) + len(v) + if self.code: + v = self.code + if not isinstance(v, int): + v = v.value + size += len(b"\x18") + Varint.size_varint_u32(v) + return size + + def write_to(self, out: bytearray) -> None: + if self.message: + v = self.message.encode("utf-8") + out += b"\x12" + Varint.write_varint_u32(out, len(v)) + out += v + if self.code: + v = self.code + if not isinstance(v, int): + v = v.value + out += b"\x18" + Varint.write_varint_u32(out, v) + + class StatusCode(Enum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 diff --git a/src/snowflake/telemetry/_internal/serialize/__init__.py b/src/snowflake/telemetry/_internal/serialize/__init__.py new file mode 100644 index 0000000..89e2da8 --- /dev/null +++ b/src/snowflake/telemetry/_internal/serialize/__init__.py @@ -0,0 +1,294 @@ +# +# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. +# + +from __future__ import annotations + +import struct +from enum import IntEnum +from typing import List, Union, Dict, Any + +# Alias Enum to IntEnum +Enum = IntEnum + +# Static class to handle varint encoding +# There is code duplication for performance reasons +# https://developers.google.com/protocol-buffers/docs/encoding#varints +class Varint: + @staticmethod + def size_varint_u32(value: int) -> int: + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + size_varint_u64 = size_varint_u32 + + @staticmethod + def size_varint_i32(value: int) -> int: + value = value + (1 << 32) if value < 0 else value + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + @staticmethod + def size_varint_i64(value: int) -> int: + value = value + (1 << 64) if value < 0 else value + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + @staticmethod + def size_varint_s32(value: int) -> int: + value = value << 1 if value >= 0 else (value << 1) ^ (~0) + size = 1 + while value >= 128: + value >>= 7 + size += 1 + return size + + size_varint_s64 = size_varint_s32 + + @staticmethod + def write_varint_u32(out: bytearray, value: int) -> None: + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + write_varint_u64 = write_varint_u32 + + @staticmethod + def write_varint_i32(out: bytearray, value: int) -> None: + value = value + (1 << 32) if value < 0 else value + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + @staticmethod + def write_varint_i64(out: bytearray, value: int) -> None: + value = value + (1 << 64) if value < 0 else value + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + @staticmethod + def write_varint_s32(out: bytearray, value: int) -> None: + value = value << 1 if value >= 0 else (value << 1) ^ (~0) + while value >= 128: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + + write_varint_s64 = write_varint_s32 + +# Base class for all custom messages +class MessageMarshaler: + # There is a high overhead for creating an empty dict + # For this reason, the cache dict is lazily initialized + @property + def marshaler_cache(self) -> Dict[bytes, Any]: + if not hasattr(self, "_marshaler_cache"): + self._marshaler_cache = {} + return self._marshaler_cache + + def write_to(self, out: bytearray) -> None: + ... + + def calculate_size(self) -> int: + ... + + def _get_size(self) -> int: + if not hasattr(self, "_size"): + self._size = self.calculate_size() + return self._size + + def SerializeToString(self) -> bytes: + # size MUST be calculated before serializing since some preprocessing is done + self._get_size() + stream = bytearray() + self.write_to(stream) + return bytes(stream) + + def __bytes__(self) -> bytes: + return self.SerializeToString() + + # The following size and serialize functions may be inlined by the code generator + # The following strings are replaced by the code generator for inlining: + # - TAG + # - FIELD_ATTR + + def size_bool(self, TAG: bytes, _) -> int: + return len(TAG) + 1 + + def size_enum(self, TAG: bytes, FIELD_ATTR: Union[Enum, int]) -> int: + v = FIELD_ATTR + if not isinstance(v, int): + v = v.value + return len(TAG) + Varint.size_varint_u32(v) + + def size_uint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR) + + def size_uint64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_u64(FIELD_ATTR) + + def size_sint32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_s32(FIELD_ATTR) + + def size_sint64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_s64(FIELD_ATTR) + + def size_int32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_i32(FIELD_ATTR) + + def size_int64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + Varint.size_varint_i64(FIELD_ATTR) + + def size_float(self, TAG: bytes, FIELD_ATTR: float) -> int: + return len(TAG) + 4 + + def size_double(self, TAG: bytes, FIELD_ATTR: float) -> int: + return len(TAG) + 8 + + def size_fixed32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 4 + + def size_fixed64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 8 + + def size_sfixed32(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 4 + + def size_sfixed64(self, TAG: bytes, FIELD_ATTR: int) -> int: + return len(TAG) + 8 + + def size_bytes(self, TAG: bytes, FIELD_ATTR: bytes) -> int: + return len(TAG) + Varint.size_varint_u32(len(FIELD_ATTR)) + len(FIELD_ATTR) + + def size_string(self, TAG: bytes, FIELD_ATTR: str) -> int: + v = FIELD_ATTR.encode("utf-8") + return len(TAG) + Varint.size_varint_u32(len(v)) + len(v) + + def size_message(self, TAG: bytes, FIELD_ATTR: MessageMarshaler) -> int: + return len(TAG) + Varint.size_varint_u32(FIELD_ATTR._get_size()) + FIELD_ATTR._get_size() + + def size_repeated_message(self, TAG: bytes, FIELD_ATTR: List[MessageMarshaler]) -> int: + return sum(message._get_size() + len(TAG) + Varint.size_varint_u32(message._get_size()) for message in FIELD_ATTR) + + def size_repeated_double(self, TAG: bytes, FIELD_ATTR: List[float]): + return len(TAG) + len(FIELD_ATTR) * 8 + Varint.size_varint_u32(len(FIELD_ATTR) * 8) + + def size_repeated_fixed64(self, TAG: bytes, FIELD_ATTR: List[int]): + return len(TAG) + len(FIELD_ATTR) * 8 + Varint.size_varint_u32(len(FIELD_ATTR) * 8) + + def size_repeated_uint64(self, TAG: bytes, FIELD_ATTR: List[int]): + s = sum(Varint.size_varint_u64(uint64) for uint64 in FIELD_ATTR) + self.marshaler_cache[TAG] = s + return len(TAG) + s + Varint.size_varint_u32(s) + + def serialize_bool(self, out: bytearray, TAG: bytes, FIELD_ATTR: bool) -> None: + out += TAG + Varint.write_varint_u32(out, 1 if FIELD_ATTR else 0) + + def serialize_enum(self, out: bytearray, TAG: bytes, FIELD_ATTR: Union[Enum, int]) -> None: + v = FIELD_ATTR + if not isinstance(v, int): + v = v.value + out += TAG + Varint.write_varint_u32(out, v) + + def serialize_uint32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_u32(out, FIELD_ATTR) + + def serialize_uint64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_u64(out, FIELD_ATTR) + + def serialize_sint32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_s32(out, FIELD_ATTR) + + def serialize_sint64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_s64(out, FIELD_ATTR) + + def serialize_int32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_i32(out, FIELD_ATTR) + + def serialize_int64(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + Varint.write_varint_i64(out, FIELD_ATTR) + + def serialize_fixed32(self, out: bytearray, TAG: bytes, FIELD_ATTR: int) -> None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR)) + out += FIELD_ATTR + + def serialize_string(self, out: bytearray, TAG: bytes, FIELD_ATTR: str) -> None: + v = FIELD_ATTR.encode("utf-8") + out += TAG + Varint.write_varint_u32(out, len(v)) + out += v + + def serialize_message(self, out: bytearray, TAG: bytes, FIELD_ATTR: MessageMarshaler) -> None: + out += TAG + Varint.write_varint_u32(out, FIELD_ATTR._get_size()) + FIELD_ATTR.write_to(out) + + def serialize_repeated_message(self, out: bytearray, TAG: bytes, FIELD_ATTR: List[MessageMarshaler]) -> None: + for v in FIELD_ATTR: + out += TAG + Varint.write_varint_u32(out, v._get_size()) + v.write_to(out) + + def serialize_repeated_double(self, out: bytearray, TAG: bytes, FIELD_ATTR: List[float]) -> None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR) * 8) + for v in FIELD_ATTR: + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, len(FIELD_ATTR) * 8) + for v in FIELD_ATTR: + out += struct.pack(" None: + out += TAG + Varint.write_varint_u32(out, self.marshaler_cache[TAG]) + for v in FIELD_ATTR: + Varint.write_varint_u64(out, v) diff --git a/src/snowflake/telemetry/version.py b/src/snowflake/telemetry/version.py index 8eeb137..2c851d2 100644 --- a/src/snowflake/telemetry/version.py +++ b/src/snowflake/telemetry/version.py @@ -4,4 +4,4 @@ # """Update this for the versions.""" -VERSION = "0.5.0" +VERSION = "0.6.0.dev" diff --git a/tests/snowflake-telemetry-test-utils/setup.py b/tests/snowflake-telemetry-test-utils/setup.py index a1a6f65..6254955 100644 --- a/tests/snowflake-telemetry-test-utils/setup.py +++ b/tests/snowflake-telemetry-test-utils/setup.py @@ -15,8 +15,15 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, install_requires=[ + "opentelemetry-exporter-otlp-proto-common == 1.26.0", "pytest >= 7.0.0", - "snowflake-telemetry-python == 0.5.0", + "snowflake-telemetry-python == 0.6.0.dev", + "Jinja2 == 3.1.4", + "grpcio-tools >= 1.62.3", + "black >= 24.1.0", + "isort >= 5.12.0", + "hypothesis >= 6.0.0", + "google-benchmark", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_log_encoder.py b/tests/test_log_encoder.py index cf189d6..dcf0e4e 100644 --- a/tests/test_log_encoder.py +++ b/tests/test_log_encoder.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest -from typing import List, Tuple +from typing import Sequence, Tuple from opentelemetry._logs import SeverityNumber from opentelemetry.exporter.otlp.proto.common._internal import ( @@ -22,7 +22,7 @@ _encode_trace_id, _encode_value, ) -from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( ExportLogsServiceRequest, ) @@ -40,8 +40,7 @@ from opentelemetry.proto.resource.v1.resource_pb2 import ( Resource as PB2Resource, ) -from opentelemetry.sdk._logs import LogData, LogLimits -from opentelemetry.sdk._logs import LogRecord as SDKLogRecord +from opentelemetry.sdk._logs import LogData, LogLimits, LogRecord as SDKLogRecord from opentelemetry.sdk.resources import Resource as SDKResource from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.trace import TraceFlags @@ -56,7 +55,7 @@ class TestOTLPLogEncoder(unittest.TestCase): def test_encode(self): sdk_logs, expected_encoding = self.get_test_logs() - self.assertEqual(encode_logs(sdk_logs), expected_encoding) + self.assertEqual(encode_logs(sdk_logs).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_log_exporter(self): sdk_logs, expected_encoding = self.get_test_logs() @@ -70,11 +69,13 @@ def test_proto_log_exporter(self): def test_dropped_attributes_count(self): sdk_logs = self._get_test_logs_dropped_attributes() - encoded_logs = encode_logs(sdk_logs) + encoded_logs = bytes(encode_logs(sdk_logs)) + decoded_logs = PB2LogsData() + decoded_logs.ParseFromString(encoded_logs) self.assertTrue(hasattr(sdk_logs[0].log_record, "dropped_attributes")) self.assertEqual( # pylint:disable=no-member - encoded_logs.resource_logs[0] + decoded_logs.resource_logs[0] .scope_logs[0] .log_records[0] .dropped_attributes_count, @@ -82,10 +83,11 @@ def test_dropped_attributes_count(self): ) @staticmethod - def _get_sdk_log_data() -> List[LogData]: + def _get_sdk_log_data() -> Sequence[LogData]: log1 = LogData( log_record=SDKLogRecord( timestamp=1644650195189786880, + observed_timestamp=1644660000000000000, trace_id=89564621134313219400156819398935297684, span_id=1312458408527513268, trace_flags=TraceFlags(0x01), @@ -106,6 +108,7 @@ def _get_sdk_log_data() -> List[LogData]: log2 = LogData( log_record=SDKLogRecord( timestamp=1644650249738562048, + observed_timestamp=1644660000000000000, trace_id=0, span_id=0, trace_flags=TraceFlags.DEFAULT, @@ -123,6 +126,7 @@ def _get_sdk_log_data() -> List[LogData]: log3 = LogData( log_record=SDKLogRecord( timestamp=1644650427658989056, + observed_timestamp=1644660000000000000, trace_id=271615924622795969659406376515024083555, span_id=4242561578944770265, trace_flags=TraceFlags(0x01), @@ -138,6 +142,7 @@ def _get_sdk_log_data() -> List[LogData]: log4 = LogData( log_record=SDKLogRecord( timestamp=1644650584292683008, + observed_timestamp=1644660000000000000, trace_id=212592107417388365804938480559624925555, span_id=6077757853989569223, trace_flags=TraceFlags(0x01), @@ -159,7 +164,7 @@ def _get_sdk_log_data() -> List[LogData]: def get_test_logs( self, - ) -> Tuple[List[SDKLogRecord], ExportLogsServiceRequest]: + ) -> Tuple[Sequence[LogData], ExportLogsServiceRequest]: sdk_logs = self._get_sdk_log_data() pb2_service_request = ExportLogsServiceRequest( @@ -181,13 +186,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650195189786880, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 89564621134313219400156819398935297684 ), span_id=_encode_span_id( 1312458408527513268 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="WARN", severity_number=SeverityNumber.WARN.value, body=_encode_value( @@ -207,13 +213,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650584292683008, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 212592107417388365804938480559624925555 ), span_id=_encode_span_id( 6077757853989569223 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="INFO", severity_number=SeverityNumber.INFO.value, body=_encode_value( @@ -249,9 +256,8 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650249738562048, - trace_id=_encode_trace_id(0), - span_id=_encode_span_id(0), - flags=int(TraceFlags.DEFAULT), + observed_time_unix_nano=1644660000000000000, + flags=int(0x00), severity_text="WARN", severity_number=SeverityNumber.WARN.value, body=_encode_value( @@ -266,13 +272,14 @@ def get_test_logs( log_records=[ PB2LogRecord( time_unix_nano=1644650427658989056, + observed_time_unix_nano=1644660000000000000, trace_id=_encode_trace_id( 271615924622795969659406376515024083555 ), span_id=_encode_span_id( 4242561578944770265 ), - flags=int(TraceFlags(0x01)), + flags=int(0x01), severity_text="DEBUG", severity_number=SeverityNumber.DEBUG.value, body=_encode_value("To our galaxy"), @@ -290,7 +297,7 @@ def get_test_logs( return sdk_logs, pb2_service_request @staticmethod - def _get_test_logs_dropped_attributes() -> List[LogData]: + def _get_test_logs_dropped_attributes() -> Sequence[LogData]: log1 = LogData( log_record=SDKLogRecord( timestamp=1644650195189786880, diff --git a/tests/test_metrics_encoder.py b/tests/test_metrics_encoder.py index bb4b1d8..7a473ed 100644 --- a/tests/test_metrics_encoder.py +++ b/tests/test_metrics_encoder.py @@ -15,7 +15,7 @@ # pylint: disable=protected-access import unittest -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( @@ -33,10 +33,8 @@ from opentelemetry.sdk.metrics.export import AggregationTemporality, Buckets from opentelemetry.sdk.metrics.export import ( ExponentialHistogram as ExponentialHistogramType, -) -from opentelemetry.sdk.metrics.export import ExponentialHistogramDataPoint -from opentelemetry.sdk.metrics.export import Histogram as HistogramType -from opentelemetry.sdk.metrics.export import ( + Histogram as HistogramType, + ExponentialHistogramDataPoint, HistogramDataPoint, Metric, MetricsData, @@ -161,8 +159,8 @@ def test_encode_sum_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -247,8 +245,8 @@ def test_encode_sum_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -330,8 +328,8 @@ def test_encode_gauge_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -413,8 +411,8 @@ def test_encode_gauge_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -504,8 +502,8 @@ def test_encode_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -732,8 +730,8 @@ def test_encode_multiple_scope_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -858,8 +856,8 @@ def test_encode_exponential_histogram(self): ] ) # pylint: disable=protected-access - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() diff --git a/tests/test_proto_serialization.py b/tests/test_proto_serialization.py new file mode 100644 index 0000000..5e3458f --- /dev/null +++ b/tests/test_proto_serialization.py @@ -0,0 +1,501 @@ +from __future__ import annotations + +from typing import ( + Any, + Dict, + List, + Mapping, +) +import unittest +import hypothesis +import hypothesis.control as hc +import hypothesis.strategies as st + +import opentelemetry.proto.logs.v1.logs_pb2 as logs_pb2 +import opentelemetry.proto.trace.v1.trace_pb2 as trace_pb2 +import opentelemetry.proto.common.v1.common_pb2 as common_pb2 +import opentelemetry.proto.metrics.v1.metrics_pb2 as metrics_pb2 +import opentelemetry.proto.resource.v1.resource_pb2 as resource_pb2 + +import snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler as logs_sf +import snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler as trace_sf +import snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler as common_sf +import snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler as metrics_sf +import snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler as resource_sf + +# Strategy for generating protobuf types +def nullable(type): return st.one_of(st.none(), type) +def pb_uint32(): return nullable(st.integers(min_value=0, max_value=2**32-1)) +def pb_uint64(): return nullable(st.integers(min_value=0, max_value=2**64-1)) +def pb_int32(): return nullable(st.integers(min_value=-2**31, max_value=2**31-1)) +def pb_int64(): return nullable(st.integers(min_value=-2**63, max_value=2**63-1)) +def pb_sint32(): return nullable(st.integers(min_value=-2**31, max_value=2**31-1)) +def pb_sint64(): return nullable(st.integers(min_value=-2**63, max_value=2**63-1)) +def pb_float(): return nullable(st.floats(allow_nan=False, allow_infinity=False, width=32)) +def pb_double(): return nullable(st.floats(allow_nan=False, allow_infinity=False, width=64)) +def draw_pb_double(draw): + # -0.0 is an edge case that is not handled by the custom serialization library + double = draw(pb_double()) + hc.assume(str(double) != "-0.0") + return double +def pb_fixed64(): return pb_uint64() +def pb_fixed32(): return pb_uint32() +def pb_sfixed64(): return pb_int64() +def pb_sfixed32(): return pb_int32() +def pb_bool(): return nullable(st.booleans()) +def pb_string(): return nullable(st.text(max_size=20)) +def pb_bytes(): return nullable(st.binary(max_size=20)) +def draw_pb_enum(draw, enum): + # Sample int val of enum, will be converted to member in encode_recurse + # Sample from pb2 values as it is the source of truth + return draw(nullable(st.sampled_from([member for member in enum.values()]))) +def pb_repeated(type): return nullable(st.lists(type, max_size=3)) # limit the size of the repeated field to speed up testing +def pb_span_id(): return nullable(st.binary(min_size=8, max_size=8)) +def pb_trace_id(): return nullable(st.binary(min_size=16, max_size=16)) +# For drawing oneof fields +# call with pb_oneof(draw, field1=pb_type1_callable, field2=pb_type2_callable, ...) +def pb_oneof(draw, **kwargs): + n = len(kwargs) + r = draw(st.integers(min_value=0, max_value=n-1)) + k, v = list(kwargs.items())[r] + return {k: draw(v())} +def pb_message(type): + return nullable(type) + +SF = "_sf" +PB = "_pb2" + +# Strategies for generating opentelemetry-proto types +@st.composite +def instrumentation_scope(draw): + return { + SF: common_sf.InstrumentationScope, + PB: common_pb2.InstrumentationScope, + "name": draw(pb_string()), + "version": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + +@st.composite +def resource(draw): + return { + SF: resource_sf.Resource, + PB: resource_pb2.Resource, + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + +@st.composite +def any_value(draw): + return { + SF: common_sf.AnyValue, + PB: common_pb2.AnyValue, + **pb_oneof( + draw, + string_value=pb_string, + bool_value=pb_bool, + int_value=pb_int64, + double_value=pb_double, + array_value=array_value, + kvlist_value=key_value_list, + bytes_value=pb_bytes, + ), + } + +@st.composite +def array_value(draw): + return { + SF: common_sf.ArrayValue, + PB: common_pb2.ArrayValue, + "values": draw(pb_repeated(any_value())), + } + +@st.composite +def key_value(draw): + return { + SF: common_sf.KeyValue, + PB: common_pb2.KeyValue, + "key": draw(pb_string()), + "value": draw(any_value()), + } + +@st.composite +def key_value_list(draw): + return { + SF: common_sf.KeyValueList, + PB: common_pb2.KeyValueList, + "values": draw(pb_repeated(key_value())), + } + +@st.composite +def logs_data(draw): + @st.composite + def log_record(draw): + return { + SF: logs_sf.LogRecord, + PB: logs_pb2.LogRecord, + "time_unix_nano": draw(pb_fixed64()), + "observed_time_unix_nano": draw(pb_fixed64()), + "severity_number": draw_pb_enum(draw, logs_pb2.SeverityNumber), + "severity_text": draw(pb_string()), + "body": draw(pb_message(any_value())), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + "span_id": draw(pb_span_id()), + "trace_id": draw(pb_trace_id()), + } + + @st.composite + def scope_logs(draw): + return { + SF: logs_sf.ScopeLogs, + PB: logs_pb2.ScopeLogs, + "scope": draw(pb_message(instrumentation_scope())), + "log_records": draw(pb_repeated(log_record())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_logs(draw): + return { + SF: logs_sf.ResourceLogs, + PB: logs_pb2.ResourceLogs, + "resource": draw(pb_message(resource())), + "scope_logs": draw(pb_repeated(scope_logs())), + "schema_url": draw(pb_string()), + } + + return { + SF: logs_sf.LogsData, + PB: logs_pb2.LogsData, + "resource_logs": draw(pb_repeated(resource_logs())), + } + +@st.composite +def traces_data(draw): + @st.composite + def event(draw): + return { + SF: trace_sf.Span.Event, + PB: trace_pb2.Span.Event, + "time_unix_nano": draw(pb_fixed64()), + "name": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + } + + @st.composite + def link(draw): + return { + SF: trace_sf.Span.Link, + PB: trace_pb2.Span.Link, + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "attributes": draw(pb_repeated(key_value())), + "dropped_attributes_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + } + + @st.composite + def status(draw): + return { + SF: trace_sf.Status, + PB: trace_pb2.Status, + "code": draw_pb_enum(draw, trace_pb2.Status.StatusCode), + "message": draw(pb_string()), + } + + @st.composite + def span(draw): + return { + SF: trace_sf.Span, + PB: trace_pb2.Span, + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "trace_state": draw(pb_string()), + "parent_span_id": draw(pb_span_id()), + "name": draw(pb_string()), + "kind": draw_pb_enum(draw, trace_pb2.Span.SpanKind), + "start_time_unix_nano": draw(pb_fixed64()), + "end_time_unix_nano": draw(pb_fixed64()), + "attributes": draw(pb_repeated(key_value())), + "events": draw(pb_repeated(event())), + "links": draw(pb_repeated(link())), + "status": draw(pb_message(status())), + "dropped_attributes_count": draw(pb_uint32()), + "dropped_events_count": draw(pb_uint32()), + "dropped_links_count": draw(pb_uint32()), + "flags": draw(pb_fixed32()), + } + + @st.composite + def scope_spans(draw): + return { + SF: trace_sf.ScopeSpans, + PB: trace_pb2.ScopeSpans, + "scope": draw(pb_message(instrumentation_scope())), + "spans": draw(pb_repeated(span())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_spans(draw): + return { + SF: trace_sf.ResourceSpans, + PB: trace_pb2.ResourceSpans, + "resource": draw(pb_message(resource())), + "scope_spans": draw(pb_repeated(scope_spans())), + "schema_url": draw(pb_string()), + } + + return { + SF: trace_sf.TracesData, + PB: trace_pb2.TracesData, + "resource_spans": draw(pb_repeated(resource_spans())), + } + +@st.composite +def metrics_data(draw): + @st.composite + def exemplar(draw): + return { + SF: metrics_sf.Exemplar, + PB: metrics_pb2.Exemplar, + **pb_oneof( + draw, + as_double=pb_double, + as_int=pb_sfixed64, + ), + "time_unix_nano": draw(pb_fixed64()), + "trace_id": draw(pb_trace_id()), + "span_id": draw(pb_span_id()), + "filtered_attributes": draw(pb_repeated(key_value())), + } + + @st.composite + def value_at_quantile(draw): + return { + SF: metrics_sf.SummaryDataPoint.ValueAtQuantile, + PB: metrics_pb2.SummaryDataPoint.ValueAtQuantile, + "quantile": draw_pb_double(draw), + "value": draw_pb_double(draw), + } + + @st.composite + def summary_data_point(draw): + return { + SF: metrics_sf.SummaryDataPoint, + PB: metrics_pb2.SummaryDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "quantile_values": draw(pb_repeated(value_at_quantile())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + } + + @st.composite + def buckets(draw): + return { + SF: metrics_sf.ExponentialHistogramDataPoint.Buckets, + PB: metrics_pb2.ExponentialHistogramDataPoint.Buckets, + "offset": draw(pb_sint32()), + "bucket_counts": draw(pb_repeated(pb_uint64())), + } + + @st.composite + def exponential_histogram_data_point(draw): + return { + SF: metrics_sf.ExponentialHistogramDataPoint, + PB: metrics_pb2.ExponentialHistogramDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + **pb_oneof( + draw, + positive=buckets, + negative=buckets, + ), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "max": draw_pb_double(draw), + "zero_threshold": draw_pb_double(draw), + } + + @st.composite + def histogram_data_point(draw): + return { + SF: metrics_sf.HistogramDataPoint, + PB: metrics_pb2.HistogramDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + "count": draw(pb_fixed64()), + "sum": draw_pb_double(draw), + "bucket_counts": draw(pb_repeated(pb_uint64())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + "exemplars": draw(pb_repeated(exemplar())), + "explicit_bounds": draw(pb_repeated(pb_double())), + **pb_oneof( + draw, + max=pb_double, + min=pb_double, + ), + } + + @st.composite + def number_data_point(draw): + return { + SF: metrics_sf.NumberDataPoint, + PB: metrics_pb2.NumberDataPoint, + "start_time_unix_nano": draw(pb_fixed64()), + "time_unix_nano": draw(pb_fixed64()), + **pb_oneof( + draw, + as_int=pb_sfixed64, + as_double=pb_double, + ), + "exemplars": draw(pb_repeated(exemplar())), + "attributes": draw(pb_repeated(key_value())), + "flags": draw(pb_uint32()), + } + + @st.composite + def summary(draw): + return { + SF: metrics_sf.Summary, + PB: metrics_pb2.Summary, + "data_points": draw(pb_repeated(summary_data_point())), + } + + @st.composite + def exponential_histogram(draw): + return { + SF: metrics_sf.ExponentialHistogram, + PB: metrics_pb2.ExponentialHistogram, + "data_points": draw(pb_repeated(exponential_histogram_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + } + + @st.composite + def histogram(draw): + return { + SF: metrics_sf.Histogram, + PB: metrics_pb2.Histogram, + "data_points": draw(pb_repeated(histogram_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + } + + @st.composite + def sum(draw): + return { + SF: metrics_sf.Sum, + PB: metrics_pb2.Sum, + "data_points": draw(pb_repeated(number_data_point())), + "aggregation_temporality": draw_pb_enum(draw, metrics_pb2.AggregationTemporality), + "is_monotonic": draw(pb_bool()), + } + + @st.composite + def gauge(draw): + return { + SF: metrics_sf.Gauge, + PB: metrics_pb2.Gauge, + "data_points": draw(pb_repeated(number_data_point())), + } + + @st.composite + def metric(draw): + return { + SF: metrics_sf.Metric, + PB: metrics_pb2.Metric, + "name": draw(pb_string()), + "description": draw(pb_string()), + "unit": draw(pb_string()), + **pb_oneof( + draw, + gauge=gauge, + sum=sum, + summary=summary, + histogram=histogram, + exponential_histogram=exponential_histogram, + ), + "metadata": draw(pb_repeated(key_value())), + } + + @st.composite + def scope_metrics(draw): + return { + SF: metrics_sf.ScopeMetrics, + PB: metrics_pb2.ScopeMetrics, + "scope": draw(pb_message(instrumentation_scope())), + "metrics": draw(pb_repeated(metric())), + "schema_url": draw(pb_string()), + } + + @st.composite + def resource_metrics(draw): + return { + SF: metrics_sf.ResourceMetrics, + PB: metrics_pb2.ResourceMetrics, + "resource": draw(pb_message(resource())), + "scope_metrics": draw(pb_repeated(scope_metrics())), + "schema_url": draw(pb_string()), + } + + return { + SF: metrics_sf.MetricsData, + PB: metrics_pb2.MetricsData, + "resource_metrics": draw(pb_repeated(resource_metrics())), + } + + +# Helper functions to recursively encode protobuf types using the generated args +# and the given serialization strategy +def encode_recurse(obj: Dict[str, Any], strategy: str) -> Any: + kwargs = {} + for key, value in obj.items(): + if key in [SF, PB]: + continue + elif value is None: + continue + elif isinstance(value, Mapping): + kwargs[key] = encode_recurse(value, strategy) + elif isinstance(value, List) and value and isinstance(value[0], Mapping): + kwargs[key] = [encode_recurse(v, strategy) for v in value if v is not None] + elif isinstance(value, List): + kwargs[key] = [v for v in value if v is not None] + else: + kwargs[key] = value + return obj[strategy](**kwargs) + +class TestProtoSerialization(unittest.TestCase): + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(logs_data()) + def test_log_data(self, logs_data): + self.assertEqual( + encode_recurse(logs_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(logs_data, SF)) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(traces_data()) + def test_trace_data(self, traces_data): + self.assertEqual( + encode_recurse(traces_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(traces_data, SF)) + ) + + @hypothesis.settings(suppress_health_check=[hypothesis.HealthCheck.too_slow]) + @hypothesis.given(metrics_data()) + def test_metrics_data(self, metrics_data): + self.assertEqual( + encode_recurse(metrics_data, PB).SerializeToString(deterministic=True), + bytes(encode_recurse(metrics_data, SF)) + ) diff --git a/tests/test_protoc_plugin.py b/tests/test_protoc_plugin.py new file mode 100644 index 0000000..c27d923 --- /dev/null +++ b/tests/test_protoc_plugin.py @@ -0,0 +1,93 @@ +""" +Test protoc code generator plugin for custom protoc message types +""" +import unittest +import tempfile +import subprocess +import os + +# Import into globals() so generated code string can be compiled +from snowflake.telemetry._internal.serialize import * + +class TestProtocPlugin(unittest.TestCase): + def namespace_serialize_message(self, message_type: str, local_namespace: dict, **kwargs) -> bytes: + assert message_type in local_namespace, f"Message type {message_type} not found in local namespace" + return local_namespace[message_type](**kwargs).SerializeToString() + + def test_protoc_plugin(self): + with tempfile.NamedTemporaryFile(suffix=".proto", mode="w", delete=False) as proto_file: + # Define a simple proto file + proto_file.write( + """syntax = "proto3"; +package opentelemetry.proto.common.v1; + +message AnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +message ArrayValue { + repeated AnyValue values = 1; +} + +message KeyValueList { + repeated KeyValue values = 1; +} + +message KeyValue { + string key = 1; + AnyValue value = 2; +} + +message InstrumentationScope { + string name = 1; + string version = 2; + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} +""" + ) + proto_file.flush() + proto_file.close() + + proto_file_dir = os.path.dirname(proto_file.name) + proto_file_name = os.path.basename(proto_file.name) + + os.environ["OPENTELEMETRY_PROTO_DIR"] = proto_file_dir + + # Run protoc with custom plugin to generate serialization code for messages + result = subprocess.run([ + "python", + "-m", + "grpc_tools.protoc", + "-I", + proto_file_dir, + "--plugin=protoc-gen-custom-plugin=scripts/plugin.py", + f"--custom-plugin_out={tempfile.gettempdir()}", + proto_file_name, + ], capture_output=True) + + # Ensure protoc ran successfully + self.assertEqual(result.returncode, 0) + + generated_code_file_dir = tempfile.gettempdir() + generated_code_file_name = proto_file_name.replace(".proto", "_marshaler.py") + generated_code_file = os.path.join(generated_code_file_dir, generated_code_file_name) + + # Ensure generated code file exists + self.assertTrue(os.path.exists(generated_code_file)) + + # Ensure code can be executed and serializes correctly + with open(generated_code_file, "r") as f: + generated_code = f.read() + local_namespace = {} + eval(compile(generated_code, generated_code_file, "exec"), globals(), local_namespace) + + self.assertEqual(b'\n\x04test', self.namespace_serialize_message("AnyValue", local_namespace, string_value="test")) diff --git a/tests/test_trace_encoder.py b/tests/test_trace_encoder.py index 0fc767f..d8b326e 100644 --- a/tests/test_trace_encoder.py +++ b/tests/test_trace_encoder.py @@ -25,7 +25,7 @@ _SPAN_KIND_MAP, _encode_status, ) -from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( ExportTraceServiceRequest as PB2ExportTraceServiceRequest, ) @@ -40,10 +40,11 @@ from opentelemetry.proto.trace.v1.trace_pb2 import ( ResourceSpans as PB2ResourceSpans, TracesData as PB2TracesData, + ScopeSpans as PB2ScopeSpans, + Span as PB2Span, + SpanFlags as PB2SpanFlags, + Status as PB2Status, ) -from opentelemetry.proto.trace.v1.trace_pb2 import ScopeSpans as PB2ScopeSpans -from opentelemetry.proto.trace.v1.trace_pb2 import Span as PB2SPan -from opentelemetry.proto.trace.v1.trace_pb2 import Status as PB2Status from opentelemetry.sdk.trace import Event as SDKEvent from opentelemetry.sdk.trace import Resource as SDKResource from opentelemetry.sdk.trace import SpanContext as SDKSpanContext @@ -66,7 +67,7 @@ class TestOTLPTraceEncoder(unittest.TestCase): def test_encode_spans(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() - self.assertEqual(encode_spans(otel_spans), expected_encoding) + self.assertEqual(encode_spans(otel_spans).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_span_exporter(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() @@ -97,7 +98,7 @@ def get_exhaustive_otel_span_list() -> List[SDKSpan]: ) parent_span_context = SDKSpanContext( - trace_id, 0x1111111111111111, is_remote=False + trace_id, 0x1111111111111111, is_remote=True ) other_context = SDKSpanContext( @@ -185,15 +186,15 @@ def get_exhaustive_test_spans( PB2ScopeSpans( scope=PB2InstrumentationScope(), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[0].context.span_id ), - trace_state=None, parent_span_id=_encode_span_id( otel_spans[0].parent.span_id ), + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK|PB2SpanFlags.SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK, name=otel_spans[0].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -221,7 +222,7 @@ def get_exhaustive_test_spans( ), ], events=[ - PB2SPan.Event( + PB2Span.Event( name="event0", time_unix_nano=otel_spans[0] .events[0] @@ -249,7 +250,7 @@ def get_exhaustive_test_spans( ) ], links=[ - PB2SPan.Link( + PB2Span.Link( trace_id=_encode_trace_id( otel_spans[0] .links[0] @@ -260,6 +261,7 @@ def get_exhaustive_test_spans( .links[0] .context.span_id ), + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, attributes=[ PB2KeyValue( key="key_bool", @@ -283,13 +285,12 @@ def get_exhaustive_test_spans( version="version", ), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[3].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[3].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -320,13 +321,12 @@ def get_exhaustive_test_spans( PB2ScopeSpans( scope=PB2InstrumentationScope(), spans=[ - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[1].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[1].name, kind=span_kind, start_time_unix_nano=otel_spans[ @@ -338,13 +338,12 @@ def get_exhaustive_test_spans( links=None, status={}, ), - PB2SPan( + PB2Span( trace_id=trace_id, span_id=_encode_span_id( otel_spans[2].context.span_id ), - trace_state=None, - parent_span_id=None, + flags=PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK, name=otel_spans[2].name, kind=span_kind, start_time_unix_nano=otel_spans[ diff --git a/tests/test_vendored_exporter_version.py b/tests/test_vendored_exporter_version.py new file mode 100644 index 0000000..d6509b1 --- /dev/null +++ b/tests/test_vendored_exporter_version.py @@ -0,0 +1,11 @@ +import unittest + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ as vendored_version + +from opentelemetry.sdk.version import __version__ as sdk_version +from opentelemetry.exporter.otlp.proto.common.version import __version__ as exporter_version + +class TestVendoredExporterVersion(unittest.TestCase): + def test_version(self): + self.assertEqual(sdk_version, vendored_version, "SDK version should match vendored version") + self.assertEqual(exporter_version, vendored_version, "Exporter version should match vendored version")