From 2a92cfd9b1f262d478fddeaf2eb1f06a6e6100e6 Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Wed, 13 Nov 2024 14:17:54 +1300 Subject: [PATCH 1/2] Add orjson dependency --- requirements/requirements.in | 1 + requirements/requirements.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index c53f4162..9d44f577 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -3,6 +3,7 @@ certifi click~=8.1.7 docutils<0.18 msgpack~=0.6.1 +orjson Pygments pymysql rst2txt diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f94696e8..e4024157 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -22,7 +22,7 @@ certifi==2022.12.7 # reflink click==8.1.7 # via -r requirements.in -#cryptography==42.0.4 +#cryptography==43.0.1 # via -r requirements/vendor-wheels.txt docutils==0.17.1 # via @@ -38,6 +38,8 @@ jsonschema==4.17.3 # via -r requirements.in msgpack==0.6.2 # via -r requirements.in +orjson==3.10.11 + # via -r requirements.in #psycopg2==2.9.9 # via -r requirements/vendor-wheels.txt pycparser==2.21 From 7c0e513021e1745b996465d6f6385eb2a7cef02e Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Wed, 13 Nov 2024 15:33:04 +1300 Subject: [PATCH 2/2] Use `orjson` for faster JSONL output refs #1018 When generating a large (2GB) diff as JSON-Lines this takes 20-30% less time than the stdlib. It may be possible to use this in other places, but note that orjson doesn't support streaming encoding (iterencode), which means it is of limited utility where we're trying to stream JSON diffs of huge datasets. This change uses it for individual features in JSONL diffs only where the lack of iterencode() isn't a concern. orjson is MIT licensed. --- CHANGELOG.md | 1 + kart/json_diff_writers.py | 16 +++++++++++---- kart/output_util.py | 25 ++++++++++++++++++++++- tests/test_diff.py | 42 +++++++++++++++++++-------------------- 4 files changed, 58 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c3ff6f8..b7987481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ _When adding new entries to the changelog, please include issue/PR numbers where ## Unreleased +- diff: Use [orjson](https://github.com/ijl/orjson?tab=readme-ov-file#orjson) for faster JSON-Lines output. [#1019](https://github.com/koordinates/kart/pull/1019) - Upgrade to PDAL 2.7 [#1005](https://github.com/koordinates/kart/pull/1005) - Adds a `--drop-empty-geometry-features` option to `kart export`. [#1007](https://github.com/koordinates/kart/pull/1007) - Adds diagnostic output to Kart when `KART_DIAGNOSTICS=1` environment variable is set. [#1013](https://github.com/koordinates/kart/pull/1013) diff --git a/kart/json_diff_writers.py b/kart/json_diff_writers.py index 48509fff..b492069d 100644 --- a/kart/json_diff_writers.py +++ b/kart/json_diff_writers.py @@ -1,4 +1,4 @@ -import json +import orjson import logging import threading from datetime import datetime, timedelta, timezone @@ -19,7 +19,11 @@ from kart.diff_structs import FILES_KEY, BINARY_FILE, DatasetDiff from kart.key_filters import DeltaFilter from kart.log import commit_obj_to_json -from kart.output_util import dump_json_output, resolve_output_path +from kart.output_util import ( + dump_json_output, + resolve_output_path, + orjson_encode_default, +) from kart.tabular.feature_output import feature_as_geojson, feature_as_json from kart.timestamps import datetime_to_iso8601_utc, timedelta_to_iso8601_tz @@ -241,9 +245,13 @@ def __init__(self, *args, diff_estimate_accuracy=None, delta_filter=None, **kwar self._output_lock = threading.RLock() def dump(self, obj): + output: bytes = orjson.dumps( + obj, + default=orjson_encode_default, + option=orjson.OPT_APPEND_NEWLINE | orjson.OPT_NON_STR_KEYS, + ) with self._output_lock: - json.dump(obj, self.fp, separators=self.separators) - self.fp.write("\n") + self.fp.buffer.write(output) def write_header(self): self.dump( diff --git a/kart/output_util.py b/kart/output_util.py index 78af8e7a..b46b2f22 100644 --- a/kart/output_util.py +++ b/kart/output_util.py @@ -8,6 +8,7 @@ import types from pathlib import Path +import orjson import pygments from pygments.lexers import JsonLexer @@ -15,10 +16,19 @@ _terminal_formatter = None +# note: `json` and `orjson` libraries aren't quite interchangeable. +# * orjson is much faster, so we use it where we can +# * orjson doesn't support custom separators +# * orjson doesn't support iterencode(), so can't stream unbounded iterators to stdout :( +ORJSON_OPTIONS = { + "compact": 0, # orjson doesn't support custom separators, so extracompact and compact look identical + "extracompact": 0, + "pretty": orjson.OPT_INDENT_2, +} JSON_PARAMS = { "compact": {}, - "pretty": {"indent": 2}, "extracompact": {"separators": (",", ":")}, + "pretty": {"indent": 2}, } @@ -37,6 +47,19 @@ def __iter__(self): return itertools.chain(self._head, *self[:1]) +def orjson_encode_default(obj): + """ + Hook to extend the default serialisation of `orjson.dumps()` + """ + if isinstance(obj, tuple): + return list(obj) + + if hasattr(obj, "__json__"): + return obj.__json__() + + raise TypeError + + class ExtendedJsonEncoder(json.JSONEncoder): """A JSONEncoder that tries calling __json__() if it can't serialise an object another way.""" diff --git a/tests/test_diff.py b/tests/test_diff.py index fe09a211..32ae8cef 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -268,13 +268,13 @@ def test_diff_points(output_format, data_working_copy, cli_runner): elif output_format == "json-lines": assert r.exit_code == 0, r assert r.stdout.splitlines() == [ - '{"type": "version", "version": "kart.diff/v2", "outputFormat": "JSONL+hexwkb"}', - '{"type": "datasetInfo", "path": "nz_pa_points_topo_150k", "value": {"type": "table", "version": 3}}', - '{"type": "metaInfo", "dataset": "nz_pa_points_topo_150k", "key": "schema.json", "value": [{"id": "e97b4015-2765-3a33-b174-2ece5c33343b", "name": "fid", "dataType": "integer", "primaryKeyIndex": 0, "size": 64}, {"id": "f488ae9b-6e15-1fe3-0bda-e0d5d38ea69e", "name": "geom", "dataType": "geometry", "geometryType": "POINT", "geometryCRS": "EPSG:4326"}, {"id": "4a1c7a86-c425-ea77-7f1a-d74321a10edc", "name": "t50_fid", "dataType": "integer", "size": 32}, {"id": "d2a62351-a66d-bde2-ce3e-356fec9641e9", "name": "name_ascii", "dataType": "text", "length": 75}, {"id": "c3389414-a511-5385-7dcd-891c4ead1663", "name": "macronated", "dataType": "text", "length": 1}, {"id": "45b00eaa-5700-662d-8a21-9614e40c437b", "name": "name", "dataType": "text", "length": 75}]}', - '{"type": "feature", "dataset": "nz_pa_points_topo_150k", "change": {"-": {"fid": 1, "geom": "010100000097F3EF201223664087D715268E0043C0", "t50_fid": 2426271, "name_ascii": null, "macronated": "N", "name": null}, "+": {"fid": 9998, "geom": "010100000097F3EF201223664087D715268E0043C0", "t50_fid": 2426271, "name_ascii": null, "macronated": "N", "name": null}}}', - '{"type": "feature", "dataset": "nz_pa_points_topo_150k", "change": {"-": {"fid": 2, "geom": "0101000000E702F16784226640ADE666D77CFE42C0", "t50_fid": 2426272, "name_ascii": null, "macronated": "N", "name": null}, "+": {"fid": 2, "geom": "0101000000E702F16784226640ADE666D77CFE42C0", "t50_fid": null, "name_ascii": null, "macronated": "N", "name": "test"}}}', - '{"type": "feature", "dataset": "nz_pa_points_topo_150k", "change": {"-": {"fid": 3, "geom": "0101000000459AAFB247226640C6DAE2735FFD42C0", "t50_fid": 2426273, "name_ascii": "Tauwhare Pa", "macronated": "N", "name": "Tauwhare Pa"}}}', - '{"type": "feature", "dataset": "nz_pa_points_topo_150k", "change": {"+": {"fid": 9999, "geom": "010100000000000000000000000000000000000000", "t50_fid": 9999999, "name_ascii": "Te Motu-a-kore", "macronated": "N", "name": "Te Motu-a-kore"}}}', + '{"type":"version","version":"kart.diff/v2","outputFormat":"JSONL+hexwkb"}', + '{"type":"datasetInfo","path":"nz_pa_points_topo_150k","value":{"type":"table","version":3}}', + '{"type":"metaInfo","dataset":"nz_pa_points_topo_150k","key":"schema.json","value":[{"id":"e97b4015-2765-3a33-b174-2ece5c33343b","name":"fid","dataType":"integer","primaryKeyIndex":0,"size":64},{"id":"f488ae9b-6e15-1fe3-0bda-e0d5d38ea69e","name":"geom","dataType":"geometry","geometryType":"POINT","geometryCRS":"EPSG:4326"},{"id":"4a1c7a86-c425-ea77-7f1a-d74321a10edc","name":"t50_fid","dataType":"integer","size":32},{"id":"d2a62351-a66d-bde2-ce3e-356fec9641e9","name":"name_ascii","dataType":"text","length":75},{"id":"c3389414-a511-5385-7dcd-891c4ead1663","name":"macronated","dataType":"text","length":1},{"id":"45b00eaa-5700-662d-8a21-9614e40c437b","name":"name","dataType":"text","length":75}]}', + '{"type":"feature","dataset":"nz_pa_points_topo_150k","change":{"-":{"fid":1,"geom":"010100000097F3EF201223664087D715268E0043C0","t50_fid":2426271,"name_ascii":null,"macronated":"N","name":null},"+":{"fid":9998,"geom":"010100000097F3EF201223664087D715268E0043C0","t50_fid":2426271,"name_ascii":null,"macronated":"N","name":null}}}', + '{"type":"feature","dataset":"nz_pa_points_topo_150k","change":{"-":{"fid":2,"geom":"0101000000E702F16784226640ADE666D77CFE42C0","t50_fid":2426272,"name_ascii":null,"macronated":"N","name":null},"+":{"fid":2,"geom":"0101000000E702F16784226640ADE666D77CFE42C0","t50_fid":null,"name_ascii":null,"macronated":"N","name":"test"}}}', + '{"type":"feature","dataset":"nz_pa_points_topo_150k","change":{"-":{"fid":3,"geom":"0101000000459AAFB247226640C6DAE2735FFD42C0","t50_fid":2426273,"name_ascii":"Tauwhare Pa","macronated":"N","name":"Tauwhare Pa"}}}', + '{"type":"feature","dataset":"nz_pa_points_topo_150k","change":{"+":{"fid":9999,"geom":"010100000000000000000000000000000000000000","t50_fid":9999999,"name_ascii":"Te Motu-a-kore","macronated":"N","name":"Te Motu-a-kore"}}}', ] elif output_format == "html": @@ -730,13 +730,13 @@ def test_diff_polygons(output_format, data_working_copy, cli_runner): elif output_format == "json-lines": assert r.exit_code == 0, r assert r.stdout.splitlines() == [ - '{"type": "version", "version": "kart.diff/v2", "outputFormat": "JSONL+hexwkb"}', - '{"type": "datasetInfo", "path": "nz_waca_adjustments", "value": {"type": "table", "version": 3}}', - '{"type": "metaInfo", "dataset": "nz_waca_adjustments", "key": "schema.json", "value": [{"id": "79d3c4ca-3abd-0a30-2045-45169357113c", "name": "id", "dataType": "integer", "primaryKeyIndex": 0, "size": 64}, {"id": "c1d4dea1-c0ad-0255-7857-b5695e3ba2e9", "name": "geom", "dataType": "geometry", "geometryType": "MULTIPOLYGON", "geometryCRS": "EPSG:4167"}, {"id": "d3d4b64b-d48e-4069-4bb5-dfa943d91e6b", "name": "date_adjusted", "dataType": "timestamp", "timezone": "UTC"}, {"id": "dff34196-229d-f0b5-7fd4-b14ecf835b2c", "name": "survey_reference", "dataType": "text", "length": 50}, {"id": "13dc4918-974e-978f-05ce-3b4321077c50", "name": "adjusted_nodes", "dataType": "integer", "size": 32}]}', - '{"type": "feature", "dataset": "nz_waca_adjustments", "change": {"-": {"id": 1424927, "geom": "01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0", "date_adjusted": "2011-03-25T07:30:45", "survey_reference": null, "adjusted_nodes": 1122}, "+": {"id": 9998, "geom": "01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0", "date_adjusted": "2011-03-25T07:30:45", "survey_reference": null, "adjusted_nodes": 1122}}}', - '{"type": "feature", "dataset": "nz_waca_adjustments", "change": {"-": {"id": 1443053, "geom": "0106000000010000000103000000010000000B000000DDEF0B89EEC665400CAB8C50D98E43C0AA7883AEBCC66540F6237BC40C8843C0D25EEE2300C7654002A1BF90B18543C0218DAFE279C76540391485E7938543C09EE81AACF7C76540E85798D99E8843C02E055F7296C765405BFD22B2598D43C0EA119EE595C765406BD26D895C8D43C087CDFB1423C76540723E2B1FB88E43C08DFCB0941BC7654054B82FB1C38E43C0A00948100AC76540FB04E1A5D38E43C0DDEF0B89EEC665400CAB8C50D98E43C0", "date_adjusted": "2011-05-10T12:09:10", "survey_reference": null, "adjusted_nodes": 1238}, "+": {"id": 1443053, "geom": "0106000000010000000103000000010000000B000000DDEF0B89EEC665400CAB8C50D98E43C0AA7883AEBCC66540F6237BC40C8843C0D25EEE2300C7654002A1BF90B18543C0218DAFE279C76540391485E7938543C09EE81AACF7C76540E85798D99E8843C02E055F7296C765405BFD22B2598D43C0EA119EE595C765406BD26D895C8D43C087CDFB1423C76540723E2B1FB88E43C08DFCB0941BC7654054B82FB1C38E43C0A00948100AC76540FB04E1A5D38E43C0DDEF0B89EEC665400CAB8C50D98E43C0", "date_adjusted": "2019-01-01T00:00:00", "survey_reference": "test", "adjusted_nodes": 1238}}}', - '{"type": "feature", "dataset": "nz_waca_adjustments", "change": {"-": {"id": 1452332, "geom": "01060000000100000001030000000100000018000000C43FCCA465D7654049FCE5EE4E6642C031DD1F0460D765406D606177F06542C064343C0760D765408E68DDEBED6542C0774AC25F66D7654003E4041CD46542C00442E6DF6AD765405B0AD914C76542C00F9E1F7B6BD76540B7354771C56542C099152AB96BD76540ED1D93E0C46542C03E5700F86CD76540F85610F9C16542C01E90DF366ED76540FDC68D11BF6542C056546E3273D765402D735F73B36542C056C5C5E175D76540EFB2BA30AD6542C06AC54D4277D76540182AC9FAA96542C09C400C8977D7654048F61C62A96542C03590D37C7AD76540168A743FA76542C0F38A07DA7CD7654069796568AA6542C0FF12A7497FD76540FD8AFFFBAF6542C0D5F5B5BE91D765406A7190D0F26542C049E06AF891D76540BCC23B6FF56542C08B3858D991D76540B6662B2FF96542C07E0C0C0F90D76540E2CF4B20006642C03FF664C98ED7654020CAD027046642C020E67C7C74D765406A7528F9476642C052A1D0E771D76540D9BFA1A64C6642C0C43FCCA465D7654049FCE5EE4E6642C0", "date_adjusted": "2011-06-07T15:22:58", "survey_reference": null, "adjusted_nodes": 558}}}', - '{"type": "feature", "dataset": "nz_waca_adjustments", "change": {"+": {"id": 9999999, "geom": "01060000000100000001030000000100000005000000000000000000000000000000000000000000000000000000FCA9F1D24D62503FFCA9F1D24D62503FFCA9F1D24D62503FFCA9F1D24D62503F000000000000000000000000000000000000000000000000", "date_adjusted": "2019-07-05T13:04:00", "survey_reference": "Null Island\\u2122 \\ud83d\\uddfa", "adjusted_nodes": 123}}}', + '{"type":"version","version":"kart.diff/v2","outputFormat":"JSONL+hexwkb"}', + '{"type":"datasetInfo","path":"nz_waca_adjustments","value":{"type":"table","version":3}}', + '{"type":"metaInfo","dataset":"nz_waca_adjustments","key":"schema.json","value":[{"id":"79d3c4ca-3abd-0a30-2045-45169357113c","name":"id","dataType":"integer","primaryKeyIndex":0,"size":64},{"id":"c1d4dea1-c0ad-0255-7857-b5695e3ba2e9","name":"geom","dataType":"geometry","geometryType":"MULTIPOLYGON","geometryCRS":"EPSG:4167"},{"id":"d3d4b64b-d48e-4069-4bb5-dfa943d91e6b","name":"date_adjusted","dataType":"timestamp","timezone":"UTC"},{"id":"dff34196-229d-f0b5-7fd4-b14ecf835b2c","name":"survey_reference","dataType":"text","length":50},{"id":"13dc4918-974e-978f-05ce-3b4321077c50","name":"adjusted_nodes","dataType":"integer","size":32}]}', + '{"type":"feature","dataset":"nz_waca_adjustments","change":{"-":{"id":1424927,"geom":"01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0","date_adjusted":"2011-03-25T07:30:45","survey_reference":null,"adjusted_nodes":1122},"+":{"id":9998,"geom":"01060000000100000001030000000100000012000000D2B47A3DAEEB65402E86A80212EF42C01D23796880EB6540D54A46E909EE42C03E7210197BEB6540B164332CEBED42C003ECE8DE70EB6540C99AB69AACED42C0916A8E626FEB654040F4DAAC9EED42C0615CA5D035EB6540F2B295FC50EB42C04AA3B89940EB6540D90F9D94DCEA42C00937B99972EB6540163FEB35F4E942C0B9103A5876EB65408D6D995DE5E942C008A85AD68FEB654069D2CB43DDE942C0D24A26924CEC6540C455AF6CB0EC42C0D21275304CEC6540E6CE3803B6EC42C018EA6B3714EC6540D17726991DEE42C00D91731C00EC65401BE20E8A9CEE42C0EBE45150F7EB6540D10F6A10D4EE42C01C6BD51EEDEB6540CD6886390AEF42C0FB975FA7EBEB6540DB85E63A0DEF42C0D2B47A3DAEEB65402E86A80212EF42C0","date_adjusted":"2011-03-25T07:30:45","survey_reference":null,"adjusted_nodes":1122}}}', + '{"type":"feature","dataset":"nz_waca_adjustments","change":{"-":{"id":1443053,"geom":"0106000000010000000103000000010000000B000000DDEF0B89EEC665400CAB8C50D98E43C0AA7883AEBCC66540F6237BC40C8843C0D25EEE2300C7654002A1BF90B18543C0218DAFE279C76540391485E7938543C09EE81AACF7C76540E85798D99E8843C02E055F7296C765405BFD22B2598D43C0EA119EE595C765406BD26D895C8D43C087CDFB1423C76540723E2B1FB88E43C08DFCB0941BC7654054B82FB1C38E43C0A00948100AC76540FB04E1A5D38E43C0DDEF0B89EEC665400CAB8C50D98E43C0","date_adjusted":"2011-05-10T12:09:10","survey_reference":null,"adjusted_nodes":1238},"+":{"id":1443053,"geom":"0106000000010000000103000000010000000B000000DDEF0B89EEC665400CAB8C50D98E43C0AA7883AEBCC66540F6237BC40C8843C0D25EEE2300C7654002A1BF90B18543C0218DAFE279C76540391485E7938543C09EE81AACF7C76540E85798D99E8843C02E055F7296C765405BFD22B2598D43C0EA119EE595C765406BD26D895C8D43C087CDFB1423C76540723E2B1FB88E43C08DFCB0941BC7654054B82FB1C38E43C0A00948100AC76540FB04E1A5D38E43C0DDEF0B89EEC665400CAB8C50D98E43C0","date_adjusted":"2019-01-01T00:00:00","survey_reference":"test","adjusted_nodes":1238}}}', + '{"type":"feature","dataset":"nz_waca_adjustments","change":{"-":{"id":1452332,"geom":"01060000000100000001030000000100000018000000C43FCCA465D7654049FCE5EE4E6642C031DD1F0460D765406D606177F06542C064343C0760D765408E68DDEBED6542C0774AC25F66D7654003E4041CD46542C00442E6DF6AD765405B0AD914C76542C00F9E1F7B6BD76540B7354771C56542C099152AB96BD76540ED1D93E0C46542C03E5700F86CD76540F85610F9C16542C01E90DF366ED76540FDC68D11BF6542C056546E3273D765402D735F73B36542C056C5C5E175D76540EFB2BA30AD6542C06AC54D4277D76540182AC9FAA96542C09C400C8977D7654048F61C62A96542C03590D37C7AD76540168A743FA76542C0F38A07DA7CD7654069796568AA6542C0FF12A7497FD76540FD8AFFFBAF6542C0D5F5B5BE91D765406A7190D0F26542C049E06AF891D76540BCC23B6FF56542C08B3858D991D76540B6662B2FF96542C07E0C0C0F90D76540E2CF4B20006642C03FF664C98ED7654020CAD027046642C020E67C7C74D765406A7528F9476642C052A1D0E771D76540D9BFA1A64C6642C0C43FCCA465D7654049FCE5EE4E6642C0","date_adjusted":"2011-06-07T15:22:58","survey_reference":null,"adjusted_nodes":558}}}', + '{"type":"feature","dataset":"nz_waca_adjustments","change":{"+":{"id":9999999,"geom":"01060000000100000001030000000100000005000000000000000000000000000000000000000000000000000000FCA9F1D24D62503FFCA9F1D24D62503FFCA9F1D24D62503FFCA9F1D24D62503F000000000000000000000000000000000000000000000000","date_adjusted":"2019-07-05T13:04:00","survey_reference":"Null Island™ 🗺","adjusted_nodes":123}}}', ] elif output_format == "html": @@ -1047,13 +1047,13 @@ def test_diff_table(output_format, data_working_copy, cli_runner): elif output_format == "json-lines": assert r.exit_code == 0, r assert r.stdout.splitlines() == [ - '{"type": "version", "version": "kart.diff/v2", "outputFormat": "JSONL+hexwkb"}', - '{"type": "datasetInfo", "path": "countiestbl", "value": {"type": "table", "version": 3}}', - '{"type": "metaInfo", "dataset": "countiestbl", "key": "schema.json", "value": [{"id": "1ec8704f-4d90-08ca-8a94-2cc59dcf63bd", "name": "OBJECTID", "dataType": "integer", "primaryKeyIndex": 0, "size": 64}, {"id": "e160628f-d812-2b8d-4507-585e9e4950e2", "name": "NAME", "dataType": "text", "length": 32}, {"id": "1b9bc047-3358-2b0a-6350-861d0f3cb91b", "name": "STATE_NAME", "dataType": "text", "length": 25}, {"id": "b8ee8bdc-eeb3-776d-556e-052e6138172b", "name": "STATE_FIPS", "dataType": "text", "length": 2}, {"id": "a3c0c4b5-b2c8-8b67-2832-e75612668d98", "name": "CNTY_FIPS", "dataType": "text", "length": 3}, {"id": "000d0d61-4e56-a711-5b95-68d744c96cef", "name": "FIPS", "dataType": "text", "length": 5}, {"id": "a75c1017-b76f-d2ea-d3e9-48845bec11ff", "name": "AREA", "dataType": "float", "size": 64}, {"id": "e28f5b52-e90c-88ca-928e-0e3b0046cb2a", "name": "POP1990", "dataType": "float", "size": 64}, {"id": "c9dd2446-3d80-e799-9a87-72088b4d79df", "name": "POP2000", "dataType": "float", "size": 64}, {"id": "bb6c3a7a-7ed5-d384-aed5-b328fb2adf0b", "name": "POP90_SQMI", "dataType": "integer", "size": 32}, {"id": "e5ab8cbf-f768-0ef4-e685-b6cacf5075bf", "name": "Shape_Leng", "dataType": "float", "size": 64}, {"id": "45e4a30c-8dc1-4210-8843-88d28eb5d3b2", "name": "Shape_Area", "dataType": "float", "size": 64}]}', - '{"type": "feature", "dataset": "countiestbl", "change": {"-": {"OBJECTID": 1, "NAME": "Lake of the Woods", "STATE_NAME": "Minnesota", "STATE_FIPS": "27", "CNTY_FIPS": "077", "FIPS": "27077", "AREA": 1784.0634, "POP1990": 4076.0, "POP2000": 4651.0, "POP90_SQMI": 2, "Shape_Leng": 4.055459982439919, "Shape_Area": 0.5654499337414509}, "+": {"OBJECTID": 9998, "NAME": "Lake of the Woods", "STATE_NAME": "Minnesota", "STATE_FIPS": "27", "CNTY_FIPS": "077", "FIPS": "27077", "AREA": 1784.0634, "POP1990": 4076.0, "POP2000": 4651.0, "POP90_SQMI": 2, "Shape_Leng": 4.055459982439919, "Shape_Area": 0.5654499337414509}}}', - '{"type": "feature", "dataset": "countiestbl", "change": {"-": {"OBJECTID": 2, "NAME": "Ferry", "STATE_NAME": "Washington", "STATE_FIPS": "53", "CNTY_FIPS": "019", "FIPS": "53019", "AREA": 2280.2319, "POP1990": 6295.0, "POP2000": 7199.0, "POP90_SQMI": 3, "Shape_Leng": 3.786160993863997, "Shape_Area": 0.7180593026451161}, "+": {"OBJECTID": 2, "NAME": "test", "STATE_NAME": "Washington", "STATE_FIPS": "53", "CNTY_FIPS": "019", "FIPS": "53019", "AREA": 2280.2319, "POP1990": 6295.0, "POP2000": 9867.0, "POP90_SQMI": 3, "Shape_Leng": 3.786160993863997, "Shape_Area": 0.7180593026451161}}}', - '{"type": "feature", "dataset": "countiestbl", "change": {"-": {"OBJECTID": 3, "NAME": "Stevens", "STATE_NAME": "Washington", "STATE_FIPS": "53", "CNTY_FIPS": "065", "FIPS": "53065", "AREA": 2529.9794, "POP1990": 30948.0, "POP2000": 40652.0, "POP90_SQMI": 12, "Shape_Leng": 4.876296245235406, "Shape_Area": 0.7954858988987561}}}', - '{"type": "feature", "dataset": "countiestbl", "change": {"+": {"OBJECTID": 9999, "NAME": "Lake of the Gruffalo", "STATE_NAME": "Minnesota", "STATE_FIPS": "27", "CNTY_FIPS": "077", "FIPS": "27077", "AREA": 1784.0634, "POP1990": 4076.0, "POP2000": 4651.0, "POP90_SQMI": 2, "Shape_Leng": 4.05545998243992, "Shape_Area": 0.565449933741451}}}', + '{"type":"version","version":"kart.diff/v2","outputFormat":"JSONL+hexwkb"}', + '{"type":"datasetInfo","path":"countiestbl","value":{"type":"table","version":3}}', + '{"type":"metaInfo","dataset":"countiestbl","key":"schema.json","value":[{"id":"1ec8704f-4d90-08ca-8a94-2cc59dcf63bd","name":"OBJECTID","dataType":"integer","primaryKeyIndex":0,"size":64},{"id":"e160628f-d812-2b8d-4507-585e9e4950e2","name":"NAME","dataType":"text","length":32},{"id":"1b9bc047-3358-2b0a-6350-861d0f3cb91b","name":"STATE_NAME","dataType":"text","length":25},{"id":"b8ee8bdc-eeb3-776d-556e-052e6138172b","name":"STATE_FIPS","dataType":"text","length":2},{"id":"a3c0c4b5-b2c8-8b67-2832-e75612668d98","name":"CNTY_FIPS","dataType":"text","length":3},{"id":"000d0d61-4e56-a711-5b95-68d744c96cef","name":"FIPS","dataType":"text","length":5},{"id":"a75c1017-b76f-d2ea-d3e9-48845bec11ff","name":"AREA","dataType":"float","size":64},{"id":"e28f5b52-e90c-88ca-928e-0e3b0046cb2a","name":"POP1990","dataType":"float","size":64},{"id":"c9dd2446-3d80-e799-9a87-72088b4d79df","name":"POP2000","dataType":"float","size":64},{"id":"bb6c3a7a-7ed5-d384-aed5-b328fb2adf0b","name":"POP90_SQMI","dataType":"integer","size":32},{"id":"e5ab8cbf-f768-0ef4-e685-b6cacf5075bf","name":"Shape_Leng","dataType":"float","size":64},{"id":"45e4a30c-8dc1-4210-8843-88d28eb5d3b2","name":"Shape_Area","dataType":"float","size":64}]}', + '{"type":"feature","dataset":"countiestbl","change":{"-":{"OBJECTID":1,"NAME":"Lake of the Woods","STATE_NAME":"Minnesota","STATE_FIPS":"27","CNTY_FIPS":"077","FIPS":"27077","AREA":1784.0634,"POP1990":4076.0,"POP2000":4651.0,"POP90_SQMI":2,"Shape_Leng":4.055459982439919,"Shape_Area":0.5654499337414509},"+":{"OBJECTID":9998,"NAME":"Lake of the Woods","STATE_NAME":"Minnesota","STATE_FIPS":"27","CNTY_FIPS":"077","FIPS":"27077","AREA":1784.0634,"POP1990":4076.0,"POP2000":4651.0,"POP90_SQMI":2,"Shape_Leng":4.055459982439919,"Shape_Area":0.5654499337414509}}}', + '{"type":"feature","dataset":"countiestbl","change":{"-":{"OBJECTID":2,"NAME":"Ferry","STATE_NAME":"Washington","STATE_FIPS":"53","CNTY_FIPS":"019","FIPS":"53019","AREA":2280.2319,"POP1990":6295.0,"POP2000":7199.0,"POP90_SQMI":3,"Shape_Leng":3.786160993863997,"Shape_Area":0.7180593026451161},"+":{"OBJECTID":2,"NAME":"test","STATE_NAME":"Washington","STATE_FIPS":"53","CNTY_FIPS":"019","FIPS":"53019","AREA":2280.2319,"POP1990":6295.0,"POP2000":9867.0,"POP90_SQMI":3,"Shape_Leng":3.786160993863997,"Shape_Area":0.7180593026451161}}}', + '{"type":"feature","dataset":"countiestbl","change":{"-":{"OBJECTID":3,"NAME":"Stevens","STATE_NAME":"Washington","STATE_FIPS":"53","CNTY_FIPS":"065","FIPS":"53065","AREA":2529.9794,"POP1990":30948.0,"POP2000":40652.0,"POP90_SQMI":12,"Shape_Leng":4.876296245235406,"Shape_Area":0.7954858988987561}}}', + '{"type":"feature","dataset":"countiestbl","change":{"+":{"OBJECTID":9999,"NAME":"Lake of the Gruffalo","STATE_NAME":"Minnesota","STATE_FIPS":"27","CNTY_FIPS":"077","FIPS":"27077","AREA":1784.0634,"POP1990":4076.0,"POP2000":4651.0,"POP90_SQMI":2,"Shape_Leng":4.05545998243992,"Shape_Area":0.565449933741451}}}', ] elif output_format == "html":