Skip to content

Commit

Permalink
Count the number of bytes we've had to copy in concat
Browse files Browse the repository at this point in the history
If this is more than a small faction of bytes, it's worth tuning
consolidation to return larger chunks of output.
  • Loading branch information
bkirwi committed Sep 19, 2024
1 parent 95e3fe6 commit 6aa51b5
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 1 deletion.
8 changes: 7 additions & 1 deletion src/persist/src/indexed/encoding.rs
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,13 @@ impl BlobTraceUpdates {
key: ::arrow::compute::concat(&keys)?,
val: ::arrow::compute::concat(&vals)?,
};
Ok(Self::Both(records, ext))

let out = Self::Both(records, ext);
metrics
.arrow
.concat_bytes
.inc_by(u64::cast_from(out.goodbytes()));
Ok(out)
}
}

Expand Down
6 changes: 6 additions & 0 deletions src/persist/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ pub struct ArrowMetrics {
pub(crate) val: ArrowColumnMetrics,
pub(crate) part_build_seconds: Counter,
pub(crate) part_build_count: IntCounter,
pub(crate) concat_bytes: IntCounter,
}

impl ArrowMetrics {
Expand All @@ -109,12 +110,17 @@ impl ArrowMetrics {
name: "mz_persist_columnar_part_build_count",
help: "number of times we've encoded our structured columnar format",
));
let concat_bytes: IntCounter = registry.register(metric!(
name: "mz_persist_columnar_part_concat_bytes",
help: "number of bytes we've copied when concatenating updates",
));

ArrowMetrics {
key: ArrowColumnMetrics::new(&op_count, &op_seconds, "key"),
val: ArrowColumnMetrics::new(&op_count, &op_seconds, "val"),
part_build_seconds,
part_build_count,
concat_bytes,
}
}

Expand Down

0 comments on commit 6aa51b5

Please sign in to comment.