Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[persist] Thread chunks of data from consolidation to the batch builder #29577

Merged
merged 9 commits into from
Sep 19, 2024
+270 −138
Merged

[persist] Thread chunks of data from consolidation to the batch builder #29577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
372cd05
Minor fixups
bkirwi Sep 17, 2024
5c082fb
Pass around BlobTraceUpdates in the batch builder
bkirwi Sep 16, 2024
b525f43
Remove the per-item schemas in favour of batch schemas
bkirwi Sep 16, 2024
b58eb98
Allow pushing entire sets of updates into the builder
bkirwi Sep 16, 2024
b345465
Estimate the output size via ArrayOrd
bkirwi Sep 16, 2024
095fdf5
BlobTraceUpdates::concat
bkirwi Sep 18, 2024
95e3fe6
Work harder to hit the target size
bkirwi Sep 18, 2024
6aa51b5
Count the number of bytes we've had to copy in concat
bkirwi Sep 18, 2024
463af6f
cargo test: Rework test_builtin_connection_alterations_are_preserved_…
def- Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 17 additions & 23 deletions src/environmentd/tests/server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4627,19 +4627,20 @@ async fn test_cert_reloading() {
check_pgwire(&conn_str, &ca.ca_cert_path(), next_x509.clone()).await;
}

#[mz_ore::test(tokio::test(flavor = "multi_thread", worker_threads = 1))]
#[mz_ore::test]
#[cfg_attr(miri, ignore)] // too slow
async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
fn test_builtin_connection_alterations_are_preserved_across_restarts() {
let data_dir = tempfile::tempdir().unwrap();
let harness = test_util::TestHarness::default().data_directory(data_dir.path());
let harness = test_util::TestHarness::default()
.data_directory(data_dir.path())
.unsafe_mode();

{
let server = harness.clone().start().await;
let client = server
.connect()
.internal()
let server = harness.clone().start_blocking();
let mut client = server
.pg_config_internal()
.user(&ANALYTICS_USER.name)
.await
.connect(postgres::NoTls)
.unwrap();

let row = client
Expand All @@ -4649,7 +4650,6 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
WHERE name = 'mz_analytics'",
&[],
)
.await
.expect("success");
let sql: String = row.get("create_sql");
assert_eq!(
Expand All @@ -4659,12 +4659,11 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
}

{
let server = harness.clone().start().await;
let client = server
.connect()
.internal()
let server = harness.clone().start_blocking();
let mut client = server
.pg_config_internal()
.user(&ANALYTICS_USER.name)
.await
.connect(postgres::NoTls)
.unwrap();

let row = client
Expand All @@ -4674,7 +4673,6 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
WHERE name = 'mz_analytics'",
&[],
)
.await
.expect("success");
let sql: String = row.get("create_sql");
assert_eq!(
Expand All @@ -4687,7 +4685,6 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
"ALTER CONNECTION mz_internal.mz_analytics SET (ASSUME ROLE ARN = 'foo')",
&[],
)
.await
.expect("success");

let row = client
Expand All @@ -4697,7 +4694,6 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
WHERE name = 'mz_analytics'",
&[],
)
.await
.expect("success");
let sql: String = row.get("create_sql");
assert_eq!(
Expand All @@ -4707,12 +4703,11 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
}

{
let server = harness.clone().start().await;
let client = server
.connect()
.internal()
let server = harness.clone().start_blocking();
let mut client = server
.pg_config_internal()
.user(&ANALYTICS_USER.name)
.await
.connect(postgres::NoTls)
.unwrap();

let row = client
Expand All @@ -4722,7 +4717,6 @@ async fn test_builtin_connection_alterations_are_preserved_across_restarts() {
WHERE name = 'mz_analytics'",
&[],
)
.await
.expect("success");
let sql: String = row.get("create_sql");
assert_eq!(
Expand Down
Loading
Loading