internal: pass non-zero chunk size to UploadMany

All calls to UploadMany were passing a zero chunk size, meaning that the request to BigQuery would immediately fail if there was too much data. Specify a positive chunk size, so UploadMany can retry the upload with less data. Change-Id: I9d545b6ecf4da2ca6dd90aeefb05d8691cd135ec Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/643435 LUCI-TryBot-Result: Go LUCI <[email protected]> Reviewed-by: Zvonimir Pavlinovic <[email protected]>
golang · Jan 21, 2025 · 76b4e21 · 76b4e21
1 parent 121801a
commit 76b4e21
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/internal/bigquery/bigquery.go b/internal/bigquery/bigquery.go
@@ -20,6 +20,7 @@ import (
 	bq "cloud.google.com/go/bigquery"
 	"cloud.google.com/go/civil"
 	"golang.org/x/pkgsite-metrics/internal/derrors"
+	"golang.org/x/pkgsite-metrics/internal/log"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
 )
@@ -215,6 +216,7 @@ func UploadMany[T Row](ctx context.Context, client *Client, tableID string, rows
 			end = len(rows)
 		}
 		for {
+			log.Infof(ctx, "bigquery.UploadMany: uploading rows[%d:%d]", start, end)
 			if err := ins.Put(ctx, rows[start:end]); err == nil {
 				break
 			} else if hasCode(err, http.StatusRequestEntityTooLarge) && end-start > 1 {

diff --git a/internal/vulndbreqs/bq.go b/internal/vulndbreqs/bq.go
@@ -61,13 +61,13 @@ func writeToBigQuery(ctx context.Context, client *bigquery.Client, rcs []*Reques
 	if _, err := client.CreateOrUpdateTable(ctx, RequestCountTableName); err != nil {
 		return err
 	}
-	if err := bigquery.UploadMany(ctx, client, RequestCountTableName, rcs, 0); err != nil {
+	if err := bigquery.UploadMany(ctx, client, RequestCountTableName, rcs, 100); err != nil {
 		return err
 	}
 	if _, err := client.CreateOrUpdateTable(ctx, IPRequestCountTableName); err != nil {
 		return err
 	}
-	return bigquery.UploadMany(ctx, client, IPRequestCountTableName, ircs, 0)
+	return bigquery.UploadMany(ctx, client, IPRequestCountTableName, ircs, 100)
 }
 
 // ReadRequestCountsFromBigQuery returns daily counts for requests to the vuln DB, most recent first.

diff --git a/internal/worker/scan.go b/internal/worker/scan.go
@@ -181,7 +181,7 @@ func writeResults(ctx context.Context, serve bool, w http.ResponseWriter, client
 		log.Infof(ctx, "bigquery disabled, not uploading")
 		return nil
 	}
-	return bigquery.UploadMany(ctx, client, table, rows, 0)
+	return bigquery.UploadMany(ctx, client, table, rows, 100)
 }
 
 func serveJSON(ctx context.Context, content interface{}, w http.ResponseWriter) error {