From 2c3bfbe4d939933f7851123763569f10a782161b Mon Sep 17 00:00:00 2001 From: Andrew Brudnak Date: Wed, 26 Feb 2025 11:18:18 -0700 Subject: [PATCH] vai node metrics tables --- validation/steve/vai/README.md | 2 +- validation/steve/vai/scripts/script.sh | 338 ++++++++++++++--- .../steve/vai/secret_sort_test_cases.go | 5 +- validation/steve/vai/vai.go | 3 +- validation/steve/vai/vai_test.go | 342 ++++++++++++------ 5 files changed, 533 insertions(+), 157 deletions(-) diff --git a/validation/steve/vai/README.md b/validation/steve/vai/README.md index fdcccdb23d..af85b9c2c9 100644 --- a/validation/steve/vai/README.md +++ b/validation/steve/vai/README.md @@ -9,7 +9,7 @@ Your GO suite should be set to `-run ^TestVaiTestSuite$`. In your config file, set the following: ```yaml -rancher: +rancher: host: "rancher_server_address" adminToken: "rancher_admin_token" insecure: True # optional diff --git a/validation/steve/vai/scripts/script.sh b/validation/steve/vai/scripts/script.sh index 5e08871ea5..3845a35f58 100644 --- a/validation/steve/vai/scripts/script.sh +++ b/validation/steve/vai/scripts/script.sh @@ -1,7 +1,39 @@ #!/bin/sh set -e -echo "Starting script execution..." +MAX_RETRIES=3 +BUILD_TIMEOUT=300 # 5 minutes timeout +CACHE_DIR="/var/cache/vai-query" +GO_VERSION="1.23.6" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $1" +} + +error() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $1" >&2 +} + +# Execute command with timeout +run_with_timeout() { + command="$1" + timeout="$2" + + # Create a timeout process + ( + eval "$command" & + cmd_pid=$! + + ( + sleep $timeout + kill $cmd_pid 2>/dev/null + ) & + timeout_pid=$! + + wait $cmd_pid 2>/dev/null + kill $timeout_pid 2>/dev/null + ) +} # Function to check if Go is installed and working check_go() { @@ -11,26 +43,36 @@ check_go() { return 1 } -# Install Go if not already installed -if ! check_go; then - echo "Go not found. Installing Go..." - curl -L -o go1.23.5.linux-amd64.tar.gz https://go.dev/dl/go1.23.5.linux-amd64.tar.gz --insecure - tar -C /usr/local -xzf go1.23.5.linux-amd64.tar.gz - rm go1.23.5.linux-amd64.tar.gz - echo "Go installed successfully." -else - echo "Go is already installed." -fi +# Install Go with retries +install_go() { + mkdir -p $CACHE_DIR + GO_ARCHIVE="$CACHE_DIR/go${GO_VERSION}.linux-amd64.tar.gz" -# Always set the PATH to include Go -export PATH=$PATH:/usr/local/go/bin + for i in $(seq 1 $MAX_RETRIES); do + log "Attempting to install Go (attempt $i of $MAX_RETRIES)..." -echo "Checking Go version:" -go version + if [ ! -f "$GO_ARCHIVE" ]; then + if ! curl -L -o "$GO_ARCHIVE" "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" --insecure; then + error "Failed to download Go (attempt $i)" + continue + fi + fi -# Check if vai-query already exists -if [ ! -f /usr/local/bin/vai-query ]; then - echo "vai-query not found. Building vai-query program..." + if tar -C /usr/local -xzf "$GO_ARCHIVE"; then + log "Go installed successfully" + return 0 + else + error "Failed to extract Go (attempt $i)" + rm -f "$GO_ARCHIVE" + fi + done + + error "Failed to install Go after $MAX_RETRIES attempts" + return 1 +} + +# Build vai-query with retries +build_vai_query() { mkdir -p /root/vai-query cd /root/vai-query @@ -40,69 +82,271 @@ if [ ! -f /usr/local/bin/vai-query ]; then fi # Create or update main.go - cat << EOF > main.go + cat << 'EOF' > main.go package main import ( "database/sql" + "encoding/json" "fmt" "log" "os" "strings" + "time" + "context" "github.com/pkg/errors" _ "modernc.org/sqlite" ) func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + log.Println("Starting VAI database query...") + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // Clean up any existing snapshot + os.Remove("/tmp/snapshot.db") + + // First connection to create snapshot + log.Println("Opening connection to original database...") db, err := sql.Open("sqlite", "/var/lib/rancher/informer_object_cache.db") if err != nil { - log.Fatal(err) + log.Fatalf("Failed to open original database: %v", err) } - defer db.Close() - fmt.Println("Creating database snapshot...") - _, err = db.Exec("VACUUM INTO '/tmp/snapshot.db'") + log.Println("Creating database snapshot...") + _, err = db.ExecContext(ctx, "VACUUM INTO '/tmp/snapshot.db'") if err != nil { - log.Fatal(err) + log.Fatalf("Failed to create snapshot: %v", err) } + db.Close() - tableName := strings.ReplaceAll(os.Getenv("TABLE_NAME"), "\"", "") - resourceName := os.Getenv("RESOURCE_NAME") + // Wait a moment for filesystem to sync + time.Sleep(time.Second) - query := fmt.Sprintf("SELECT \"metadata.name\" FROM \"%s\" WHERE \"metadata.name\" = ?", tableName) - stmt, err := db.Prepare(query) + // Open the snapshot for querying + log.Println("Opening connection to snapshot database...") + db, err = sql.Open("sqlite", "/tmp/snapshot.db") if err != nil { - log.Fatal(err) + log.Fatalf("Failed to open snapshot: %v", err) } - defer stmt.Close() + defer db.Close() - var result string - err = stmt.QueryRow(resourceName).Scan(&result) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - fmt.Println("Resource not found") + // Get query parameters + sqlQuery := os.Getenv("SQL_QUERY") + tableName := strings.ReplaceAll(os.Getenv("TABLE_NAME"), "\"", "") + resourceName := os.Getenv("RESOURCE_NAME") + outputFormat := os.Getenv("OUTPUT_FORMAT") + + if sqlQuery != "" { + // Execute custom SQL query + log.Printf("Executing custom SQL query: %s", sqlQuery) + + rows, err := db.QueryContext(ctx, sqlQuery) + if err != nil { + log.Fatalf("Query error: %v", err) + } + defer rows.Close() + + // Get column names + columns, err := rows.Columns() + if err != nil { + log.Fatalf("Failed to get column names: %v", err) + } + + // Prepare result storage + values := make([]interface{}, len(columns)) + valuePtrs := make([]interface{}, len(columns)) + for i := range columns { + valuePtrs[i] = &values[i] + } + + results := []map[string]interface{}{} + + // Fetch all rows + for rows.Next() { + err := rows.Scan(valuePtrs...) + if err != nil { + log.Fatalf("Error scanning row: %v", err) + } + + // Convert to map for each row + row := make(map[string]interface{}) + for i, col := range columns { + val := values[i] + + // Handle nil values + if val == nil { + row[col] = nil + } else { + // Handle different value types + switch v := val.(type) { + case []byte: + row[col] = string(v) + default: + row[col] = v + } + } + } + results = append(results, row) + } + + if err := rows.Err(); err != nil { + log.Fatalf("Error iterating rows: %v", err) + } + + // Output results + if strings.ToLower(outputFormat) == "json" { + jsonOutput, err := json.MarshalIndent(results, "", " ") + if err != nil { + log.Fatalf("Error marshaling to JSON: %v", err) + } + fmt.Println(string(jsonOutput)) + } else { + // Default to plain text format + if len(results) == 0 { + fmt.Println("No results found") + } else { + // Print header + for i, col := range columns { + if i > 0 { + fmt.Print("\t") + } + fmt.Print(col) + } + fmt.Println() + + // Print separator + for i, col := range columns { + if i > 0 { + fmt.Print("\t") + } + fmt.Print(strings.Repeat("-", len(col))) + } + fmt.Println() + + // Print rows + for _, result := range results { + for i, col := range columns { + if i > 0 { + fmt.Print("\t") + } + value := result[col] + if value == nil { + fmt.Print("") + } else { + fmt.Print(value) + } + } + fmt.Println() + } + + fmt.Printf("\n%d row(s) returned\n", len(results)) + } + } + } else if tableName != "" && resourceName != "" { + // Legacy mode: search for a specific resource in a table + log.Printf("Querying table '%s' for resource '%s'", tableName, resourceName) + + query := fmt.Sprintf("SELECT \"metadata.name\" FROM \"%s\" WHERE \"metadata.name\" = ?", tableName) + stmt, err := db.PrepareContext(ctx, query) + if err != nil { + log.Fatalf("Failed to prepare query: %v", err) + } + defer stmt.Close() + + var result string + err = stmt.QueryRowContext(ctx, resourceName).Scan(&result) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + log.Printf("Resource '%s' not found in table '%s'", resourceName, tableName) + fmt.Printf("Resource not found\n") + } else { + log.Printf("Query error: %v", err) + fmt.Printf("Query error: %v\n", err) + } } else { - log.Fatal(err) + log.Printf("Found resource '%s' in table '%s'", result, tableName) + fmt.Printf("Found resource: %s\n", result) } } else { - fmt.Println("Found resource:", result) + log.Println("No valid query parameters provided") + fmt.Println("Usage options:") + fmt.Println("1. Set SQL_QUERY environment variable for custom SQL queries") + fmt.Println(" Optional: Set OUTPUT_FORMAT=json for JSON output") + fmt.Println("2. Set TABLE_NAME and RESOURCE_NAME environment variables for resource lookup") } + + // Clean up + log.Println("Cleaning up snapshot...") + os.Remove("/tmp/snapshot.db") + log.Println("Query operation completed") } EOF - # Get dependencies - go get github.com/pkg/errors - go get modernc.org/sqlite + for i in $(seq 1 $MAX_RETRIES); do + log "Building vai-query (attempt $i of $MAX_RETRIES)..." + + # Get dependencies with timeout + if ! run_with_timeout "go get github.com/pkg/errors" $BUILD_TIMEOUT; then + error "Failed to get pkg/errors dependency (attempt $i)" + continue + fi - # Build the program - go build -o /usr/local/bin/vai-query main.go - echo "Pure Go vai-query program built successfully." + if ! run_with_timeout "go get modernc.org/sqlite" $BUILD_TIMEOUT; then + error "Failed to get sqlite dependency (attempt $i)" + continue + fi + + # Build with timeout + if run_with_timeout "go build -o /usr/local/bin/vai-query main.go" $BUILD_TIMEOUT; then + log "vai-query built successfully" + return 0 + else + error "Build failed (attempt $i)" + fi + done + + error "Failed to build vai-query after $MAX_RETRIES attempts" + return 1 +} + +# Main execution starts here +log "Starting script execution..." + +# Ensure cache directory exists +mkdir -p $CACHE_DIR + +# Install Go if needed +if ! check_go; then + log "Go not found. Installing Go..." + if ! install_go; then + error "Failed to install Go. Exiting." + exit 1 + fi +fi + +# Always set the PATH to include Go +export PATH=$PATH:/usr/local/go/bin + +log "Checking Go version:" +go version + +# Build vai-query if needed +if [ ! -f /usr/local/bin/vai-query ]; then + log "vai-query not found. Building vai-query program..." + if ! build_vai_query; then + error "Failed to build vai-query. Exiting." + exit 1 + fi else - echo "vai-query program already exists. Using existing binary." + log "vai-query program already exists. Using existing binary." fi -echo "Executing the query program..." -TABLE_NAME="${TABLE_NAME}" RESOURCE_NAME="${RESOURCE_NAME}" /usr/local/bin/vai-query +log "Executing the query program..." +SQL_QUERY="${SQL_QUERY}" TABLE_NAME="${TABLE_NAME}" RESOURCE_NAME="${RESOURCE_NAME}" OUTPUT_FORMAT="${OUTPUT_FORMAT}" /usr/local/bin/vai-query -echo "Script execution completed." \ No newline at end of file +log "Script execution completed." \ No newline at end of file diff --git a/validation/steve/vai/secret_sort_test_cases.go b/validation/steve/vai/secret_sort_test_cases.go index 3134f33327..b08adeea5c 100644 --- a/validation/steve/vai/secret_sort_test_cases.go +++ b/validation/steve/vai/secret_sort_test_cases.go @@ -2,12 +2,13 @@ package vai import ( "fmt" - namegen "github.com/rancher/shepherd/pkg/namegenerator" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net/url" "sort" "strings" + namegen "github.com/rancher/shepherd/pkg/namegenerator" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/api/core/v1" ) diff --git a/validation/steve/vai/vai.go b/validation/steve/vai/vai.go index b83bd2c81e..c9cd411fc2 100644 --- a/validation/steve/vai/vai.go +++ b/validation/steve/vai/vai.go @@ -2,8 +2,9 @@ package vai import ( "fmt" - "github.com/rancher/shepherd/clients/rancher" "strings" + + "github.com/rancher/shepherd/clients/rancher" ) const ( diff --git a/validation/steve/vai/vai_test.go b/validation/steve/vai/vai_test.go index 8f4b9f4036..5710dc111c 100644 --- a/validation/steve/vai/vai_test.go +++ b/validation/steve/vai/vai_test.go @@ -1,4 +1,4 @@ -//go:build (validation || infra.any || cluster.any || extended) && !stress +//go:build (validation || infra.any || cluster.any || extended) && !stress && !2.8 package vai @@ -7,10 +7,8 @@ import ( "net/url" "strings" "testing" - "time" "github.com/rancher/shepherd/clients/rancher" - management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" steveV1 "github.com/rancher/shepherd/clients/rancher/v1" "github.com/rancher/shepherd/extensions/kubectl" "github.com/rancher/shepherd/extensions/vai" @@ -24,14 +22,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const scriptURL = "https://raw.githubusercontent.com/rancher/tests/main/validation/steve/vai/scripts/script.sh" +const ( + scriptURL = "https://raw.githubusercontent.com/rancher/tests/refs/heads/main/validation/steve/vai/scripts/script.sh" + logBufferSize = "256MB" +) type VaiTestSuite struct { suite.Suite client *rancher.Client steveClient *steveV1.Client session *session.Session - cluster management.Cluster vaiEnabled bool } @@ -83,92 +83,6 @@ func (v *VaiTestSuite) ensureVaiDisabled() { } } -// TestVaiEnabled runs all tests with VAI enabled -func (v *VaiTestSuite) TestVaiEnabled() { - v.ensureVaiEnabled() - - v.Run("SecretFilters", func() { - supportedWithVai := filterTestCases(secretFilterTestCases, true) - v.runSecretFilterTestCases(supportedWithVai) - }) - - v.Run("PodFilters", func() { - supportedWithVai := filterTestCases(podFilterTestCases, true) - v.runPodFilterTestCases(supportedWithVai) - }) - - v.Run("SecretSorting", func() { - supportedWithVai := filterTestCases(secretSortTestCases, true) - v.runSecretSortTestCases(supportedWithVai) - }) - - v.Run("SecretLimit", func() { - supportedWithVai := filterTestCases(secretLimitTestCases, true) - v.runSecretLimitTestCases(supportedWithVai) - }) - - v.Run("CheckDBFilesInPods", v.checkDBFilesInPods) - v.Run("CheckSecretInDB", v.checkSecretInVAIDatabase) - v.Run("CheckNamespaceInAllVAIDatabases", v.checkNamespaceInAllVAIDatabases) -} - -// TestVaiDisabled runs all tests with VAI disabled -func (v *VaiTestSuite) TestVaiDisabled() { - v.ensureVaiDisabled() - - v.Run("SecretFilters", func() { - unsupportedWithVai := filterTestCases(secretFilterTestCases, false) - v.runSecretFilterTestCases(unsupportedWithVai) - }) - - v.Run("PodFilters", func() { - unsupportedWithVai := filterTestCases(podFilterTestCases, false) - v.runPodFilterTestCases(unsupportedWithVai) - }) - - v.Run("SecretSorting", func() { - unsupportedWithVai := filterTestCases(secretSortTestCases, false) - v.runSecretSortTestCases(unsupportedWithVai) - }) - - v.Run("SecretLimit", func() { - unsupportedWithVai := filterTestCases(secretLimitTestCases, false) - v.runSecretLimitTestCases(unsupportedWithVai) - }) - - v.Run("NormalOperations", func() { - pods, err := v.client.Steve.SteveType("pod").List(nil) - require.NoError(v.T(), err) - require.NotEmpty(v.T(), pods.Data, "Should be able to list pods even with VAI disabled") - }) -} - -func TestVaiTestSuite(t *testing.T) { - suite.Run(t, new(VaiTestSuite)) -} - -func formatDuration(d time.Duration) string { - d = d.Round(time.Second) - h := d / time.Hour - d -= h * time.Hour - m := d / time.Minute - d -= m * time.Minute - s := d / time.Second - if h > 0 { - return fmt.Sprintf("%d hours %d minutes %d seconds", h, m, s) - } else if m > 0 { - return fmt.Sprintf("%d minutes %d seconds", m, s) - } else { - return fmt.Sprintf("%d seconds", s) - } -} - -func (v *VaiTestSuite) testNormalOperationsWithVaiDisabled() { - pods, err := v.client.Steve.SteveType("pod").List(nil) - require.NoError(v.T(), err) - assert.NotEmpty(v.T(), pods.Data, "Should be able to list pods even with VAI disabled") -} - func (v *VaiTestSuite) runSecretFilterTestCases(testCases []secretFilterTestCase) { secretClient := v.steveClient.SteveType("secret") namespaceClient := v.steveClient.SteveType("namespace") @@ -387,7 +301,7 @@ func (v *VaiTestSuite) checkDBFilesInPods() { for _, pod := range rancherPods { v.T().Run(fmt.Sprintf("Checking pod %s", pod), func(t *testing.T) { lsCmd := []string{"kubectl", "exec", pod, "-n", "cattle-system", "--", "ls"} - output, err := kubectl.Command(v.client, nil, "local", lsCmd, "") + output, err := kubectl.Command(v.client, nil, "local", lsCmd, logBufferSize) if err != nil { t.Errorf("Error executing command in pod %s: %v", pod, err) return @@ -463,7 +377,7 @@ func (v *VaiTestSuite) checkSecretInVAIDatabase() { } v.T().Logf("Executing command on pod %s", pod) - output, err := kubectl.Command(v.client, nil, "local", cmd, "") + output, err := kubectl.Command(v.client, nil, "local", cmd, logBufferSize) if err != nil { v.T().Logf("Error executing script on pod %s: %v", pod, err) continue @@ -491,9 +405,60 @@ func (v *VaiTestSuite) checkSecretInVAIDatabase() { v.T().Log("checkSecretInVAIDatabase test completed") } +func (v *VaiTestSuite) buildVAIQueryOnPods() error { + v.T().Log("Building VAI query program on all pods...") + + rancherPods, err := listRancherPods(v.client) + if err != nil { + return fmt.Errorf("failed to list Rancher pods: %v", err) + } + + for _, pod := range rancherPods { + v.T().Logf("Building VAI query on pod %s", pod) + + cmd := []string{ + "kubectl", "exec", pod, "-n", "cattle-system", "--", + "/bin/sh", "-c", + fmt.Sprintf("curl -k -sSL %s | sh", scriptURL), + } + + v.T().Logf("Executing command: %s", strings.Join(cmd, " ")) + + output, err := kubectl.Command(v.client, nil, "local", cmd, "") + if err != nil { + v.T().Logf("Error building VAI query on pod %s: %v", pod, err) + v.T().Logf("Command output: %s", output) + return fmt.Errorf("failed to build vai-query on pod %s: %v", pod, err) + } + + v.T().Logf("Successfully built VAI query on pod %s", pod) + } + + return nil +} + +// checkNamespaceInPod checks if a namespace exists in a specific pod's VAI database +func (v *VaiTestSuite) checkNamespaceInPod(pod, namespaceName string) (bool, string, error) { + v.T().Logf("Checking namespace %s in pod %s", namespaceName, pod) + cmd := []string{ + "kubectl", "exec", pod, "-n", "cattle-system", "--", + "sh", "-c", + fmt.Sprintf("TABLE_NAME='_v1_Namespace_fields' RESOURCE_NAME='%s' /usr/local/bin/vai-query", namespaceName), + } + + output, err := kubectl.Command(v.client, nil, "local", cmd, logBufferSize) + if err != nil { + return false, output, fmt.Errorf("failed to execute command on pod %s: %v", pod, err) + } + + return strings.Contains(output, namespaceName), output, nil +} + +// checkNamespaceInAllVAIDatabases checks if a namespace exists in all Rancher pods' VAI databases func (v *VaiTestSuite) checkNamespaceInAllVAIDatabases() { v.T().Log("Starting TestCheckNamespaceInAllVAIDatabases test") + // Create namespace to test namespaceName := fmt.Sprintf("db-namespace-%s", namegen.RandStringLower(randomStringLength)) v.T().Logf("Generated namespace name: %s", namespaceName) @@ -507,9 +472,62 @@ func (v *VaiTestSuite) checkNamespaceInAllVAIDatabases() { require.NoError(v.T(), err) v.T().Log("Namespace created successfully") + // Ensure namespace is in the list _, err = v.client.Steve.SteveType("namespace").List(nil) require.NoError(v.T(), err) + // Get list of pods + rancherPods, err := listRancherPods(v.client) + require.NoError(v.T(), err) + podCount := len(rancherPods) + v.T().Logf("Found %d Rancher pods", podCount) + require.NotEmpty(v.T(), rancherPods, "No Rancher pods found") + + // Check each pod + var outputs []string + namespaceFoundCount := 0 + + for i, pod := range rancherPods { + v.T().Logf("Processing pod %d: %s", i+1, pod) + + found, output, err := v.checkNamespaceInPod(pod, namespaceName) + require.NoError(v.T(), err) + + outputs = append(outputs, fmt.Sprintf("Output from pod %s:\n%s", pod, output)) + + if found { + v.T().Logf("Namespace found in pod %s", pod) + namespaceFoundCount++ + } else { + v.T().Logf("Namespace not found in pod %s", pod) + } + } + + // Log all outputs + v.T().Log("Logging all outputs:") + for i, output := range outputs { + v.T().Logf("Output %d:\n%s", i+1, output) + } + + v.T().Logf("Namespace found count: %d", namespaceFoundCount) + assert.Equal(v.T(), podCount, namespaceFoundCount, + fmt.Sprintf("Namespace %s not found in all Rancher pods' databases. Found in %d out of %d pods.", + namespaceName, namespaceFoundCount, podCount)) + v.T().Log("TestCheckNamespaceInAllVAIDatabases test completed") +} + +// checkMetricTablesInVAIDatabase checks if the required metric tables exist in the VAI database +func (v *VaiTestSuite) checkMetricTablesInVAIDatabase() { + v.T().Log("Starting checkMetricTablesInVAIDatabase test") + + // First, access the metrics endpoint to ensure tables are created + err := v.accessMetricsToCreateTables() + if err != nil { + v.T().Logf("Warning: Failed to access metrics endpoint: %v", err) + // Continue with the test even if this fails + } + + // List Rancher pods v.T().Log("Listing Rancher pods...") rancherPods, err := listRancherPods(v.client) require.NoError(v.T(), err) @@ -517,32 +535,50 @@ func (v *VaiTestSuite) checkNamespaceInAllVAIDatabases() { v.T().Logf("Using script URL: %s", scriptURL) + foundTables := make(map[string]bool) + expectedTables := []string{ + "metrics.k8s.io_v1beta1_NodeMetrics", + "metrics.k8s.io_v1beta1_NodeMetrics_fields", + "metrics.k8s.io_v1beta1_NodeMetrics_indices", + } var outputs []string - namespaceFoundCount := 0 for i, pod := range rancherPods { v.T().Logf("Processing pod %d: %s", i+1, pod) + cmd := []string{ "kubectl", "exec", pod, "-n", "cattle-system", "--", "sh", "-c", - fmt.Sprintf("curl -k -sSL %s | TABLE_NAME='_v1_Namespace_fields' RESOURCE_NAME='%s' sh", scriptURL, namespaceName), + fmt.Sprintf(`curl -k -sSL %s | SQL_QUERY='SELECT name FROM sqlite_master WHERE type="table" AND name IN ("metrics.k8s.io_v1beta1_NodeMetrics", "metrics.k8s.io_v1beta1_NodeMetrics_fields", "metrics.k8s.io_v1beta1_NodeMetrics_indices")' OUTPUT_FORMAT=text sh`, scriptURL), } v.T().Logf("Executing command on pod %s", pod) - output, err := kubectl.Command(v.client, nil, "local", cmd, "") + output, err := kubectl.Command(v.client, nil, "local", cmd, logBufferSize) if err != nil { v.T().Logf("Error executing script on pod %s: %v", pod, err) + v.T().Logf("Error output: %s", output) continue } v.T().Logf("Command executed successfully on pod %s", pod) outputs = append(outputs, fmt.Sprintf("Output from pod %s:\n%s", pod, output)) - if strings.Contains(output, namespaceName) { - v.T().Logf("Namespace found in pod %s", pod) - namespaceFoundCount++ - } else { - v.T().Logf("Namespace not found in pod %s", pod) + // Parse output to find tables + lines := strings.Split(output, "\n") + + // Check if output has at least header and separator line + if len(lines) >= 2 { + // Skip the header and separator rows (first two lines) + for i := 2; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if line != "" && !strings.Contains(line, "row(s) returned") { + // Check if this is a table name and not log output + if !strings.Contains(line, "[") && !strings.Contains(line, "INFO") && !strings.Contains(line, "ERROR") { + foundTables[line] = true + v.T().Logf("Found table: %s in pod %s", line, pod) + } + } + } } } @@ -551,9 +587,103 @@ func (v *VaiTestSuite) checkNamespaceInAllVAIDatabases() { v.T().Logf("Output %d:\n%s", i+1, output) } - v.T().Logf("Namespace found count: %d", namespaceFoundCount) - assert.Equal(v.T(), len(rancherPods), namespaceFoundCount, - fmt.Sprintf("Namespace %s not found in all Rancher pods' databases. Found in %d out of %d pods.", - namespaceName, namespaceFoundCount, len(rancherPods))) - v.T().Log("TestCheckNamespaceInAllVAIDatabases test completed") + // Check if all required tables were found + allTablesFound := true + for _, tableName := range expectedTables { + if !foundTables[tableName] { + allTablesFound = false + v.T().Logf("Required table not found: %s", tableName) + } + } + + assert.True(v.T(), allTablesFound, "Not all required metric tables were found in the VAI database") + v.T().Log("checkMetricTablesInVAIDatabase test completed") +} + +func (v *VaiTestSuite) accessMetricsToCreateTables() error { + v.T().Log("Accessing metrics endpoint to ensure tables are created") + + metricsClient := v.client.Steve.SteveType("metrics.k8s.io.nodemetrics") + + // List the metrics resources + v.T().Log("Listing metrics.k8s.io.nodemetrics resources") + _, err := metricsClient.List(nil) + if err != nil { + return fmt.Errorf("failed to list metrics.k8s.io.nodemetrics: %v", err) + } + + v.T().Log("Successfully accessed metrics endpoint") + return nil +} + +// TestVaiDisabled runs all tests with VAI disabled +func (v *VaiTestSuite) TestVaiDisabled() { + v.ensureVaiDisabled() + + v.Run("SecretFilters", func() { + unsupportedWithVai := filterTestCases(secretFilterTestCases, false) + v.runSecretFilterTestCases(unsupportedWithVai) + }) + + v.Run("PodFilters", func() { + unsupportedWithVai := filterTestCases(podFilterTestCases, false) + v.runPodFilterTestCases(unsupportedWithVai) + }) + + v.Run("SecretSorting", func() { + unsupportedWithVai := filterTestCases(secretSortTestCases, false) + v.runSecretSortTestCases(unsupportedWithVai) + }) + + v.Run("SecretLimit", func() { + unsupportedWithVai := filterTestCases(secretLimitTestCases, false) + v.runSecretLimitTestCases(unsupportedWithVai) + }) + + v.Run("NormalOperations", func() { + pods, err := v.client.Steve.SteveType("pod").List(nil) + require.NoError(v.T(), err) + require.NotEmpty(v.T(), pods.Data, "Should be able to list pods even with VAI disabled") + }) +} + +// TestVaiEnabled runs all VAI-enabled tests +func (v *VaiTestSuite) TestVaiEnabled() { + v.ensureVaiEnabled() + + // First ensure VAI query program is built on each pod + v.Run("SetupVAIQuery", func() { + err := v.buildVAIQueryOnPods() + require.NoError(v.T(), err, "Failed to build VAI query program on pods") + }) + + // Run all VAI-dependent tests + v.Run("SecretFilters", func() { + supportedWithVai := filterTestCases(secretFilterTestCases, true) + v.runSecretFilterTestCases(supportedWithVai) + }) + + v.Run("PodFilters", func() { + supportedWithVai := filterTestCases(podFilterTestCases, true) + v.runPodFilterTestCases(supportedWithVai) + }) + + v.Run("SecretSorting", func() { + supportedWithVai := filterTestCases(secretSortTestCases, true) + v.runSecretSortTestCases(supportedWithVai) + }) + + v.Run("SecretLimit", func() { + supportedWithVai := filterTestCases(secretLimitTestCases, true) + v.runSecretLimitTestCases(supportedWithVai) + }) + + v.Run("CheckDBFilesInPods", v.checkDBFilesInPods) + v.Run("CheckSecretInDB", v.checkSecretInVAIDatabase) + v.Run("CheckNamespaceInAllVAIDatabases", v.checkNamespaceInAllVAIDatabases) + v.Run("checkMetricTablesInVAIDatabase", v.checkMetricTablesInVAIDatabase) +} + +func TestVaiTestSuite(t *testing.T) { + suite.Run(t, new(VaiTestSuite)) }