diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..293f3b7 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,147 @@ +name: Smoke Tests + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + +jobs: + smoke: + name: Run Smoke Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Download dependencies + run: go mod download + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: stackrox-mcp-smoke + + - name: Clone StackRox repository + run: git clone --depth 1 https://github.com/stackrox/stackrox.git stackrox-repo + + - name: Deploy StackRox Central + env: + MAIN_IMAGE_TAG: latest + SENSOR_HELM_DEPLOY: "true" + ROX_SCANNER_V4: "false" + run: | + cd stackrox-repo + ./deploy/k8s/deploy-local.sh + + - name: Deploy vulnerable workload + run: kubectl apply -f smoke/testdata/vulnerable-deployment.yaml + + - name: Wait for vulnerable deployment + run: kubectl wait --for=condition=available --timeout=120s deployment/vulnerable-app -n vulnerable-apps + + - name: Wait for Central pods ready + run: kubectl wait --for=condition=ready --timeout=180s pod -l app=central -n stackrox + + - name: Wait for Sensor pods ready + run: kubectl wait --for=condition=ready --timeout=300s pod -l app=sensor -n stackrox || echo "Sensor pods not ready yet, will check cluster health" + + - name: Extract Central password + id: extract-password + run: | + PASSWORD="$(cat stackrox-repo/deploy/k8s/central-deploy/password)" + echo "::add-mask::${PASSWORD}" + echo "password=${PASSWORD}" >> "$GITHUB_OUTPUT" + + - name: Wait for cluster to be healthy + run: | + echo "Waiting for cluster to register and become healthy..." + PASSWORD="$(cat stackrox-repo/deploy/k8s/central-deploy/password)" + + for i in {1..60}; do + CLUSTER_HEALTH=$(curl -k -s -u "admin:${PASSWORD}" \ + https://localhost:8000/v1/clusters 2>/dev/null | \ + jq -r '.clusters[0].healthStatus.overallHealthStatus // "NOT_FOUND"') + + echo "Attempt $i/60: Cluster health status: $CLUSTER_HEALTH" + + if [ "$CLUSTER_HEALTH" = "HEALTHY" ]; then + echo "Cluster is healthy and ready for testing" + break + fi + + if [ $i -eq 60 ]; then + echo "ERROR: Cluster did not become healthy after 60 attempts (2 minutes)" + echo "Current status: $CLUSTER_HEALTH" + exit 1 + fi + + sleep 2 + done + + - name: Install go-junit-report + run: go install github.com/jstemmer/go-junit-report/v2@v2.1.0 + + - name: Run smoke tests with JUnit output + env: + ROX_ENDPOINT: localhost:8000 + ROX_PASSWORD: ${{ steps.extract-password.outputs.password }} + run: | + go test -v -tags=smoke -cover -race -coverprofile=coverage-smoke.out -timeout=20m ./smoke 2>&1 | \ + tee /dev/stderr | \ + go-junit-report -set-exit-code -out junit-smoke.xml + + - name: Upload JUnit test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-smoke-results + path: junit-smoke.xml + if-no-files-found: error + + - name: Upload test results to Codecov + if: always() + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: junit-smoke.xml + + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: ./coverage-smoke.out + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + flags: smoke + name: smoke-tests + + - name: Collect logs + if: always() + run: | + mkdir -p logs + kubectl get pods -A > logs/pods.txt || true + kubectl get events -A --sort-by='.lastTimestamp' > logs/events.txt || true + kubectl logs -n vulnerable-apps deployment/vulnerable-app --all-containers=true > logs/vulnerable-app.log || true + kubectl logs -n stackrox deployment/central > logs/central.log || true + kubectl logs -n stackrox deployment/scanner > logs/scanner.log || true + kubectl describe pod -n vulnerable-apps > logs/vulnerable-app-describe.txt || true + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoke-test-logs + path: logs/ + if-no-files-found: ignore diff --git a/internal/testutil/integration_helpers.go b/internal/testutil/integration_helpers.go index 21b48e7..260f146 100644 --- a/internal/testutil/integration_helpers.go +++ b/internal/testutil/integration_helpers.go @@ -8,10 +8,8 @@ import ( "testing" "time" - "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stackrox/stackrox-mcp/internal/app" "github.com/stackrox/stackrox-mcp/internal/config" - "github.com/stretchr/testify/require" ) // CreateIntegrationTestConfig creates a test configuration for integration tests. @@ -55,37 +53,3 @@ func CreateIntegrationMCPClient(t *testing.T) (*MCPTestClient, error) { return NewMCPTestClient(t, runFunc) } - -// SetupInitializedClient creates an initialized MCP client for testing with automatic cleanup. -func SetupInitializedClient(t *testing.T, createClient func(*testing.T) (*MCPTestClient, error)) *MCPTestClient { - t.Helper() - - client, err := createClient(t) - require.NoError(t, err, "Failed to create MCP client") - t.Cleanup(func() { client.Close() }) - - return client -} - -// CallToolAndGetResult calls a tool and verifies it succeeds. -func CallToolAndGetResult(t *testing.T, client *MCPTestClient, toolName string, args map[string]any) *mcp.CallToolResult { - t.Helper() - - ctx := context.Background() - result, err := client.CallTool(ctx, toolName, args) - require.NoError(t, err) - RequireNoError(t, result) - - return result -} - -// GetTextContent extracts text from the first content item. -func GetTextContent(t *testing.T, result *mcp.CallToolResult) string { - t.Helper() - require.NotEmpty(t, result.Content, "should have content in response") - - textContent, ok := result.Content[0].(*mcp.TextContent) - require.True(t, ok, "expected TextContent, got %T", result.Content[0]) - - return textContent.Text -} diff --git a/internal/testutil/test_helpers.go b/internal/testutil/test_helpers.go new file mode 100644 index 0000000..5f995f5 --- /dev/null +++ b/internal/testutil/test_helpers.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "context" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/require" +) + +// SetupInitializedClient creates an initialized MCP client with automatic cleanup. +func SetupInitializedClient(t *testing.T, createClient func(*testing.T) (*MCPTestClient, error)) *MCPTestClient { + t.Helper() + + client, err := createClient(t) + require.NoError(t, err, "Failed to create MCP client") + t.Cleanup(func() { _ = client.Close() }) + + return client +} + +// CallToolAndGetResult calls a tool and verifies it succeeds. +func CallToolAndGetResult( + t *testing.T, + client *MCPTestClient, + toolName string, + args map[string]any, +) *mcp.CallToolResult { + t.Helper() + + ctx := context.Background() + result, err := client.CallTool(ctx, toolName, args) + require.NoError(t, err) + RequireNoError(t, result) + + return result +} + +// GetTextContent extracts text from the first content item. +func GetTextContent(t *testing.T, result *mcp.CallToolResult) string { + t.Helper() + require.NotEmpty(t, result.Content, "should have content in response") + + textContent, ok := result.Content[0].(*mcp.TextContent) + require.True(t, ok, "expected TextContent, got %T", result.Content[0]) + + return textContent.Text +} diff --git a/smoke/smoke_test.go b/smoke/smoke_test.go new file mode 100644 index 0000000..1781ce6 --- /dev/null +++ b/smoke/smoke_test.go @@ -0,0 +1,192 @@ +//go:build smoke + +package smoke + +import ( + "context" + "encoding/json" + "io" + "os" + "testing" + "time" + + "github.com/stackrox/stackrox-mcp/internal/app" + "github.com/stackrox/stackrox-mcp/internal/config" + "github.com/stackrox/stackrox-mcp/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func waitForImageScan(t *testing.T, client *testutil.MCPTestClient, cveName string) { + t.Helper() + + assert.Eventually(t, func() bool { + ctx := context.Background() + result, err := client.CallTool(ctx, "get_deployments_for_cve", map[string]any{ + "cveName": cveName, + }) + + if err != nil || result.IsError { + return false + } + + responseText := testutil.GetTextContent(t, result) + var data struct { + Deployments []any `json:"deployments"` + } + + if err := json.Unmarshal([]byte(responseText), &data); err != nil { + return false + } + + if len(data.Deployments) > 0 { + t.Logf("Image scan completed, found %d deployment(s) with CVE %s", len(data.Deployments), cveName) + return true + } + + t.Logf("Waiting for image scan (CVE: %s)...", cveName) + return false + }, 5*time.Minute, 5*time.Second, "Image scan did not complete for CVE %s", cveName) +} + +func TestSmoke_RealCluster(t *testing.T) { + if testing.Short() { + t.Skip("Skipping smoke test in short mode") + } + + endpoint := os.Getenv("ROX_ENDPOINT") + apiToken := os.Getenv("ROX_API_TOKEN") + password := os.Getenv("ROX_PASSWORD") + + if endpoint == "" { + t.Fatal("ROX_ENDPOINT environment variable must be set") + } + + // Generate token if password provided but no token + if apiToken == "" && password != "" { + t.Log("No API token provided, generating one using password...") + + // Wait for Central to be ready + if err := WaitForCentralReady(endpoint, password, 12); err != nil { + t.Fatalf("Failed waiting for Central: %v", err) + } + t.Log("Central API is ready") + + // Generate token + token, err := GenerateAPIToken(endpoint, password) + if err != nil { + t.Fatalf("Failed to generate API token: %v", err) + } + apiToken = token + t.Log("Successfully generated API token") + } + + if apiToken == "" { + t.Fatal("Either ROX_API_TOKEN or ROX_PASSWORD must be set") + } + + client := createSmokeTestClient(t, endpoint, apiToken) + + // nginx:1.14 has CVE-2019-9511 (HTTP/2 vulnerabilities) + waitForImageScan(t, client, "CVE-2019-9511") + + tests := map[string]struct { + toolName string + args map[string]any + validateFunc func(*testing.T, string) + }{ + "list_clusters": { + toolName: "list_clusters", + args: map[string]any{}, + validateFunc: func(t *testing.T, result string) { + t.Helper() + var data struct { + Clusters []struct { + Name string `json:"name"` + } `json:"clusters"` + } + require.NoError(t, json.Unmarshal([]byte(result), &data)) + assert.NotEmpty(t, data.Clusters, "should have at least one cluster") + t.Logf("Found %d cluster(s)", len(data.Clusters)) + }, + }, + "get_deployments_for_cve with known CVE": { + toolName: "get_deployments_for_cve", + args: map[string]any{"cveName": "CVE-2019-11043"}, + validateFunc: func(t *testing.T, result string) { + t.Helper() + var data struct { + Deployments []struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + } `json:"deployments"` + } + require.NoError(t, json.Unmarshal([]byte(result), &data)) + + if len(data.Deployments) == 0 { + t.Log("Warning: No deployments found with CVE. Deployment may not be scanned yet.") + } else { + t.Logf("Found %d deployment(s) with CVE", len(data.Deployments)) + } + }, + }, + "get_deployments_for_cve with non-existent CVE": { + toolName: "get_deployments_for_cve", + args: map[string]any{"cveName": "CVE-9999-99999"}, + validateFunc: func(t *testing.T, result string) { + t.Helper() + var data struct { + Deployments []any `json:"deployments"` + } + require.NoError(t, json.Unmarshal([]byte(result), &data)) + assert.Empty(t, data.Deployments, "should have no deployments for non-existent CVE") + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + result := testutil.CallToolAndGetResult(t, client, tt.toolName, tt.args) + responseText := testutil.GetTextContent(t, result) + tt.validateFunc(t, responseText) + }) + } +} + +func createSmokeTestClient(t *testing.T, endpoint, apiToken string) *testutil.MCPTestClient { + t.Helper() + + cfg := &config.Config{ + Central: config.CentralConfig{ + URL: endpoint, + AuthType: "static", + APIToken: apiToken, + InsecureSkipTLSVerify: true, + RequestTimeout: 30 * time.Second, + MaxRetries: 3, + InitialBackoff: time.Second, + MaxBackoff: 10 * time.Second, + }, + Server: config.ServerConfig{ + Type: "stdio", + }, + Tools: config.ToolsConfig{ + Vulnerability: config.ToolsetVulnerabilityConfig{ + Enabled: true, + }, + ConfigManager: config.ToolConfigManagerConfig{ + Enabled: true, + }, + }, + } + + runFunc := func(ctx context.Context, stdin io.ReadCloser, stdout io.WriteCloser) error { + return app.Run(ctx, cfg, stdin, stdout) + } + + client, err := testutil.NewMCPTestClient(t, runFunc) + require.NoError(t, err, "Failed to create MCP client") + t.Cleanup(func() { client.Close() }) + + return client +} diff --git a/smoke/testdata/vulnerable-deployment.yaml b/smoke/testdata/vulnerable-deployment.yaml new file mode 100644 index 0000000..bc7bdf4 --- /dev/null +++ b/smoke/testdata/vulnerable-deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: vulnerable-apps +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vulnerable-app + namespace: vulnerable-apps + labels: + app: vulnerable-app +spec: + replicas: 1 + selector: + matchLabels: + app: vulnerable-app + template: + metadata: + labels: + app: vulnerable-app + spec: + containers: + - name: vulnerable-app + image: nginx:1.14 + ports: + - containerPort: 80 + resources: + requests: + cpu: "0" + memory: "0" diff --git a/smoke/token_helper.go b/smoke/token_helper.go new file mode 100644 index 0000000..eb76b5b --- /dev/null +++ b/smoke/token_helper.go @@ -0,0 +1,115 @@ +package smoke + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +type GenerateTokenRequest struct { + Name string `json:"name"` + Role string `json:"role,omitempty"` +} + +type GenerateTokenResponse struct { + Token string `json:"token"` +} + +// GenerateAPIToken generates an API token using basic authentication. +// It calls the /v1/apitokens/generate endpoint with admin credentials. +func GenerateAPIToken(endpoint, password string) (string, error) { + tokenReq := GenerateTokenRequest{ + Name: "smoke-test-token", + Role: "Admin", + } + + reqBody, err := json.Marshal(tokenReq) + if err != nil { + return "", fmt.Errorf("failed to marshal request: %w", err) + } + + url := fmt.Sprintf("https://%s/v1/apitokens/generate", endpoint) + + req, err := http.NewRequest("POST", url, bytes.NewReader(reqBody)) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + req.SetBasicAuth("admin", password) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to make request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token generation failed (status %d): %s", resp.StatusCode, string(body)) + } + + var tokenResp GenerateTokenResponse + if err := json.Unmarshal(body, &tokenResp); err != nil { + return "", fmt.Errorf("failed to parse response: %w", err) + } + + if tokenResp.Token == "" { + return "", fmt.Errorf("received empty token in response") + } + + return tokenResp.Token, nil +} + +// WaitForCentralReady polls the /v1/ping endpoint until Central is ready. +func WaitForCentralReady(endpoint, password string, maxAttempts int) error { + url := fmt.Sprintf("https://%s/v1/ping", endpoint) + + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + for attempt := 0; attempt < maxAttempts; attempt++ { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.SetBasicAuth("admin", password) + + resp, err := client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + resp.Body.Close() + return nil + } + if resp != nil { + resp.Body.Close() + } + + // Exponential backoff: 2, 4, 8, 16... seconds (max 30) + sleepTime := 1 << attempt + if sleepTime > 30 { + sleepTime = 30 + } + time.Sleep(time.Duration(sleepTime) * time.Second) + } + + return fmt.Errorf("Central did not become ready after %d attempts", maxAttempts) +}