diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a5b0067..17bd973 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -60,16 +60,6 @@ jobs: chmod +x /usr/local/bin/roxie roxie version - - name: Install roxctl - env: - ROXCTL_VERSION: "4.10.0" - ROXCTL_SHA256: "5db647b14569465866c0162522e83393ebf02f671f4556b1b3ed551b9f8433bc" - run: | - curl -fsSLo /usr/local/bin/roxctl \ - "https://mirror.openshift.com/pub/rhacs/assets/${ROXCTL_VERSION}/bin/Linux/roxctl" - echo "${ROXCTL_SHA256} /usr/local/bin/roxctl" | sha256sum -c - - chmod +x /usr/local/bin/roxctl - roxctl version - name: Authenticate to GCloud if: inputs.cluster-type == 'gke' diff --git a/README.md b/README.md index 7d773e5..ca5184c 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,6 @@ the cluster to succeed. Prerequisites: - `kubectl` configured to point at your target cluster -- `roxctl` CLI is installed - `roxie` CLI is installed Deploy using: diff --git a/internal/deployer/crs.go b/internal/deployer/crs.go index f2bb9e9..85e6e2b 100644 --- a/internal/deployer/crs.go +++ b/internal/deployer/crs.go @@ -2,45 +2,256 @@ package deployer import ( "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" "errors" "fmt" + "io" + "net" + "net/http" + "os" + "slices" "strings" + "time" "github.com/stackrox/roxie/internal/k8s" ) -// generateCRS generates the Central Resource Secret using roxctl +const ( + ServiceCACommonName string = `StackRox Certificate Authority` + CentralCommonName string = `CENTRAL_SERVICE: Central` +) + +var ( + retryableSubstrings = []string{ + "connection refused", + "connection reset", + "connection timed out", + "timed out", + "timeout", + "network is unreachable", + "temporary failure in name resolution", + "no route to host", + "tls handshake timeout", + "eof", + "bad gateway", + "service unavailable", + "context deadline exceeded", + "no such host", + } +) + +type crsGenRequest struct { + Name string `json:"name"` +} + +type crsGenResponse struct { + CRS []byte `json:"crs"` +} + +// generateCRS is a retrying wrapper on top of generateCRSOnce. func (d *Deployer) generateCRS(ctx context.Context, clusterName string) (string, error) { - crsName := fmt.Sprintf("%s-crs", clusterName) - d.logger.Infof("Generating CRS named %q with roxctl...", crsName) - - result, err := d.runRoxctl(ctx, RoxctlOptions{ - Args: []string{ - "-e", d.centralEndpoint, - "central", - "crs", - "generate", - crsName, - "--output=-", // Output to stdout + const maxAttempts = 5 + const baseRetryDelay = 10 + + client, err := d.centralHTTPClient() + if err != nil { + return "", fmt.Errorf("failed to create HTTP client: %w", err) + } + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if attempt > 1 { + waitTime := time.Duration(attempt*baseRetryDelay) * time.Second + d.logger.Infof("Retrying CRS generation (attempt %d/%d) after %v...", attempt, maxAttempts, waitTime) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(waitTime): + } + } + + // Include attempt in name for uniqueness, in case a failure is between addition to DB and reading response. + crsName := fmt.Sprintf("%s-crs-%d", clusterName, attempt) + + crsContent, err := d.generateCRSOnce(ctx, client, crsName) + if err != nil { + if d.isRetryableError(err) { + d.logger.Warningf("Transient error generating CRS: %v", err) + lastErr = err + continue + } + return "", fmt.Errorf("CRS generation failed with non-retryable error: %w", err) + } + + d.logger.Success("✓ CRS generated") + return crsContent, nil + } + + return "", fmt.Errorf("CRS generation failed after %d attempts: %w", maxAttempts, lastErr) +} + +// centralHTTPClient returns an HTTP client configured for talking to Central. +func (d *Deployer) centralHTTPClient() (*http.Client, error) { + tlsConfig := &tls.Config{} + + if d.roxCACertFile != "" { + pemData, err := os.ReadFile(d.roxCACertFile) + if err != nil { + return nil, fmt.Errorf("reading CA cert file %s: %w", d.roxCACertFile, err) + } + pool := x509.NewCertPool() + var caCertsAdded int + for block, rest := pem.Decode(pemData); block != nil; block, rest = pem.Decode(rest) { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parsing CA certificate from %s: %w", d.roxCACertFile, err) + } + pool.AddCert(cert) + caCertsAdded++ + } + d.logger.Infof("Loaded %d CA certificate(s) from %q", caCertsAdded, d.roxCACertFile) + tlsConfig.RootCAs = pool + tlsConfig.InsecureSkipVerify = true + host, _, err := net.SplitHostPort(d.centralEndpoint) + if err != nil { + return nil, fmt.Errorf("parsing central endpoint %q: %w", d.centralEndpoint, err) + } + tlsConfig.VerifyPeerCertificate = centralVerifyFunc(host, tlsConfig) + } + + return &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, }, - UseAuthentication: true, - MaxAttempts: 5, - RetryDelay: 10, - }) + }, nil +} + +// generateCRSOnce generates a Cluster Registration Secret via Central's REST API. +func (d *Deployer) generateCRSOnce(ctx context.Context, client *http.Client, crsName string) (string, error) { + d.logger.Infof("Generating CRS named %q via Central API...", crsName) + + reqBody, err := json.Marshal(crsGenRequest{Name: crsName}) + if err != nil { + return "", fmt.Errorf("failed to marshal CRS request: %w", err) + } + + url := fmt.Sprintf("https://%s/v1/cluster-init/crs", d.centralEndpoint) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(string(reqBody))) if err != nil { - return "", fmt.Errorf("failed to generate CRS: %w", err) + return "", fmt.Errorf("failed to create HTTP request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(AdminUsername, d.centralPassword) + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("executing HTTP request: %w", err) + } + + body, readErr := io.ReadAll(resp.Body) + closeErr := resp.Body.Close() + if readErr != nil { + return "", fmt.Errorf("failed to read CRS response body: %w", readErr) + } + if closeErr != nil { + return "", fmt.Errorf("failed to close CRS response body: %w", closeErr) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("server returned status %s: %s", resp.Status, string(body)) } - crsContent := strings.TrimSpace(result.Stdout) + var crsResp crsGenResponse + if err := json.Unmarshal(body, &crsResp); err != nil { + return "", fmt.Errorf("failed to parse CRS response: %w", err) + } + + crsContent := strings.TrimSpace(string(crsResp.CRS)) if crsContent == "" { return "", errors.New("CRS content is empty") } - d.logger.Success("✓ CRS generated") return crsContent, nil } +func (d *Deployer) isRetryableError(err error) bool { + errLower := strings.ToLower(err.Error()) + return slices.ContainsFunc(retryableSubstrings, func(sub string) bool { + return strings.Contains(errLower, sub) + }) +} + +// centralVerifyFunc returns a custom TLS peer certificate verifier for Central. +// Central's self-signed serving cert uses IP SANs and the internal DNS name +// "central.stackrox" rather than the external hostname roxie connects to (e.g. +// a port-forwarded localhost or LoadBalancer IP). Go's default TLS verification +// rejects the cert because the hostname doesn't match any SAN. We work around +// this by setting InsecureSkipVerify and performing chain verification ourselves: +// first against the actual hostname (which may work if the user added a matching +// SAN), then falling back to "central.stackrox" for certs issued by Central's +// own service CA. +func centralVerifyFunc(hostname string, conf *tls.Config) func([][]byte, [][]*x509.Certificate) error { + return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("remote peer presented no certificates") + } + + certs := make([]*x509.Certificate, 0, len(rawCerts)) + for _, rawCert := range rawCerts { + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + return fmt.Errorf("failed to parse peer certificate: %w", err) + } + certs = append(certs, cert) + } + + leaf := certs[0] + intermediates := x509.NewCertPool() + for _, cert := range certs[1:] { + intermediates.AddCert(cert) + } + + systemVerifyOpts := x509.VerifyOptions{ + DNSName: hostname, + Intermediates: intermediates, + Roots: conf.RootCAs, + } + + _, systemVerifyErr := leaf.Verify(systemVerifyOpts) + if systemVerifyErr == nil || !isACentralCert(leaf) { + return systemVerifyErr + } + + serviceVerifyOpts := x509.VerifyOptions{ + DNSName: "central.stackrox", + Intermediates: intermediates, + Roots: conf.RootCAs, + } + + _, serviceVerifyErr := leaf.Verify(serviceVerifyOpts) + if serviceVerifyErr == nil { + return nil + } + return errors.Join(systemVerifyErr, serviceVerifyErr) + } +} + +// isACentralCert returns true if the cert's issuer and subject CNs claim look like central's. +func isACentralCert(cert *x509.Certificate) bool { + if cert.Issuer.CommonName != ServiceCACommonName { + return false + } + if cert.Subject.CommonName == CentralCommonName { + return true + } + return false +} + // applyCRS applies the CRS content to the sensor namespace func (d *Deployer) applyCRS(ctx context.Context, crsContent string) error { d.logger.Info("Applying CRS to sensor namespace") diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 9cc3d8b..77a1b93 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -546,7 +546,7 @@ func (d *Deployer) waitForNamespaceDeletion(namespace string) error { // checkRequiredTools verifies that required CLI tools are available func checkRequiredTools() error { - requiredTools := []string{"kubectl", "roxctl"} + requiredTools := []string{"kubectl"} var missing []string for _, tool := range requiredTools { diff --git a/internal/deployer/roxctl.go b/internal/deployer/roxctl.go deleted file mode 100644 index 5439006..0000000 --- a/internal/deployer/roxctl.go +++ /dev/null @@ -1,114 +0,0 @@ -package deployer - -import ( - "bytes" - "context" - "fmt" - "os" - "os/exec" - "strings" - "time" -) - -// RoxctlOptions contains options for running roxctl commands -type RoxctlOptions struct { - Args []string // Command arguments (e.g., ["central", "crs", "generate", "cluster-name"]) - Env map[string]string // Additional environment variables - UseAuthentication bool // Whether to set ROX_ADMIN_PASSWORD and ROX_CA_CERT_FILE - MaxAttempts int // Maximum number of retry attempts (default: 5) - RetryDelay int // Base retry delay in seconds (default: 10, multiplied by attempt number) -} - -// RoxctlResult contains the result of a roxctl command execution -type RoxctlResult struct { - Stdout string - Stderr string -} - -// runRoxctl executes a roxctl command with automatic retries on transient errors -func (d *Deployer) runRoxctl(ctx context.Context, opts RoxctlOptions) (*RoxctlResult, error) { - if opts.MaxAttempts <= 0 { - opts.MaxAttempts = 5 - } - if opts.RetryDelay <= 0 { - opts.RetryDelay = 10 - } - - // List of transient/retryable errors - retryableErrors := []string{ - "connection refused", - "connection reset", - "connection timed out", - "timed out", - "timeout", - "network is unreachable", - "temporary failure in name resolution", - "no route to host", - "tls handshake timeout", - "eof", - "bad gateway", - "service unavailable", - "context deadline exceeded", - "no such host", - } - - var lastStderr string - var lastErr error - - for attempt := 1; attempt <= opts.MaxAttempts; attempt++ { - if attempt > 1 { - waitTime := time.Duration(attempt*opts.RetryDelay) * time.Second - d.logger.Infof("Retrying roxctl command (attempt %d/%d) after %v...", attempt, opts.MaxAttempts, waitTime) - time.Sleep(waitTime) - } - - cmd := exec.CommandContext(ctx, "roxctl", opts.Args...) - - cmd.Env = os.Environ() - - if opts.UseAuthentication { - if d.centralPassword != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("ROX_ADMIN_PASSWORD=%s", d.centralPassword)) - } - if d.roxCACertFile != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("ROX_CA_CERT_FILE=%s", d.roxCACertFile)) - } - } - - for k, v := range opts.Env { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) - } - - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - lastStderr = stderr.String() - lastErr = err - - if err == nil { - return &RoxctlResult{ - Stdout: stdout.String(), - Stderr: lastStderr, - }, nil - } - - isRetryable := false - for _, retryErr := range retryableErrors { - if strings.Contains(strings.ToLower(lastStderr), strings.ToLower(retryErr)) { - isRetryable = true - break - } - } - - if !isRetryable || attempt == opts.MaxAttempts { - d.logger.Errorf("roxctl error: %s", lastStderr) - return nil, fmt.Errorf("roxctl command failed: %w", err) - } - - d.logger.Warningf("Transient error in roxctl command: %s", lastStderr) - } - - return nil, fmt.Errorf("roxctl command failed after %d attempts: %w", opts.MaxAttempts, lastErr) -}