From d45c92075e7fd2b712d492bd1a4309524a822744 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Mon, 19 Jan 2026 18:17:01 +0100 Subject: [PATCH 01/14] feat(autotls): implement autotls check functionality --- config/local.yaml | 9 + pkg/bee/api/node.go | 22 +++ pkg/bee/client.go | 24 +++ pkg/check/autotls/autotls.go | 353 +++++++++++++++++++++++++++++++++++ pkg/config/check.go | 20 ++ 5 files changed, 428 insertions(+) create mode 100644 pkg/check/autotls/autotls.go diff --git a/config/local.yaml b/config/local.yaml index a29fb7df..48d39715 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -398,3 +398,12 @@ checks: postage-depth: 21 postage-label: test-label type: feed + ci-autotls: + options: + expected-domain: "local.test" + target-node-groups: + - bootnode + - bee + connect-timeout: 30s + timeout: 5m + type: autotls diff --git a/pkg/bee/api/node.go b/pkg/bee/api/node.go index de9423bc..f7d2c6ac 100644 --- a/pkg/bee/api/node.go +++ b/pkg/bee/api/node.go @@ -122,6 +122,28 @@ func (n *NodeService) Peers(ctx context.Context) (resp Peers, err error) { return resp, err } +// ConnectResponse represents the response from the connect endpoint +type ConnectResponse struct { + Address string `json:"address"` +} + +// Connect connects to a peer using the provided multiaddress. +// The multiaddr should be in the format: /ip4/x.x.x.x/tcp/port/... +// Returns the overlay address of the connected peer. +func (n *NodeService) Connect(ctx context.Context, multiaddr string) (resp ConnectResponse, err error) { + // The bee API expects the multiaddr as a path parameter without the leading slash + // since the handler adds it back: mux.Vars(r)["multi-address"] = "/" + mux.Vars(r)["multi-address"] + path := multiaddr[1:] + + err = n.client.requestJSON(ctx, http.MethodPost, "/connect/"+path, nil, &resp) + return resp, err +} + +// Disconnect disconnects from a peer with the given overlay address. +func (n *NodeService) Disconnect(ctx context.Context, overlay swarm.Address) error { + return n.client.requestJSON(ctx, http.MethodDelete, "/peers/"+overlay.String(), nil, nil) +} + // Readiness represents node's readiness type Readiness struct { Status string `json:"status"` diff --git a/pkg/bee/client.go b/pkg/bee/client.go index d1404c50..66cbf4d2 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -318,6 +318,30 @@ func (c *Client) Peers(ctx context.Context) (peers []swarm.Address, err error) { return peers, err } +// Connect connects to a peer using the provided multiaddress. +// Returns the overlay address of the connected peer. +func (c *Client) Connect(ctx context.Context, multiaddr string) (swarm.Address, error) { + resp, err := c.api.Node.Connect(ctx, multiaddr) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("connect to %s: %w", multiaddr, err) + } + + addr, err := swarm.ParseHexAddress(resp.Address) + if err != nil { + return swarm.ZeroAddress, fmt.Errorf("parse overlay address %s: %w", resp.Address, err) + } + + return addr, nil +} + +// Disconnect disconnects from a peer with the given overlay address. +func (c *Client) Disconnect(ctx context.Context, overlay swarm.Address) error { + if err := c.api.Node.Disconnect(ctx, overlay); err != nil { + return fmt.Errorf("disconnect from %s: %w", overlay, err) + } + return nil +} + // PinRootHash pins root hash of given reference. func (c *Client) PinRootHash(ctx context.Context, ref swarm.Address) error { return c.api.Pinning.PinRootHash(ctx, ref) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go new file mode 100644 index 00000000..a9cc5147 --- /dev/null +++ b/pkg/check/autotls/autotls.go @@ -0,0 +1,353 @@ +package autotls + +import ( + "context" + "errors" + "fmt" + "net" + "regexp" + "slices" + "sort" + "strings" + "time" + + "github.com/ethersphere/bee/v2/pkg/swarm" + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" +) + +// Options represents check options +type Options struct { + ExpectedDomain string // expected domain suffix in SNI (e.g., "local.test") + TargetNodeGroups []string + ConnectTimeout time.Duration +} + +// NewDefaultOptions returns new default options +func NewDefaultOptions() Options { + return Options{ + ExpectedDomain: "", + ConnectTimeout: 30 * time.Second, + } +} + +// compile check whether Check implements interface +var _ beekeeper.Action = (*Check)(nil) + +// Check instance. +type Check struct { + logger logging.Logger +} + +// NewCheck returns a new check instance. +func NewCheck(logger logging.Logger) beekeeper.Action { + return &Check{ + logger: logger, + } +} + +// WSSUnderlay represents a parsed WSS underlay address +type WSSUnderlay struct { + Raw string + IP string + Port string + SNIDomain string + PeerID string + IsIPv6 bool +} + +var ( + errNoWSSNodes = errors.New("no nodes with WSS underlay addresses found") + errInvalidWSSFormat = errors.New("invalid WSS address format") + errWSSConnect = errors.New("WSS connection failed") +) + +// Pre-compiled regex patterns for WSS multiaddr parsing +var ( + ipv4WSSPattern = regexp.MustCompile(`^/ip4/([0-9.]+)/tcp/(\d+)/tls/sni/([^/]+)/ws/p2p/(.+)$`) + ipv6WSSPattern = regexp.MustCompile(`^/ip6/([^/]+)/tcp/(\d+)/tls/sni/([^/]+)/ws/p2p/(.+)$`) +) + +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return fmt.Errorf("invalid options type") + } + + c.logger.Info("starting AutoTLS check") + + wssNodes, err := c.discoverWSSNodes(ctx, cluster, o.TargetNodeGroups) + if err != nil { + return fmt.Errorf("discover WSS nodes: %w", err) + } + + if len(wssNodes) == 0 { + return errNoWSSNodes + } + + c.logger.Infof("found %d nodes with WSS underlays", len(wssNodes)) + + nodeNames := make([]string, 0, len(wssNodes)) + for name := range wssNodes { + nodeNames = append(nodeNames, name) + } + sort.Strings(nodeNames) + + for _, nodeName := range nodeNames { + underlays := wssNodes[nodeName] + c.logger.Infof("validating WSS addresses for node %s", nodeName) + + for _, underlay := range underlays { + parsed, err := parseWSSUnderlay(underlay) + if err != nil { + return fmt.Errorf("node %s: %w", nodeName, err) + } + + if err := c.validateWSSAddress(parsed, o.ExpectedDomain); err != nil { + return fmt.Errorf("node %s: address %s: %w", nodeName, underlay, err) + } + + c.logger.Infof("node %s: valid WSS address: %s", nodeName, underlay) + } + } + + if err := c.testWSSConnectivity(ctx, cluster, wssNodes, o); err != nil { + return fmt.Errorf("WSS connectivity test: %w", err) + } + + c.logger.Info("AutoTLS check completed successfully") + return nil +} + +// discoverWSSNodes finds all nodes that have WSS underlay addresses +func (c *Check) discoverWSSNodes(ctx context.Context, cluster orchestration.Cluster, targetGroups []string) (map[string][]string, error) { + wssNodes := make(map[string][]string) + + addresses, err := cluster.Addresses(ctx) + if err != nil { + return nil, fmt.Errorf("get cluster addresses: %w", err) + } + + for groupName, groupAddrs := range addresses { + // Filter by target groups if specified + if len(targetGroups) > 0 && !slices.Contains(targetGroups, groupName) { + continue + } + + for nodeName, nodeAddrs := range groupAddrs { + wssUnderlays := filterWSSUnderlays(nodeAddrs.Underlay) + if len(wssUnderlays) > 0 { + wssNodes[nodeName] = wssUnderlays + c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) + } + } + } + + return wssNodes, nil +} + +// filterWSSUnderlays returns only underlay addresses that contain TLS/WSS components +func filterWSSUnderlays(underlays []string) []string { + var wss []string + for _, u := range underlays { + if isWSSUnderlay(u) { + wss = append(wss, u) + } + } + return wss +} + +// isWSSUnderlay checks if an underlay address is a WSS address +// WSS addresses contain /tls/ and /ws/ components +func isWSSUnderlay(underlay string) bool { + return strings.Contains(underlay, "/tls/") && strings.Contains(underlay, "/ws/") +} + +// parseWSSUnderlay parses a WSS multiaddr into its components +// Expected formats: +// - IPv4: /ip4/x.x.x.x/tcp/port/tls/sni/domain/ws/p2p/peerID +// - IPv6: /ip6/::1/tcp/port/tls/sni/domain/ws/p2p/peerID +func parseWSSUnderlay(underlay string) (*WSSUnderlay, error) { + re := ipv4WSSPattern + matches := re.FindStringSubmatch(underlay) + if matches != nil { + return &WSSUnderlay{ + Raw: underlay, + IP: matches[1], + Port: matches[2], + SNIDomain: matches[3], + PeerID: matches[4], + IsIPv6: false, + }, nil + } + + re = ipv6WSSPattern + matches = re.FindStringSubmatch(underlay) + if matches != nil { + return &WSSUnderlay{ + Raw: underlay, + IP: matches[1], + Port: matches[2], + SNIDomain: matches[3], + PeerID: matches[4], + IsIPv6: true, + }, nil + } + + return nil, fmt.Errorf("%w: %s", errInvalidWSSFormat, underlay) +} + +// validateWSSAddress validates the WSS address components +func (c *Check) validateWSSAddress(wss *WSSUnderlay, expectedDomain string) error { + if wss.IP == "" { + return fmt.Errorf("missing IP address") + } + + if ip := net.ParseIP(wss.IP); ip == nil { + return fmt.Errorf("invalid IP address: %s", wss.IP) + } + + if wss.Port == "" { + return fmt.Errorf("missing port") + } + + if wss.SNIDomain == "" { + return fmt.Errorf("missing SNI domain") + } + + if expectedDomain != "" && !strings.HasSuffix(wss.SNIDomain, expectedDomain) { + return fmt.Errorf("SNI domain %q does not have expected suffix %q", wss.SNIDomain, expectedDomain) + } + + if !isValidAutoTLSSNI(wss.SNIDomain, wss.IP, wss.IsIPv6) { + c.logger.Warningf("SNI domain %s may not follow AutoTLS naming convention for IP %s", wss.SNIDomain, wss.IP) + } + + if wss.PeerID == "" { + return fmt.Errorf("missing peer ID") + } + + return nil +} + +func isValidAutoTLSSNI(sni, ip string, isIPv6 bool) bool { + var ipWithDashes string + if isIPv6 { + ipWithDashes = ip + if strings.HasPrefix(ipWithDashes, "::") { + ipWithDashes = "0" + ipWithDashes + } + ipWithDashes = strings.ReplaceAll(ipWithDashes, ":", "-") + } else { + ipWithDashes = strings.ReplaceAll(ip, ".", "-") + } + return strings.HasPrefix(sni, ipWithDashes+".") +} + +func (c *Check) testWSSConnectivity(ctx context.Context, cluster orchestration.Cluster, wssNodes map[string][]string, opts Options) error { + clients, err := cluster.NodesClients(ctx) + if err != nil { + return fmt.Errorf("get node clients: %w", err) + } + + if len(clients) == 0 { + return fmt.Errorf("no nodes available for connectivity test") + } + + clientNames := make([]string, 0, len(clients)) + for name := range clients { + clientNames = append(clientNames, name) + } + sort.Strings(clientNames) + + var nonWSSSource *bee.Client + var nonWSSName string + var wssSource *bee.Client + var wssSourceName string + + for _, name := range clientNames { + client := clients[name] + if _, hasWSS := wssNodes[name]; hasWSS { + if wssSource == nil { + wssSource = client + wssSourceName = name + } + } else { + if nonWSSSource == nil { + nonWSSSource = client + nonWSSName = name + } + } + } + + targetNames := make([]string, 0, len(wssNodes)) + for name := range wssNodes { + targetNames = append(targetNames, name) + } + sort.Strings(targetNames) + + if nonWSSSource != nil { + c.logger.Infof("testing cross-protocol: %s (non-WSS) to WSS nodes", nonWSSName) + if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, wssNodes, targetNames, opts); err != nil { + return fmt.Errorf("cross-protocol test: %w", err) + } + } else { + c.logger.Warning("no non-WSS nodes available, skipping cross-protocol test") + } + + if wssSource != nil { + c.logger.Infof("testing WSS-to-WSS: %s to WSS nodes", wssSourceName) + if err := c.testConnectivity(ctx, wssSource, wssSourceName, wssNodes, targetNames, opts); err != nil { + return fmt.Errorf("WSS-to-WSS test: %w", err) + } + } else { + c.logger.Warning("no WSS source nodes available, skipping WSS-to-WSS test") + } + + return nil +} + +func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, wssNodes map[string][]string, targetNames []string, opts Options) error { + for _, targetName := range targetNames { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if targetName == sourceName { + continue + } + + underlays := wssNodes[targetName] + for _, underlay := range underlays { + c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) + + connectCtx, cancel := context.WithTimeout(ctx, opts.ConnectTimeout) + start := time.Now() + + overlay, err := sourceClient.Connect(connectCtx, underlay) + duration := time.Since(start) + cancel() + + if err != nil { + return fmt.Errorf("%w: from %s to %s via %s: %v", errWSSConnect, sourceName, targetName, underlay, err) + } + + c.logger.Infof("WSS connection successful: %s to %s (overlay: %s, duration: %v)", + sourceName, targetName, overlay, duration) + + if overlay.Equal(swarm.ZeroAddress) { + return fmt.Errorf("received zero overlay address after connecting to %s", targetName) + } + + if err := sourceClient.Disconnect(ctx, overlay); err != nil { + return fmt.Errorf("failed to disconnect from %s: %w", targetName, err) + } + } + } + + return nil +} diff --git a/pkg/config/check.go b/pkg/config/check.go index b130c6e4..d18907fe 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -8,6 +8,7 @@ import ( "github.com/ethersphere/beekeeper/pkg/beekeeper" "github.com/ethersphere/beekeeper/pkg/check/act" + "github.com/ethersphere/beekeeper/pkg/check/autotls" "github.com/ethersphere/beekeeper/pkg/check/balances" "github.com/ethersphere/beekeeper/pkg/check/cashout" "github.com/ethersphere/beekeeper/pkg/check/datadurability" @@ -82,6 +83,25 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "autotls": { + NewAction: autotls.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + ExpectedDomain *string `yaml:"expected-domain"` + TargetNodeGroups *[]string `yaml:"target-node-groups"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := autotls.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "balances": { NewAction: balances.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { From ac81bb3b49c5f00967327d9869a12e3213524b30 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 20 Jan 2026 12:58:21 +0100 Subject: [PATCH 02/14] feat(autotls): enhance WSS connectivity checks and update configuration --- config/local.yaml | 16 ++- go.mod | 2 +- pkg/bee/api/node.go | 6 +- pkg/check/autotls/autotls.go | 244 ++++++----------------------------- pkg/config/check.go | 15 +-- 5 files changed, 55 insertions(+), 228 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 48d39715..699e7706 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -25,6 +25,11 @@ clusters: config: local count: 2 mode: node + wss: + bee-config: bee-local-wss + config: local + count: 2 + mode: node local-dns: _inherit: "local" node-groups: @@ -134,7 +139,7 @@ bee-configs: p2p-addr: ":1634" p2p-wss-addr: ":1635" p2p-ws-enable: false - p2p-wss-enable: true + p2p-wss-enable: false password: "beekeeper" payment-early-percent: 50 payment-threshold: 13500000 @@ -157,6 +162,9 @@ bee-configs: bootnode-local: _inherit: "bee-local" bootnode-mode: true + bee-local-wss: + _inherit: "bee-local" + p2p-wss-enable: true bee-local-dns: _inherit: "bee-local" bootnode: /dnsaddr/localhost @@ -399,11 +407,5 @@ checks: postage-label: test-label type: feed ci-autotls: - options: - expected-domain: "local.test" - target-node-groups: - - bootnode - - bee - connect-timeout: 30s timeout: 5m type: autotls diff --git a/go.mod b/go.mod index ea704f2f..76f69b8d 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-git/go-git/v5 v5.13.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 + github.com/multiformats/go-multiaddr v0.12.3 github.com/opentracing/opentracing-go v1.2.0 github.com/prometheus/client_golang v1.21.1 github.com/prometheus/common v0.62.0 @@ -94,7 +95,6 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.12.3 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect diff --git a/pkg/bee/api/node.go b/pkg/bee/api/node.go index f7d2c6ac..789eeccc 100644 --- a/pkg/bee/api/node.go +++ b/pkg/bee/api/node.go @@ -131,11 +131,7 @@ type ConnectResponse struct { // The multiaddr should be in the format: /ip4/x.x.x.x/tcp/port/... // Returns the overlay address of the connected peer. func (n *NodeService) Connect(ctx context.Context, multiaddr string) (resp ConnectResponse, err error) { - // The bee API expects the multiaddr as a path parameter without the leading slash - // since the handler adds it back: mux.Vars(r)["multi-address"] = "/" + mux.Vars(r)["multi-address"] - path := multiaddr[1:] - - err = n.client.requestJSON(ctx, http.MethodPost, "/connect/"+path, nil, &resp) + err = n.client.requestJSON(ctx, http.MethodPost, "/connect"+multiaddr, nil, &resp) return resp, err } diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index a9cc5147..d7b2860a 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -4,11 +4,6 @@ import ( "context" "errors" "fmt" - "net" - "regexp" - "slices" - "sort" - "strings" "time" "github.com/ethersphere/bee/v2/pkg/swarm" @@ -16,60 +11,31 @@ import ( "github.com/ethersphere/beekeeper/pkg/beekeeper" "github.com/ethersphere/beekeeper/pkg/logging" "github.com/ethersphere/beekeeper/pkg/orchestration" + ma "github.com/multiformats/go-multiaddr" ) -// Options represents check options type Options struct { - ExpectedDomain string // expected domain suffix in SNI (e.g., "local.test") - TargetNodeGroups []string - ConnectTimeout time.Duration + ConnectTimeout time.Duration } -// NewDefaultOptions returns new default options func NewDefaultOptions() Options { return Options{ - ExpectedDomain: "", ConnectTimeout: 30 * time.Second, } } -// compile check whether Check implements interface var _ beekeeper.Action = (*Check)(nil) -// Check instance. type Check struct { logger logging.Logger } -// NewCheck returns a new check instance. func NewCheck(logger logging.Logger) beekeeper.Action { return &Check{ logger: logger, } } -// WSSUnderlay represents a parsed WSS underlay address -type WSSUnderlay struct { - Raw string - IP string - Port string - SNIDomain string - PeerID string - IsIPv6 bool -} - -var ( - errNoWSSNodes = errors.New("no nodes with WSS underlay addresses found") - errInvalidWSSFormat = errors.New("invalid WSS address format") - errWSSConnect = errors.New("WSS connection failed") -) - -// Pre-compiled regex patterns for WSS multiaddr parsing -var ( - ipv4WSSPattern = regexp.MustCompile(`^/ip4/([0-9.]+)/tcp/(\d+)/tls/sni/([^/]+)/ws/p2p/(.+)$`) - ipv6WSSPattern = regexp.MustCompile(`^/ip6/([^/]+)/tcp/(\d+)/tls/sni/([^/]+)/ws/p2p/(.+)$`) -) - func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { o, ok := opts.(Options) if !ok { @@ -78,42 +44,29 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any c.logger.Info("starting AutoTLS check") - wssNodes, err := c.discoverWSSNodes(ctx, cluster, o.TargetNodeGroups) + clients, err := cluster.NodesClients(ctx) + if err != nil { + return fmt.Errorf("get node clients: %w", err) + } + + wssNodes, err := c.discoverWSSNodes(ctx, clients) if err != nil { return fmt.Errorf("discover WSS nodes: %w", err) } if len(wssNodes) == 0 { - return errNoWSSNodes + return errors.New("no nodes with WSS underlay addresses found") } c.logger.Infof("found %d nodes with WSS underlays", len(wssNodes)) - nodeNames := make([]string, 0, len(wssNodes)) - for name := range wssNodes { - nodeNames = append(nodeNames, name) - } - sort.Strings(nodeNames) - - for _, nodeName := range nodeNames { - underlays := wssNodes[nodeName] - c.logger.Infof("validating WSS addresses for node %s", nodeName) - + for nodeName, underlays := range wssNodes { for _, underlay := range underlays { - parsed, err := parseWSSUnderlay(underlay) - if err != nil { - return fmt.Errorf("node %s: %w", nodeName, err) - } - - if err := c.validateWSSAddress(parsed, o.ExpectedDomain); err != nil { - return fmt.Errorf("node %s: address %s: %w", nodeName, underlay, err) - } - - c.logger.Infof("node %s: valid WSS address: %s", nodeName, underlay) + c.logger.Infof("node %s: WSS address: %s", nodeName, underlay) } } - if err := c.testWSSConnectivity(ctx, cluster, wssNodes, o); err != nil { + if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { return fmt.Errorf("WSS connectivity test: %w", err) } @@ -121,154 +74,50 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -// discoverWSSNodes finds all nodes that have WSS underlay addresses -func (c *Check) discoverWSSNodes(ctx context.Context, cluster orchestration.Cluster, targetGroups []string) (map[string][]string, error) { +func (c *Check) discoverWSSNodes(ctx context.Context, clients map[string]*bee.Client) (map[string][]string, error) { wssNodes := make(map[string][]string) - addresses, err := cluster.Addresses(ctx) - if err != nil { - return nil, fmt.Errorf("get cluster addresses: %w", err) - } - - for groupName, groupAddrs := range addresses { - // Filter by target groups if specified - if len(targetGroups) > 0 && !slices.Contains(targetGroups, groupName) { - continue + for nodeName, client := range clients { + addresses, err := client.Addresses(ctx) + if err != nil { + return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) } - for nodeName, nodeAddrs := range groupAddrs { - wssUnderlays := filterWSSUnderlays(nodeAddrs.Underlay) - if len(wssUnderlays) > 0 { - wssNodes[nodeName] = wssUnderlays - c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) - } + wssUnderlays := filterWSSUnderlays(addresses.Underlay) + if len(wssUnderlays) > 0 { + wssNodes[nodeName] = wssUnderlays + c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) } } return wssNodes, nil } -// filterWSSUnderlays returns only underlay addresses that contain TLS/WSS components func filterWSSUnderlays(underlays []string) []string { var wss []string for _, u := range underlays { - if isWSSUnderlay(u) { - wss = append(wss, u) + maddr, err := ma.NewMultiaddr(u) + if err != nil { + continue } - } - return wss -} - -// isWSSUnderlay checks if an underlay address is a WSS address -// WSS addresses contain /tls/ and /ws/ components -func isWSSUnderlay(underlay string) bool { - return strings.Contains(underlay, "/tls/") && strings.Contains(underlay, "/ws/") -} - -// parseWSSUnderlay parses a WSS multiaddr into its components -// Expected formats: -// - IPv4: /ip4/x.x.x.x/tcp/port/tls/sni/domain/ws/p2p/peerID -// - IPv6: /ip6/::1/tcp/port/tls/sni/domain/ws/p2p/peerID -func parseWSSUnderlay(underlay string) (*WSSUnderlay, error) { - re := ipv4WSSPattern - matches := re.FindStringSubmatch(underlay) - if matches != nil { - return &WSSUnderlay{ - Raw: underlay, - IP: matches[1], - Port: matches[2], - SNIDomain: matches[3], - PeerID: matches[4], - IsIPv6: false, - }, nil - } - - re = ipv6WSSPattern - matches = re.FindStringSubmatch(underlay) - if matches != nil { - return &WSSUnderlay{ - Raw: underlay, - IP: matches[1], - Port: matches[2], - SNIDomain: matches[3], - PeerID: matches[4], - IsIPv6: true, - }, nil - } - - return nil, fmt.Errorf("%w: %s", errInvalidWSSFormat, underlay) -} - -// validateWSSAddress validates the WSS address components -func (c *Check) validateWSSAddress(wss *WSSUnderlay, expectedDomain string) error { - if wss.IP == "" { - return fmt.Errorf("missing IP address") - } - - if ip := net.ParseIP(wss.IP); ip == nil { - return fmt.Errorf("invalid IP address: %s", wss.IP) - } - - if wss.Port == "" { - return fmt.Errorf("missing port") - } - - if wss.SNIDomain == "" { - return fmt.Errorf("missing SNI domain") - } - - if expectedDomain != "" && !strings.HasSuffix(wss.SNIDomain, expectedDomain) { - return fmt.Errorf("SNI domain %q does not have expected suffix %q", wss.SNIDomain, expectedDomain) - } - - if !isValidAutoTLSSNI(wss.SNIDomain, wss.IP, wss.IsIPv6) { - c.logger.Warningf("SNI domain %s may not follow AutoTLS naming convention for IP %s", wss.SNIDomain, wss.IP) - } - - if wss.PeerID == "" { - return fmt.Errorf("missing peer ID") - } - - return nil -} - -func isValidAutoTLSSNI(sni, ip string, isIPv6 bool) bool { - var ipWithDashes string - if isIPv6 { - ipWithDashes = ip - if strings.HasPrefix(ipWithDashes, "::") { - ipWithDashes = "0" + ipWithDashes + if _, err := maddr.ValueForProtocol(ma.P_TLS); err != nil { + continue } - ipWithDashes = strings.ReplaceAll(ipWithDashes, ":", "-") - } else { - ipWithDashes = strings.ReplaceAll(ip, ".", "-") + if _, err := maddr.ValueForProtocol(ma.P_WS); err != nil { + continue + } + wss = append(wss, u) } - return strings.HasPrefix(sni, ipWithDashes+".") + return wss } -func (c *Check) testWSSConnectivity(ctx context.Context, cluster orchestration.Cluster, wssNodes map[string][]string, opts Options) error { - clients, err := cluster.NodesClients(ctx) - if err != nil { - return fmt.Errorf("get node clients: %w", err) - } - - if len(clients) == 0 { - return fmt.Errorf("no nodes available for connectivity test") - } - - clientNames := make([]string, 0, len(clients)) - for name := range clients { - clientNames = append(clientNames, name) - } - sort.Strings(clientNames) - +func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { var nonWSSSource *bee.Client var nonWSSName string var wssSource *bee.Client var wssSourceName string - for _, name := range clientNames { - client := clients[name] + for name, client := range clients { if _, hasWSS := wssNodes[name]; hasWSS { if wssSource == nil { wssSource = client @@ -282,15 +131,9 @@ func (c *Check) testWSSConnectivity(ctx context.Context, cluster orchestration.C } } - targetNames := make([]string, 0, len(wssNodes)) - for name := range wssNodes { - targetNames = append(targetNames, name) - } - sort.Strings(targetNames) - if nonWSSSource != nil { c.logger.Infof("testing cross-protocol: %s (non-WSS) to WSS nodes", nonWSSName) - if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, wssNodes, targetNames, opts); err != nil { + if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, wssNodes, timeout); err != nil { return fmt.Errorf("cross-protocol test: %w", err) } } else { @@ -299,7 +142,7 @@ func (c *Check) testWSSConnectivity(ctx context.Context, cluster orchestration.C if wssSource != nil { c.logger.Infof("testing WSS-to-WSS: %s to WSS nodes", wssSourceName) - if err := c.testConnectivity(ctx, wssSource, wssSourceName, wssNodes, targetNames, opts); err != nil { + if err := c.testConnectivity(ctx, wssSource, wssSourceName, wssNodes, timeout); err != nil { return fmt.Errorf("WSS-to-WSS test: %w", err) } } else { @@ -309,23 +152,22 @@ func (c *Check) testWSSConnectivity(ctx context.Context, cluster orchestration.C return nil } -func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, wssNodes map[string][]string, targetNames []string, opts Options) error { - for _, targetName := range targetNames { +func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, wssNodes map[string][]string, timeout time.Duration) error { + for targetName, underlays := range wssNodes { + if targetName == sourceName { + continue + } + select { case <-ctx.Done(): return ctx.Err() default: } - if targetName == sourceName { - continue - } - - underlays := wssNodes[targetName] for _, underlay := range underlays { c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) - connectCtx, cancel := context.WithTimeout(ctx, opts.ConnectTimeout) + connectCtx, cancel := context.WithTimeout(ctx, timeout) start := time.Now() overlay, err := sourceClient.Connect(connectCtx, underlay) @@ -333,7 +175,7 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, cancel() if err != nil { - return fmt.Errorf("%w: from %s to %s via %s: %v", errWSSConnect, sourceName, targetName, underlay, err) + return fmt.Errorf("WSS connection failed from %s to %s via %s: %w", sourceName, targetName, underlay, err) } c.logger.Infof("WSS connection successful: %s to %s (overlay: %s, duration: %v)", diff --git a/pkg/config/check.go b/pkg/config/check.go index d18907fe..ff622180 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -86,20 +86,7 @@ var Checks = map[string]CheckType{ "autotls": { NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { - checkOpts := new(struct { - ExpectedDomain *string `yaml:"expected-domain"` - TargetNodeGroups *[]string `yaml:"target-node-groups"` - ConnectTimeout *time.Duration `yaml:"connect-timeout"` - }) - if err := check.Options.Decode(checkOpts); err != nil { - return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) - } - opts := autotls.NewDefaultOptions() - - if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { - return nil, fmt.Errorf("applying options: %w", err) - } - return opts, nil + return autotls.NewDefaultOptions(), nil }, }, "balances": { From 1bd32371756443c57799f4467f2bdbe867b6c1df Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 20 Jan 2026 15:10:54 +0100 Subject: [PATCH 03/14] feat(autotls): add autotls check in config.yaml --- config/config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 9666b09a..af8a0555 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -399,7 +399,9 @@ checks: postage-depth: 21 postage-label: test-label type: feed - + autotls: + timeout: 5m + type: autotls # simulations defines simulations Beekeeper can execute against the cluster # type filed allows defining same simulation with different names and options simulations: From 836cc6b5cde750a616365284c1c457a890cca488 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Wed, 21 Jan 2026 12:00:50 +0100 Subject: [PATCH 04/14] feat(config): add WSS configuration for local setup --- config/local.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/local.yaml b/config/local.yaml index 699e7706..7d1eac77 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -47,6 +47,11 @@ clusters: config: local-dns count: 3 mode: node + wss: + bee-config: bee-local-wss + config: local-dns + count: 2 + mode: node light: bee-config: bee-local-light config: local-light From b0d601f58fe8d16a594168b45cbb3c239295c7f4 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Wed, 21 Jan 2026 12:58:25 +0100 Subject: [PATCH 05/14] fix(autotls): ensure context cancellation is handled correctly during WSS connection tests --- pkg/check/autotls/autotls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index d7b2860a..5a15588e 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -168,11 +168,11 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) connectCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() start := time.Now() overlay, err := sourceClient.Connect(connectCtx, underlay) duration := time.Since(start) - cancel() if err != nil { return fmt.Errorf("WSS connection failed from %s to %s via %s: %w", sourceName, targetName, underlay, err) From a1488a2f3163334bdc23ab124b450c53e735e399 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Wed, 21 Jan 2026 16:57:11 +0100 Subject: [PATCH 06/14] fix(autotls): update WSS connectivity test to disconnect everything before connecting --- pkg/check/autotls/autotls.go | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index 5a15588e..fd8b6e9e 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/ethersphere/beekeeper/pkg/bee" "github.com/ethersphere/beekeeper/pkg/beekeeper" "github.com/ethersphere/beekeeper/pkg/logging" @@ -133,7 +132,7 @@ func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee if nonWSSSource != nil { c.logger.Infof("testing cross-protocol: %s (non-WSS) to WSS nodes", nonWSSName) - if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, wssNodes, timeout); err != nil { + if err := c.testConnectivity(ctx, nonWSSSource, nonWSSName, clients, wssNodes, timeout); err != nil { return fmt.Errorf("cross-protocol test: %w", err) } } else { @@ -142,7 +141,7 @@ func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee if wssSource != nil { c.logger.Infof("testing WSS-to-WSS: %s to WSS nodes", wssSourceName) - if err := c.testConnectivity(ctx, wssSource, wssSourceName, wssNodes, timeout); err != nil { + if err := c.testConnectivity(ctx, wssSource, wssSourceName, clients, wssNodes, timeout); err != nil { return fmt.Errorf("WSS-to-WSS test: %w", err) } } else { @@ -152,7 +151,7 @@ func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee return nil } -func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, wssNodes map[string][]string, timeout time.Duration) error { +func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { for targetName, underlays := range wssNodes { if targetName == sourceName { continue @@ -164,15 +163,32 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, default: } + targetClient := clients[targetName] + targetAddresses, err := targetClient.Addresses(ctx) + if err != nil { + return fmt.Errorf("get target %s addresses: %w", targetName, err) + } + targetOverlay := targetAddresses.Overlay + + // Disconnect first to ensure we test actual WSS connection. + // Bee returns 200 OK for both new connections and existing ones, + // so we must disconnect first to guarantee WSS transport is used. + c.logger.Infof("disconnecting from %s before WSS test", targetName) + if err := sourceClient.Disconnect(ctx, targetOverlay); err != nil { + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) + } + + time.Sleep(500 * time.Millisecond) + for _, underlay := range underlays { c.logger.Infof("testing WSS connection from %s to %s via %s", sourceName, targetName, underlay) connectCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() start := time.Now() overlay, err := sourceClient.Connect(connectCtx, underlay) duration := time.Since(start) + cancel() if err != nil { return fmt.Errorf("WSS connection failed from %s to %s via %s: %w", sourceName, targetName, underlay, err) @@ -181,13 +197,16 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, c.logger.Infof("WSS connection successful: %s to %s (overlay: %s, duration: %v)", sourceName, targetName, overlay, duration) - if overlay.Equal(swarm.ZeroAddress) { - return fmt.Errorf("received zero overlay address after connecting to %s", targetName) + if !overlay.Equal(targetOverlay) { + return fmt.Errorf("overlay mismatch: expected %s, got %s", targetOverlay, overlay) } if err := sourceClient.Disconnect(ctx, overlay); err != nil { - return fmt.Errorf("failed to disconnect from %s: %w", targetName, err) + c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) } + + // Wait to avoid auto reconnect interference + time.Sleep(500 * time.Millisecond) } } From 43d3bf37580dc55d66a594091c261933227ca1a2 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 22 Jan 2026 13:01:32 +0100 Subject: [PATCH 07/14] fix(autotls): add WSS group option and enhance error handling for WSS connectivity --- pkg/check/autotls/autotls.go | 35 +++++++++++++++++------------------ pkg/config/check.go | 14 +++++++++++++- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index fd8b6e9e..a005d9ff 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -2,7 +2,6 @@ package autotls import ( "context" - "errors" "fmt" "time" @@ -14,11 +13,13 @@ import ( ) type Options struct { + WSSGroup string ConnectTimeout time.Duration } func NewDefaultOptions() Options { return Options{ + WSSGroup: "wss", ConnectTimeout: 30 * time.Second, } } @@ -48,21 +49,16 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("get node clients: %w", err) } - wssNodes, err := c.discoverWSSNodes(ctx, clients) - if err != nil { - return fmt.Errorf("discover WSS nodes: %w", err) - } - - if len(wssNodes) == 0 { - return errors.New("no nodes with WSS underlay addresses found") + wssClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.WSSGroup}) + if len(wssClients) == 0 { + return fmt.Errorf("no nodes found in WSS group %q", o.WSSGroup) } - c.logger.Infof("found %d nodes with WSS underlays", len(wssNodes)) + c.logger.Infof("found %d nodes in WSS group %q", len(wssClients), o.WSSGroup) - for nodeName, underlays := range wssNodes { - for _, underlay := range underlays { - c.logger.Infof("node %s: WSS address: %s", nodeName, underlay) - } + wssNodes, err := c.verifyWSSUnderlays(ctx, wssClients) + if err != nil { + return fmt.Errorf("verify WSS underlays: %w", err) } if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { @@ -73,20 +69,23 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -func (c *Check) discoverWSSNodes(ctx context.Context, clients map[string]*bee.Client) (map[string][]string, error) { +func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration.ClientList) (map[string][]string, error) { wssNodes := make(map[string][]string) - for nodeName, client := range clients { + for _, client := range wssClients { + nodeName := client.Name() addresses, err := client.Addresses(ctx) if err != nil { return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) } wssUnderlays := filterWSSUnderlays(addresses.Underlay) - if len(wssUnderlays) > 0 { - wssNodes[nodeName] = wssUnderlays - c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) + if len(wssUnderlays) == 0 { + return nil, fmt.Errorf("node %s in WSS group has no WSS underlay addresses", nodeName) } + + wssNodes[nodeName] = wssUnderlays + c.logger.Debugf("node %s has %d WSS underlay(s)", nodeName, len(wssUnderlays)) } return wssNodes, nil diff --git a/pkg/config/check.go b/pkg/config/check.go index ff622180..256057a2 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -86,7 +86,19 @@ var Checks = map[string]CheckType{ "autotls": { NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { - return autotls.NewDefaultOptions(), nil + checkOpts := new(struct { + WSSGroup *string `yaml:"wss-group"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := autotls.NewDefaultOptions() + + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil }, }, "balances": { From ca375b7f9ab94d1cf79ade58644bfd152f61ed87 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Fri, 23 Jan 2026 12:55:58 +0100 Subject: [PATCH 08/14] feat(autotls): add certificate renewal testing options and enhance configuration --- pkg/check/autotls/autotls.go | 38 +++++++++++++++++++++++++++++++----- pkg/config/check.go | 6 ++++-- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index a005d9ff..d9813b62 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -13,14 +13,18 @@ import ( ) type Options struct { - WSSGroup string - ConnectTimeout time.Duration + WSSGroup string + ConnectTimeout time.Duration + TestRenewal bool + RenewalWaitTime time.Duration } func NewDefaultOptions() Options { return Options{ - WSSGroup: "wss", - ConnectTimeout: 30 * time.Second, + WSSGroup: "wss", + ConnectTimeout: 30 * time.Second, + TestRenewal: true, + RenewalWaitTime: 5 * time.Minute, // Make sure this time is aligned with config in beelocal repo: https://github.com/ethersphere/beelocal } } @@ -65,6 +69,12 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("WSS connectivity test: %w", err) } + if o.TestRenewal { + if err := c.testCertificateRenewal(ctx, clients, wssNodes, o); err != nil { + return fmt.Errorf("certificate renewal test: %w", err) + } + } + c.logger.Info("AutoTLS check completed successfully") return nil } @@ -204,10 +214,28 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, c.logger.Warningf("failed to disconnect from %s: %v", targetName, err) } - // Wait to avoid auto reconnect interference time.Sleep(500 * time.Millisecond) } } return nil } + +func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, o Options) error { + c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", o.RenewalWaitTime) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(o.RenewalWaitTime): + } + + c.logger.Info("wait complete, re-testing WSS connectivity to verify certificates were renewed") + + if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + return fmt.Errorf("post-renewal connectivity test failed (certificates may not have been renewed): %w", err) + } + + c.logger.Info("certificate renewal test passed: WSS connectivity still works after wait period") + return nil +} diff --git a/pkg/config/check.go b/pkg/config/check.go index 256057a2..cd0f7cbe 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -87,8 +87,10 @@ var Checks = map[string]CheckType{ NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - WSSGroup *string `yaml:"wss-group"` - ConnectTimeout *time.Duration `yaml:"connect-timeout"` + WSSGroup *string `yaml:"wss-group"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` + TestRenewal *bool `yaml:"test-renewal"` + RenewalWaitTime *time.Duration `yaml:"renewal-wait-time"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From 48c5398fd0d0721ef7109a96e0ac84dd1204ff4c Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 27 Jan 2026 16:30:50 +0100 Subject: [PATCH 09/14] refactor(autotls): simplify certificate renewal options and update related configurations --- pkg/check/autotls/autotls.go | 28 ++++++++++++---------------- pkg/config/check.go | 6 ++---- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index d9813b62..dc89f310 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -13,18 +13,14 @@ import ( ) type Options struct { - WSSGroup string - ConnectTimeout time.Duration - TestRenewal bool - RenewalWaitTime time.Duration + WSSGroup string + ConnectTimeout time.Duration } func NewDefaultOptions() Options { return Options{ - WSSGroup: "wss", - ConnectTimeout: 30 * time.Second, - TestRenewal: true, - RenewalWaitTime: 5 * time.Minute, // Make sure this time is aligned with config in beelocal repo: https://github.com/ethersphere/beelocal + WSSGroup: "wss", + ConnectTimeout: 30 * time.Second, } } @@ -69,10 +65,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("WSS connectivity test: %w", err) } - if o.TestRenewal { - if err := c.testCertificateRenewal(ctx, clients, wssNodes, o); err != nil { - return fmt.Errorf("certificate renewal test: %w", err) - } + if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + return fmt.Errorf("certificate renewal test: %w", err) } c.logger.Info("AutoTLS check completed successfully") @@ -221,18 +215,20 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, return nil } -func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, o Options) error { - c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", o.RenewalWaitTime) +func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, connectTimeout time.Duration) error { + const renewalWaitTime = 350 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) + + c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", renewalWaitTime) select { case <-ctx.Done(): return ctx.Err() - case <-time.After(o.RenewalWaitTime): + case <-time.After(renewalWaitTime): } c.logger.Info("wait complete, re-testing WSS connectivity to verify certificates were renewed") - if err := c.testWSSConnectivity(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + if err := c.testWSSConnectivity(ctx, clients, wssNodes, connectTimeout); err != nil { return fmt.Errorf("post-renewal connectivity test failed (certificates may not have been renewed): %w", err) } diff --git a/pkg/config/check.go b/pkg/config/check.go index cd0f7cbe..256057a2 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -87,10 +87,8 @@ var Checks = map[string]CheckType{ NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - WSSGroup *string `yaml:"wss-group"` - ConnectTimeout *time.Duration `yaml:"connect-timeout"` - TestRenewal *bool `yaml:"test-renewal"` - RenewalWaitTime *time.Duration `yaml:"renewal-wait-time"` + WSSGroup *string `yaml:"wss-group"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From 4e3b107345ed58b6708159506f584876e891bcf6 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 27 Jan 2026 18:22:38 +0100 Subject: [PATCH 10/14] fix(autotls): increase certificate renewal wait time to 500 seconds for improved testing --- pkg/check/autotls/autotls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index dc89f310..4fe4faa8 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -216,7 +216,7 @@ func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, } func (c *Check) testCertificateRenewal(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, connectTimeout time.Duration) error { - const renewalWaitTime = 350 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) + const renewalWaitTime = 500 * time.Second // This is configured in beelocal setup (we set certificate to expire in 300 seconds) c.logger.Infof("testing certificate renewal: waiting %v then re-testing connectivity", renewalWaitTime) From 10ff08439f37f681658cc311465311261df334cb Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 27 Jan 2026 19:14:03 +0100 Subject: [PATCH 11/14] fix(autotls): extend timeout for certificate renewal to 15 minutes --- config/local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/local.yaml b/config/local.yaml index 7d1eac77..b0d65791 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -412,5 +412,5 @@ checks: postage-label: test-label type: feed ci-autotls: - timeout: 5m + timeout: 15m type: autotls From 7c95858d80d646b46ad9885bdc69a33e779d4820 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Wed, 4 Feb 2026 09:59:03 +0100 Subject: [PATCH 12/14] feat(autotls): add ultralight group support and connectivity testing --- config/local.yaml | 18 ++++++++++++++++ pkg/check/autotls/autotls.go | 40 ++++++++++++++++++++++++++++++------ pkg/config/check.go | 5 +++-- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index b0d65791..d3c6d10a 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -30,6 +30,11 @@ clusters: config: local count: 2 mode: node + ultralight: + bee-config: bee-local-ultralight + config: local + count: 1 + mode: node local-dns: _inherit: "local" node-groups: @@ -57,6 +62,11 @@ clusters: config: local-light count: 2 mode: node + ultralight: + bee-config: bee-local-ultralight + config: local + count: 1 + mode: node local-gc: _inherit: "local" node-groups: @@ -184,6 +194,12 @@ bee-configs: bee-local-gc: _inherit: "bee-local" cache-capacity: 10 + bee-local-ultralight: + _inherit: "bee-local" + blockchain-rpc-endpoint: "" + full-node: false + p2p-addr: "" + swap-enable: false bootnode-local-gc: _inherit: "bee-local" cache-capacity: 10 @@ -414,3 +430,5 @@ checks: ci-autotls: timeout: 15m type: autotls + options: + ultralight-group: ultralight diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index 4fe4faa8..03cf7d50 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -13,14 +13,16 @@ import ( ) type Options struct { - WSSGroup string - ConnectTimeout time.Duration + WSSGroup string + UltraLightGroup string + ConnectTimeout time.Duration } func NewDefaultOptions() Options { return Options{ - WSSGroup: "wss", - ConnectTimeout: 30 * time.Second, + WSSGroup: "wss", + UltraLightGroup: "ultralight", + ConnectTimeout: 30 * time.Second, } } @@ -65,10 +67,16 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("WSS connectivity test: %w", err) } - if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { - return fmt.Errorf("certificate renewal test: %w", err) + if o.UltraLightGroup != "" { + if err := c.testUltraLightConnectivity(ctx, clients, wssNodes, o.UltraLightGroup, o.ConnectTimeout); err != nil { + return fmt.Errorf("ultra-light connectivity test: %w", err) + } } + // if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + // return fmt.Errorf("certificate renewal test: %w", err) + // } + c.logger.Info("AutoTLS check completed successfully") return nil } @@ -154,6 +162,26 @@ func (c *Check) testWSSConnectivity(ctx context.Context, clients map[string]*bee return nil } +func (c *Check) testUltraLightConnectivity(ctx context.Context, clients map[string]*bee.Client, wssNodes map[string][]string, ultraLightGroup string, timeout time.Duration) error { + ultralightClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{ultraLightGroup}) + if len(ultralightClients) == 0 { + c.logger.Warningf("no nodes found in ultra-light group %q, skipping ultra-light connectivity test", ultraLightGroup) + return nil + } + + c.logger.Infof("found %d nodes in ultra-light group %q", len(ultralightClients), ultraLightGroup) + + for _, client := range ultralightClients { + nodeName := client.Name() + c.logger.Infof("testing ultra-light to WSS: %s (no listen addr) to WSS nodes", nodeName) + if err := c.testConnectivity(ctx, client, nodeName, clients, wssNodes, timeout); err != nil { + return fmt.Errorf("ultra-light %s to WSS test: %w", nodeName, err) + } + } + + return nil +} + func (c *Check) testConnectivity(ctx context.Context, sourceClient *bee.Client, sourceName string, clients map[string]*bee.Client, wssNodes map[string][]string, timeout time.Duration) error { for targetName, underlays := range wssNodes { if targetName == sourceName { diff --git a/pkg/config/check.go b/pkg/config/check.go index 256057a2..5543d172 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -87,8 +87,9 @@ var Checks = map[string]CheckType{ NewAction: autotls.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - WSSGroup *string `yaml:"wss-group"` - ConnectTimeout *time.Duration `yaml:"connect-timeout"` + WSSGroup *string `yaml:"wss-group"` + UltraLightGroup *string `yaml:"ultralight-group"` + ConnectTimeout *time.Duration `yaml:"connect-timeout"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From 04e0444cb34d464777b076a8eef16ec94407a1ca Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 5 Feb 2026 10:59:56 +0100 Subject: [PATCH 13/14] feat(config): update local.yaml for ultralight configuration and enhance autotls checks --- config/local.yaml | 16 ++++++++++++---- pkg/check/autotls/autotls.go | 13 +++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index d3c6d10a..542e0db4 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -32,7 +32,7 @@ clusters: mode: node ultralight: bee-config: bee-local-ultralight - config: local + config: local-ultralight count: 1 mode: node local-dns: @@ -64,8 +64,8 @@ clusters: mode: node ultralight: bee-config: bee-local-ultralight - config: local - count: 1 + config: local-ultralight + count: 2 mode: node local-gc: _inherit: "local" @@ -124,6 +124,14 @@ node-groups: _inherit: "local" local-light: _inherit: "local" + local-ultralight: + _inherit: "local" + labels: + app.kubernetes.io/component: "node" + app.kubernetes.io/name: "bee" + app.kubernetes.io/part-of: "bee" + app.kubernetes.io/version: "latest" + beekeeper.ethswarm.org/node-funder: "false" # bee-configs defines Bee configuration that can be assigned to node-groups bee-configs: @@ -198,7 +206,6 @@ bee-configs: _inherit: "bee-local" blockchain-rpc-endpoint: "" full-node: false - p2p-addr: "" swap-enable: false bootnode-local-gc: _inherit: "bee-local" @@ -432,3 +439,4 @@ checks: type: autotls options: ultralight-group: ultralight + wss-group: wss diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index 03cf7d50..8b0deb0e 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -50,7 +50,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any if err != nil { return fmt.Errorf("get node clients: %w", err) } - + time.Sleep(5 * time.Second) wssClients := orchestration.ClientMap(clients).FilterByNodeGroups([]string{o.WSSGroup}) if len(wssClients) == 0 { return fmt.Errorf("no nodes found in WSS group %q", o.WSSGroup) @@ -58,7 +58,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any c.logger.Infof("found %d nodes in WSS group %q", len(wssClients), o.WSSGroup) - wssNodes, err := c.verifyWSSUnderlays(ctx, wssClients) + wssNodes, err := c.verifyWSSUnderlays(ctx, wssClients, o.UltraLightGroup) if err != nil { return fmt.Errorf("verify WSS underlays: %w", err) } @@ -81,16 +81,21 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return nil } -func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration.ClientList) (map[string][]string, error) { +func (c *Check) verifyWSSUnderlays(ctx context.Context, wssClients orchestration.ClientList, excludeNodeGroup string) (map[string][]string, error) { wssNodes := make(map[string][]string) for _, client := range wssClients { + if excludeNodeGroup != "" && client.NodeGroup() == excludeNodeGroup { + c.logger.Debugf("skipping %s (node group %s has no WSS underlays)", client.Name(), excludeNodeGroup) + continue + } + nodeName := client.Name() addresses, err := client.Addresses(ctx) if err != nil { return nil, fmt.Errorf("%s: get addresses: %w", nodeName, err) } - + time.Sleep(2 * time.Second) wssUnderlays := filterWSSUnderlays(addresses.Underlay) if len(wssUnderlays) == 0 { return nil, fmt.Errorf("node %s in WSS group has no WSS underlay addresses", nodeName) From 2959ac0d2016856cd58790dc4f74da7c73f997e5 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 5 Feb 2026 12:41:12 +0100 Subject: [PATCH 14/14] fix(autotls): re-enable certificate renewal test --- pkg/check/autotls/autotls.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/check/autotls/autotls.go b/pkg/check/autotls/autotls.go index 8b0deb0e..f808141a 100644 --- a/pkg/check/autotls/autotls.go +++ b/pkg/check/autotls/autotls.go @@ -73,9 +73,9 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any } } - // if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { - // return fmt.Errorf("certificate renewal test: %w", err) - // } + if err := c.testCertificateRenewal(ctx, clients, wssNodes, o.ConnectTimeout); err != nil { + return fmt.Errorf("certificate renewal test: %w", err) + } c.logger.Info("AutoTLS check completed successfully") return nil