Skip to content

Commit eeaa854

Browse files
authored
Keep node IPs when netlink dumps are interrupted
1 parent 591e36a commit eeaa854

6 files changed

Lines changed: 258 additions & 9 deletions

File tree

‎core/node/core_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func newTestMocks(t *testing.T) *testMocks {
112112
// don't fail on host discovery calls unrelated to the scenario under test.
113113
func expectNodeInfoDiscovery(mocks *testMocks) {
114114
mocks.OsUtils.EXPECT().GetHostSystemInfo(gomock.Any()).Return(&models.HostSystem{}, nil).AnyTimes()
115-
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return(nil, nil).AnyTimes()
115+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{"192.0.2.10"}, nil).AnyTimes()
116116
mocks.OsUtils.EXPECT().NFSActiveOnHost(gomock.Any()).Return(false, nil).AnyTimes()
117117
mocks.ISCSI.EXPECT().ISCSIActiveOnHost(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes()
118118
mocks.NVMe.EXPECT().NVMeActiveOnHost(gomock.Any()).Return(false, nil).AnyTimes()

‎core/node/register.go‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package node
66

77
import (
88
"context"
9+
"runtime"
910
"time"
1011

1112
"github.com/cenkalti/backoff/v4"
@@ -26,8 +27,20 @@ var (
2627
registerBackoffMultiplier = 2.0
2728
registerBackoffMaxInterval = 120 * time.Second
2829
registerBackoffRandomizationFactor = 0.1
30+
31+
ipDiscoveryAttempts = defaultIPDiscoveryAttempts()
32+
ipDiscoveryRetryInterval = 2 * time.Second
2933
)
3034

35+
// defaultIPDiscoveryAttempts returns 1 (no retries) where IP discovery is not implemented and always
36+
// fails, such as Windows.
37+
func defaultIPDiscoveryAttempts() int {
38+
if runtime.GOOS == "linux" {
39+
return 3
40+
}
41+
return 1
42+
}
43+
3144
// ChapClient is the subset of the legacy Trident controller REST surface the node core still
3245
// needs directly: CHAP credential lookup has not been migrated onto the tridentcontroller.Client
3346
// abstraction. Any type satisfying this (e.g. controller_api.TridentController) can be combined
@@ -89,7 +102,7 @@ func (c *Core) buildNodeInfo(ctx context.Context) *models.Node {
89102
Logc(ctx).WithField("IQN", iscsiWWN).Info("Discovered iSCSI initiator name.")
90103
}
91104

92-
ips, err := c.osutils.GetIPAddresses(ctx)
105+
ips, err := c.discoverIPAddresses(ctx)
93106
if err != nil {
94107
Logc(ctx).WithField("error", err).Error("Could not get IP addresses.")
95108
} else if len(ips) == 0 {
@@ -169,6 +182,29 @@ func (c *Core) buildNodeInfo(ctx context.Context) *models.Node {
169182
}
170183
}
171184

185+
// discoverIPAddresses retries IP discovery a bounded number of times while it fails or finds no
186+
// usable addresses, which can happen while the host network is still being configured at boot.
187+
// The node's export policy rules are built from these addresses, and they are only discovered at
188+
// registration. After the last attempt, the result is returned as-is so registration never blocks
189+
// on a host that genuinely has no usable addresses.
190+
func (c *Core) discoverIPAddresses(ctx context.Context) ([]string, error) {
191+
ips, err := c.osutils.GetIPAddresses(ctx)
192+
for attempt := 1; attempt < ipDiscoveryAttempts && (err != nil || len(ips) == 0); attempt++ {
193+
Logc(ctx).WithFields(LogFields{
194+
"attempt": attempt,
195+
"error": err,
196+
}).Warn("No usable IP addresses discovered, will retry.")
197+
198+
select {
199+
case <-ctx.Done():
200+
return ips, err
201+
case <-time.After(ipDiscoveryRetryInterval):
202+
}
203+
ips, err = c.osutils.GetIPAddresses(ctx)
204+
}
205+
return ips, err
206+
}
207+
172208
// register builds this node's identity and registers it with the Trident controller, retrying
173209
// with exponential backoff until success (or until timeout elapses; zero retries forever). On
174210
// success it applies the controller's log settings, if a LogSetter was configured.

‎core/node/register_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,112 @@ package node
44

55
import (
66
"context"
7+
"runtime"
78
"testing"
9+
"time"
810

911
"github.com/stretchr/testify/assert"
1012
"go.uber.org/mock/gomock"
1113

1214
mockNode "github.com/netapp/trident/mocks/mock_core/mock_node"
1315
mockTridentController "github.com/netapp/trident/mocks/mock_frontend/mock_csi/mock_tridentcontroller"
16+
"github.com/netapp/trident/utils/errors"
1417
"github.com/netapp/trident/utils/models"
1518
)
1619

20+
// setIPDiscoveryRetries makes IP discovery retry quickly, independent of the platform default.
21+
func setIPDiscoveryRetries(t *testing.T) {
22+
originalAttempts, originalInterval := ipDiscoveryAttempts, ipDiscoveryRetryInterval
23+
ipDiscoveryAttempts, ipDiscoveryRetryInterval = 3, time.Millisecond
24+
t.Cleanup(func() { ipDiscoveryAttempts, ipDiscoveryRetryInterval = originalAttempts, originalInterval })
25+
}
26+
27+
func TestDefaultIPDiscoveryAttempts(t *testing.T) {
28+
if runtime.GOOS == "linux" {
29+
assert.Equal(t, 3, defaultIPDiscoveryAttempts())
30+
} else {
31+
assert.Equal(t, 1, defaultIPDiscoveryAttempts())
32+
}
33+
}
34+
35+
func TestDiscoverIPAddresses_NotRetriedWhenSingleAttempt(t *testing.T) {
36+
setIPDiscoveryRetries(t)
37+
ipDiscoveryAttempts = 1
38+
core, mocks := newTestCore(t)
39+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).
40+
Return(nil, errors.UnsupportedError("getIPAddresses is not supported for windows")).Times(1)
41+
42+
ips, err := core.discoverIPAddresses(context.Background())
43+
44+
assert.Error(t, err)
45+
assert.Nil(t, ips)
46+
}
47+
48+
func TestDiscoverIPAddresses_FirstAttemptSucceeds(t *testing.T) {
49+
core, mocks := newTestCore(t)
50+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{"10.58.43.100"}, nil).Times(1)
51+
52+
ips, err := core.discoverIPAddresses(context.Background())
53+
54+
assert.NoError(t, err)
55+
assert.Equal(t, []string{"10.58.43.100"}, ips)
56+
}
57+
58+
func TestDiscoverIPAddresses_RetriesUntilAddressesFound(t *testing.T) {
59+
setIPDiscoveryRetries(t)
60+
core, mocks := newTestCore(t)
61+
gomock.InOrder(
62+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return(nil, errors.New("interrupted")),
63+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{}, nil),
64+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{"10.58.43.100", "10.58.79.100"}, nil),
65+
)
66+
67+
ips, err := core.discoverIPAddresses(context.Background())
68+
69+
assert.NoError(t, err)
70+
assert.Equal(t, []string{"10.58.43.100", "10.58.79.100"}, ips)
71+
}
72+
73+
func TestDiscoverIPAddresses_GivesUpAfterBoundedAttempts(t *testing.T) {
74+
setIPDiscoveryRetries(t)
75+
core, mocks := newTestCore(t)
76+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{}, nil).Times(ipDiscoveryAttempts)
77+
78+
ips, err := core.discoverIPAddresses(context.Background())
79+
80+
assert.NoError(t, err)
81+
assert.Empty(t, ips)
82+
}
83+
84+
func TestDiscoverIPAddresses_StopsWhenContextDone(t *testing.T) {
85+
core, mocks := newTestCore(t)
86+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return(nil, errors.New("interrupted")).Times(1)
87+
ctx, cancel := context.WithCancel(context.Background())
88+
cancel()
89+
90+
ips, err := core.discoverIPAddresses(ctx)
91+
92+
assert.Error(t, err)
93+
assert.Nil(t, ips)
94+
}
95+
96+
func TestBuildNodeInfo_UsesRediscoveredIPAddresses(t *testing.T) {
97+
setIPDiscoveryRetries(t)
98+
core, mocks := newTestCore(t)
99+
mocks.OsUtils.EXPECT().GetHostSystemInfo(gomock.Any()).Return(&models.HostSystem{}, nil).AnyTimes()
100+
mocks.OsUtils.EXPECT().NFSActiveOnHost(gomock.Any()).Return(true, nil).AnyTimes()
101+
mocks.ISCSI.EXPECT().ISCSIActiveOnHost(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes()
102+
mocks.NVMe.EXPECT().NVMeActiveOnHost(gomock.Any()).Return(false, nil).AnyTimes()
103+
gomock.InOrder(
104+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return(nil, errors.New("interrupted")),
105+
mocks.OsUtils.EXPECT().GetIPAddresses(gomock.Any()).Return([]string{"10.58.43.100"}, nil),
106+
)
107+
108+
node := core.buildNodeInfo(context.Background())
109+
110+
assert.Equal(t, []string{"10.58.43.100"}, node.IPs)
111+
}
112+
17113
// TestNewController_UpdateVolumeReachable locks in the wiring that lets UpdateVolume flow through
18114
// NewController for free: tridentcontroller.Client is embedded in controllerFromClients, so adding
19115
// UpdateVolume to that interface required no change to NewController or controllerFromClients

‎utils/osutils/netlink_linux.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 NetApp, Inc. All Rights Reserved.
1+
// Copyright 2026 NetApp, Inc. All Rights Reserved.
22

33
package osutils
44

@@ -8,6 +8,8 @@ import "github.com/vishvananda/netlink"
88

99
var netLink NetLink = NewNetLinkClient()
1010

11+
// NetLink dumps may return netlink.ErrDumpInterrupted together with the data read, when the table
12+
// changed during the dump. Callers decide whether to use that data.
1113
type NetLink interface {
1214
LinkList() ([]netlink.Link, error)
1315
AddrList(link netlink.Link, family int) ([]netlink.Addr, error)
@@ -29,8 +31,15 @@ func (n *NetLinkClient) AddrList(link netlink.Link, family int) ([]netlink.Addr,
2931
return netlink.AddrList(link, family)
3032
}
3133

34+
// RouteListFiltered uses netlink.RouteListFilteredIter because netlink.RouteListFiltered discards the
35+
// routes read when the dump is interrupted.
3236
func (n *NetLinkClient) RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) {
33-
return netlink.RouteListFiltered(family, filter, filterMask)
37+
var routes []netlink.Route
38+
err := netlink.RouteListFilteredIter(family, filter, filterMask, func(route netlink.Route) bool {
39+
routes = append(routes, route)
40+
return true
41+
})
42+
return routes, err
3443
}
3544

3645
func (n *NetLinkClient) LinkByIndex(index int) (netlink.Link, error) {

‎utils/osutils/osutils_linux.go‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,11 @@ func (o *OSUtils) getIPAddressesExceptingDummyInterfaces(ctx context.Context) ([
131131

132132
allLinks, err := netLink.LinkList()
133133
if err != nil {
134-
Logc(ctx).Error(err)
135-
return nil, err
134+
if !errors.Is(err, netlink.ErrDumpInterrupted) {
135+
Logc(ctx).Error(err)
136+
return nil, err
137+
}
138+
Logc(ctx).WithError(err).Warn("Interface list may be incomplete; using the interfaces returned.")
136139
}
137140

138141
links := make([]netlink.Link, 0)
@@ -161,7 +164,12 @@ func (o *OSUtils) getIPAddressesExceptingNondefaultRoutes(ctx context.Context) (
161164
var routes []netlink.Route
162165

163166
getRoutesByDestination := func(family int, dst *net.IPNet) ([]netlink.Route, error) {
164-
return netLink.RouteListFiltered(family, &netlink.Route{Dst: dst}, netlink.RT_FILTER_DST)
167+
routes, err := netLink.RouteListFiltered(family, &netlink.Route{Dst: dst}, netlink.RT_FILTER_DST)
168+
if errors.Is(err, netlink.ErrDumpInterrupted) {
169+
Logc(ctx).WithError(err).Warn("Route list may be incomplete; using the routes returned.")
170+
return routes, nil
171+
}
172+
return routes, err
165173
}
166174

167175
ipv4Routes, err := getRoutesByDestination(netlink.FAMILY_ALL, &net.IPNet{
@@ -214,8 +222,12 @@ func (o *OSUtils) getUsableAddressesFromLinks(ctx context.Context, links []netli
214222

215223
linkAddrs, err := netLink.AddrList(link, netlink.FAMILY_ALL)
216224
if err != nil {
217-
Log().WithFields(logFields).Errorf("Could not get addresses for interface; %v", err)
218-
continue
225+
if !errors.Is(err, netlink.ErrDumpInterrupted) {
226+
Log().WithFields(logFields).Errorf("Could not get addresses for interface; %v", err)
227+
continue
228+
}
229+
Log().WithFields(logFields).WithError(err).Warn(
230+
"Address list may be incomplete; using the addresses returned.")
219231
}
220232

221233
for _, linkAddr := range linkAddrs {

‎utils/osutils/osutils_linux_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,79 @@ func TestGetIPAddressesExceptingDummyInterfaces_DummyType(t *testing.T) {
102102
assert.NoError(t, err)
103103
}
104104

105+
func TestGetIPAddressesExceptingDummyInterfaces_DumpInterrupted(t *testing.T) {
106+
link := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "ens192"}}
107+
addrs := []netlink.Addr{{IPNet: &net.IPNet{IP: net.IPv4(10, 58, 43, 100), Mask: net.CIDRMask(24, 32)}}}
108+
109+
defer func(originalNetlink NetLink) {
110+
netLink = originalNetlink
111+
}(netLink)
112+
113+
t.Run("interfaces returned are used", func(t *testing.T) {
114+
mockNetLink := mock_osutils.NewMockNetLink(gomock.NewController(t))
115+
mockNetLink.EXPECT().LinkList().Return([]netlink.Link{link}, netlink.ErrDumpInterrupted)
116+
mockNetLink.EXPECT().AddrList(link, netlink.FAMILY_ALL).Return(addrs, nil)
117+
netLink = mockNetLink
118+
119+
result, err := NewDetailed(nil, afero.NewMemMapFs()).getIPAddressesExceptingDummyInterfaces(context.TODO())
120+
121+
assert.NoError(t, err)
122+
assert.Len(t, result, 1)
123+
assert.Equal(t, "10.58.43.100/24", result[0].String())
124+
})
125+
126+
t.Run("no interfaces returned is not an error", func(t *testing.T) {
127+
mockNetLink := mock_osutils.NewMockNetLink(gomock.NewController(t))
128+
mockNetLink.EXPECT().LinkList().Return(nil, netlink.ErrDumpInterrupted)
129+
netLink = mockNetLink
130+
131+
result, err := NewDetailed(nil, afero.NewMemMapFs()).getIPAddressesExceptingDummyInterfaces(context.TODO())
132+
133+
assert.NoError(t, err)
134+
assert.Empty(t, result)
135+
})
136+
}
137+
138+
func TestGetIPAddressesExceptingNondefaultRoutes_DumpInterrupted(t *testing.T) {
139+
link := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "ens192", Index: 2}}
140+
addrs := []netlink.Addr{{IPNet: &net.IPNet{IP: net.IPv4(10, 58, 43, 100), Mask: net.CIDRMask(24, 32)}}}
141+
142+
mockNetLink := mock_osutils.NewMockNetLink(gomock.NewController(t))
143+
gomock.InOrder(
144+
mockNetLink.EXPECT().RouteListFiltered(netlink.FAMILY_ALL, gomock.Any(), uint64(netlink.RT_FILTER_DST)).
145+
Return([]netlink.Route{{LinkIndex: 2}}, netlink.ErrDumpInterrupted),
146+
mockNetLink.EXPECT().RouteListFiltered(netlink.FAMILY_ALL, gomock.Any(), uint64(netlink.RT_FILTER_DST)).
147+
Return(nil, nil),
148+
)
149+
mockNetLink.EXPECT().LinkByIndex(2).Return(link, nil)
150+
mockNetLink.EXPECT().AddrList(link, netlink.FAMILY_ALL).Return(addrs, nil)
151+
defer func(originalNetlink NetLink) {
152+
netLink = originalNetlink
153+
}(netLink)
154+
netLink = mockNetLink
155+
156+
result, err := NewDetailed(nil, afero.NewMemMapFs()).getIPAddressesExceptingNondefaultRoutes(context.TODO())
157+
158+
assert.NoError(t, err)
159+
assert.Len(t, result, 1)
160+
assert.Equal(t, "10.58.43.100/24", result[0].String())
161+
}
162+
163+
func TestGetIPAddressesExceptingNondefaultRoutes_Error(t *testing.T) {
164+
mockNetLink := mock_osutils.NewMockNetLink(gomock.NewController(t))
165+
mockNetLink.EXPECT().RouteListFiltered(netlink.FAMILY_ALL, gomock.Any(), uint64(netlink.RT_FILTER_DST)).
166+
Return(nil, errors.New("error"))
167+
defer func(originalNetlink NetLink) {
168+
netLink = originalNetlink
169+
}(netLink)
170+
netLink = mockNetLink
171+
172+
result, err := NewDetailed(nil, afero.NewMemMapFs()).getIPAddressesExceptingNondefaultRoutes(context.TODO())
173+
174+
assert.Nil(t, result)
175+
assert.Error(t, err)
176+
}
177+
105178
func TestGetIPAddressesExceptingNondefaultRoutes(t *testing.T) {
106179
osUtils := New()
107180
addrs, err := osUtils.getIPAddressesExceptingNondefaultRoutes(context.TODO())
@@ -245,3 +318,26 @@ func TestGetUsableAddressesFromLinks(t *testing.T) {
245318
assert.Equal(t, 1, len(addresses))
246319
assert.Equal(t, "10.10.10.10/24", addresses[0].String())
247320
}
321+
322+
func TestGetUsableAddressesFromLinks_DumpInterrupted(t *testing.T) {
323+
ens192 := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "ens192", Index: 2}}
324+
ens224 := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "ens224", Index: 3}}
325+
326+
mockNetLink := mock_osutils.NewMockNetLink(gomock.NewController(t))
327+
mockNetLink.EXPECT().AddrList(ens192, netlink.FAMILY_ALL).Return([]netlink.Addr{
328+
{IPNet: &net.IPNet{IP: net.IPv4(10, 58, 43, 100), Mask: net.CIDRMask(24, 32)}},
329+
}, netlink.ErrDumpInterrupted)
330+
mockNetLink.EXPECT().AddrList(ens224, netlink.FAMILY_ALL).Return([]netlink.Addr{
331+
{IPNet: &net.IPNet{IP: net.IPv4(10, 58, 79, 100), Mask: net.CIDRMask(24, 32)}},
332+
}, errors.New("error"))
333+
defer func(originalNetlink NetLink) {
334+
netLink = originalNetlink
335+
}(netLink)
336+
netLink = mockNetLink
337+
338+
osUtils := NewDetailed(exec.NewCommand(), afero.NewMemMapFs())
339+
addresses := osUtils.getUsableAddressesFromLinks(context.TODO(), []netlink.Link{ens192, ens224})
340+
341+
assert.Len(t, addresses, 1)
342+
assert.Equal(t, "10.58.43.100/24", addresses[0].String())
343+
}

0 commit comments

Comments
 (0)