Skip to content

Commit c8cd2bc

Browse files
authored
Prevent NVMe subsystem disconnect while a concurrent attach is in flight
1 parent ddf51b3 commit c8cd2bc

13 files changed

Lines changed: 1037 additions & 254 deletions

‎core/node/attach.go‎

Lines changed: 72 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ func (c *Core) ensureAttachFCPVolume(
379379

380380
func (c *Core) attachNVMeVolume(
381381
ctx context.Context, volume string, publishInfo *models.VolumePublishInfo, secrets map[string]string,
382-
) error {
382+
) (err error) {
383383
Logc(ctx).Debug(">>>> attachNVMeVolume")
384384
defer Logc(ctx).Debug("<<<< attachNVMeVolume")
385385

@@ -395,10 +395,24 @@ func (c *Core) attachNVMeVolume(
395395
defer nvmeSelfHealingLock.RUnlock()
396396
nvmeNodeOperationWaitingCount.Add(-1)
397397

398-
err = c.nvme.AttachNVMeVolumeRetry(ctx, publishInfo, nvme.NVMeAttachTimeout)
398+
addedNamespace, err := c.registerNVMeSessionForAttach(ctx, publishInfo)
399399
if err != nil {
400400
return err
401401
}
402+
// Once the volume is formatted and mounted, the namespace is in use on this node. A later failure
403+
// leaves it published, so self-healing keeps its paths up and NodeUnstageVolume (which reads the
404+
// tracking info Attach always writes) tears it down. Disconnecting the subsystem here would pull the
405+
// device out from under the mount.
406+
mounted := false
407+
defer func() {
408+
if addedNamespace && err != nil && !mounted {
409+
c.removePublishedNVMeSessionAfterAttachError(ctx, publishInfo)
410+
}
411+
}()
412+
413+
if err = c.nvme.AttachNVMeVolumeRetry(ctx, publishInfo, nvme.NVMeAttachTimeout); err != nil {
414+
return err
415+
}
402416

403417
// Cryptsetup format if necessary and map to host
404418
luksFormatted, safeToFormat, err := c.nvme.EnsureCryptsetupFormattedAndMappedOnHost(
@@ -414,6 +428,7 @@ func (c *Core) attachNVMeVolume(
414428
); err != nil {
415429
return err
416430
}
431+
mounted = true
417432

418433
if convert.ToBool(publishInfo.LUKSEncryption) {
419434
if err = betweenAttachAndLUKSPassphrase.Inject(); err != nil {
@@ -428,12 +443,60 @@ func (c *Core) attachNVMeVolume(
428443
}
429444
}
430445

431-
lockContext := "nodeStageNVMeVolume.AddSession"
432-
if !attemptLock(ctx, lockContext, nvmeSelfHealingSessionLock, sharedLocksNodeLockTimeout) {
433-
locks.Unlock(ctx, lockContext, nvmeSelfHealingSessionLock)
434-
return errors.MaxWaitExceededError("request waited too long for the lock")
435-
}
436-
c.nvme.AddPublishedNVMeSession(&publishedNVMeSessions, publishInfo)
437-
locks.Unlock(ctx, lockContext, nvmeSelfHealingSessionLock)
438446
return nil
439447
}
448+
449+
// registerNVMeSessionForAttach makes attach ownership visible before connect work begins. It
450+
// reports whether this call published the namespace, so a failed attach removes only what it
451+
// added; the published session tracks which target IPs that namespace claimed.
452+
func (c *Core) registerNVMeSessionForAttach(
453+
ctx context.Context, publishInfo *models.VolumePublishInfo,
454+
) (bool, error) {
455+
lockContext := "registerNVMeSessionForAttach"
456+
subsystemLockID := nvmeSubsystemOperationLockID(publishInfo.NVMeSubsystemNQN)
457+
if !attemptLock(ctx, lockContext, subsystemLockID, sharedLocksNodeLockTimeout) {
458+
locks.Unlock(ctx, lockContext, subsystemLockID)
459+
return false, errors.MaxWaitExceededError("request waited too long for the lock")
460+
}
461+
defer locks.Unlock(ctx, lockContext, subsystemLockID)
462+
463+
sessionLockContext := "registerNVMeSessionForAttach.AddSession"
464+
if !attemptLock(ctx, sessionLockContext, nvmeSelfHealingSessionLock, sharedLocksNodeLockTimeout) {
465+
locks.Unlock(ctx, sessionLockContext, nvmeSelfHealingSessionLock)
466+
return false, errors.MaxWaitExceededError("request waited too long for the lock")
467+
}
468+
defer locks.Unlock(ctx, sessionLockContext, nvmeSelfHealingSessionLock)
469+
470+
sessionData := publishedNVMeSessions.Info[publishInfo.NVMeSubsystemNQN]
471+
addedNamespace := sessionData == nil || !sessionData.Namespaces[publishInfo.NVMeNamespaceUUID]
472+
c.nvme.AddPublishedNVMeSession(&publishedNVMeSessions, publishInfo)
473+
return addedNamespace, nil
474+
}
475+
476+
func (c *Core) removePublishedNVMeSessionAfterAttachError(
477+
ctx context.Context, publishInfo *models.VolumePublishInfo,
478+
) {
479+
subsystemLockContext := "removePublishedNVMeSessionAfterAttachError"
480+
subsystemLockID := nvmeSubsystemOperationLockID(publishInfo.NVMeSubsystemNQN)
481+
if !attemptLock(ctx, subsystemLockContext, subsystemLockID, sharedLocksNodeLockTimeout) {
482+
Logc(ctx).Warn("NVMe session cleanup waited longer than expected for the subsystem lock.")
483+
}
484+
485+
sessionLockContext := "removePublishedNVMeSessionAfterAttachError.RemoveSession"
486+
if !attemptLock(ctx, sessionLockContext, nvmeSelfHealingSessionLock, sharedLocksNodeLockTimeout) {
487+
Logc(ctx).Warn("NVMe session cleanup waited longer than expected for the session lock.")
488+
}
489+
490+
requiresDisconnectCheck := c.nvme.RemovePublishedNVMeSession(
491+
&publishedNVMeSessions, publishInfo.NVMeSubsystemNQN, publishInfo.NVMeNamespaceUUID,
492+
)
493+
locks.Unlock(ctx, sessionLockContext, nvmeSelfHealingSessionLock)
494+
locks.Unlock(ctx, subsystemLockContext, subsystemLockID)
495+
496+
if requiresDisconnectCheck {
497+
nvmeSubsys := c.nvme.NewNVMeSubsystem(ctx, publishInfo.NVMeSubsystemNQN)
498+
if err := c.disconnectNVMeSubsystemForAttachCleanup(ctx, nvmeSubsys, publishInfo); err != nil {
499+
Logc(ctx).WithError(err).Warn("Error during subsystem disconnect check after attach cleanup.")
500+
}
501+
}
502+
}

0 commit comments

Comments
 (0)