From 412c1187caa482d41862c9030d67ab849a49c272 Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Wed, 23 May 2018 11:32:22 -0400 Subject: [PATCH 1/7] Possible fix for issue #36 for mount issue of POD with managed plugin --- common/docker/dockervol/dockervol.go | 22 ++++++++++++++++++++++ common/linux/bmount.go | 7 +++++++ 2 files changed, 29 insertions(+) diff --git a/common/docker/dockervol/dockervol.go b/common/docker/dockervol/dockervol.go index 1985f7c..b645cb1 100644 --- a/common/docker/dockervol/dockervol.go +++ b/common/docker/dockervol/dockervol.go @@ -413,3 +413,25 @@ func getV2PluginSocket(name, dockerSocket string) (string, error) { return "", fmt.Errorf("unable to find V2 plugin named %s", name) } +// GetV2PluginID - name is the name of the plugin and this function returns the +// plugin ID of the managed plugin +func GetV2PluginID(name, dockerSocket string) (string, error) { + c := dockerlt.NewDockerClient(dockerSocket) + plugins, err := c.PluginsGet() + + if err != nil { + return "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) + } + + for _, plugin := range plugins { + if strings.Compare(name, plugin.Name) == 0 || strings.Compare(fmt.Sprintf("%s:latest", name), plugin.Name) == 0 { + if !plugin.Enabled { + return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) + } + + return plugin.ID, nil + } + } + + return "", fmt.Errorf("unable to find V2 plugin ID %s", name) +} diff --git a/common/linux/bmount.go b/common/linux/bmount.go index 311155b..86b578d 100644 --- a/common/linux/bmount.go +++ b/common/linux/bmount.go @@ -18,6 +18,7 @@ package linux import ( "github.com/hpe-storage/dory/common/util" + "github.com/hpe-storage/dory/common/docker/dockervol" "strings" ) @@ -49,6 +50,12 @@ func BindMount(path, mountPoint string, rbind bool) error { out, rc, err := util.ExecCommandOutput(mountCommand, args) if err != nil { util.LogError.Printf("BindMount failed with %d. It was called with %s %s %v. Output=%v.", rc, path, mountPoint, rbind, out) + + pluginID, _ := dockervol.GetV2PluginID("hpe:latest", "") + pathForManagedPlugin := "/var/lib/docker/plugins/" + pluginID + "/rootfs"+ path + + args := []string{flag, pathForManagedPlugin, mountPoint} + _, _, err := util.ExecCommandOutput(mountCommand, args) return err } From a6fde601321c3b31d1ebbedaca5ed368180a54e7 Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Thu, 24 May 2018 05:11:21 -0400 Subject: [PATCH 2/7] Remove circular dependency on bmount.go --- common/k8s/flexvol/flexvol.go | 12 +++++++++++- common/linux/bmount.go | 6 ------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/common/k8s/flexvol/flexvol.go b/common/k8s/flexvol/flexvol.go index c540332..cea0f3a 100644 --- a/common/k8s/flexvol/flexvol.go +++ b/common/k8s/flexvol/flexvol.go @@ -228,7 +228,17 @@ func Mount(args []string) (string, error) { //Bind mount the docker path to the flexvol path err = linux.BindMount(path, args[0], false) if err != nil { - return "", err + pluginID, _ := dockervol.GetV2PluginID("hpe:latest", "") + pathForManagedPlugin := "/var/lib/docker/plugins/" + pluginID + "/rootfs"+ path + + //args := []string{flag, pathForManagedPlugin, mountPoint} + //_, _, err := util.ExecCommandOutput(mountCommand, args) + err = linux.BindMount(pathForManagedPlugin, args[0], false) + + if err != nil { + return "", err + } + path = pathForManagedPlugin } // Set selinux context if configured diff --git a/common/linux/bmount.go b/common/linux/bmount.go index 86b578d..e912708 100644 --- a/common/linux/bmount.go +++ b/common/linux/bmount.go @@ -18,7 +18,6 @@ package linux import ( "github.com/hpe-storage/dory/common/util" - "github.com/hpe-storage/dory/common/docker/dockervol" "strings" ) @@ -51,11 +50,6 @@ func BindMount(path, mountPoint string, rbind bool) error { if err != nil { util.LogError.Printf("BindMount failed with %d. It was called with %s %s %v. Output=%v.", rc, path, mountPoint, rbind, out) - pluginID, _ := dockervol.GetV2PluginID("hpe:latest", "") - pathForManagedPlugin := "/var/lib/docker/plugins/" + pluginID + "/rootfs"+ path - - args := []string{flag, pathForManagedPlugin, mountPoint} - _, _, err := util.ExecCommandOutput(mountCommand, args) return err } From a25eafab675a3bdc9afa4eada1f21921b0422b55 Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Thu, 24 May 2018 14:47:55 -0400 Subject: [PATCH 3/7] Optimized getV2PluginSocket to return socketPath, pluginID --- cmd/dory/dory.go | 1 + common/docker/dockervol/dockervol.go | 36 +++++++--------------------- common/k8s/flexvol/flexvol.go | 7 +++--- common/linux/bmount.go | 3 +-- 4 files changed, 13 insertions(+), 34 deletions(-) diff --git a/cmd/dory/dory.go b/cmd/dory/dory.go index b2a5193..12faa98 100644 --- a/cmd/dory/dory.go +++ b/cmd/dory/dory.go @@ -87,6 +87,7 @@ func main() { ListOfStorageResourceOptions: listOfStorageResourceOptions, FactorForConversion: factorForConversion, SupportsCapabilities: supportsCapabilities, + ManagedPluginID: "", } err := flexvol.Config(dockervolOptions) var mess string diff --git a/common/docker/dockervol/dockervol.go b/common/docker/dockervol/dockervol.go index b645cb1..911888e 100644 --- a/common/docker/dockervol/dockervol.go +++ b/common/docker/dockervol/dockervol.go @@ -48,6 +48,7 @@ const ( NotFound = "Unable to find" defaultSocketPath = "/run/docker/plugins/nimble.sock" + //managedPluginID = "" maxTries = 3 ) @@ -61,6 +62,7 @@ type Options struct { ListOfStorageResourceOptions []string FactorForConversion int SupportsCapabilities bool + ManagedPluginID string } //DockerVolumePlugin is the client to a specific docker volume plugin @@ -143,7 +145,7 @@ func NewDockerVolumePlugin(options *Options) (*DockerVolumePlugin, error) { var err error if !strings.HasPrefix(options.SocketPath, "/") { // this is a v2 plugin, so we need to find its socket file - options.SocketPath, err = getV2PluginSocket(options.SocketPath, "") + options.SocketPath, options.ManagedPluginID, err = getV2PluginSocket(options.SocketPath, "") } if err != nil { return nil, err @@ -394,44 +396,22 @@ func driverErrorCheck(e Errorer) error { } // name is the name of the docker volume plugin. dockerSocket is the full path to the docker socket. The default is used if an empty string is passed. -func getV2PluginSocket(name, dockerSocket string) (string, error) { +func getV2PluginSocket(name, dockerSocket string) (string, string, error) { c := dockerlt.NewDockerClient(dockerSocket) plugins, err := c.PluginsGet() if err != nil { - return "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) + return "", "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) } for _, plugin := range plugins { if strings.Compare(name, plugin.Name) == 0 || strings.Compare(fmt.Sprintf("%s:latest", name), plugin.Name) == 0 { if !plugin.Enabled { - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) + return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), "", fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) } - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), nil + return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), plugin.ID, nil } } - return "", fmt.Errorf("unable to find V2 plugin named %s", name) -} -// GetV2PluginID - name is the name of the plugin and this function returns the -// plugin ID of the managed plugin -func GetV2PluginID(name, dockerSocket string) (string, error) { - c := dockerlt.NewDockerClient(dockerSocket) - plugins, err := c.PluginsGet() - - if err != nil { - return "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) - } - - for _, plugin := range plugins { - if strings.Compare(name, plugin.Name) == 0 || strings.Compare(fmt.Sprintf("%s:latest", name), plugin.Name) == 0 { - if !plugin.Enabled { - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) - } - - return plugin.ID, nil - } - } - - return "", fmt.Errorf("unable to find V2 plugin ID %s", name) + return "", "", fmt.Errorf("unable to find V2 plugin named %s", name) } diff --git a/common/k8s/flexvol/flexvol.go b/common/k8s/flexvol/flexvol.go index cea0f3a..6b91835 100644 --- a/common/k8s/flexvol/flexvol.go +++ b/common/k8s/flexvol/flexvol.go @@ -60,6 +60,7 @@ const ( var ( //createVolumes indicate whether the driver should create missing volumes createVolumes = true + pluginID = "" dvp *dockervol.DockerVolumePlugin ) @@ -99,6 +100,7 @@ func (ar *AttachRequest) getBestName() string { func Config(options *dockervol.Options) (err error) { dvp, err = dockervol.NewDockerVolumePlugin(options) createVolumes = options.CreateVolumes + pluginID = options.ManagedPluginID return err } @@ -228,11 +230,8 @@ func Mount(args []string) (string, error) { //Bind mount the docker path to the flexvol path err = linux.BindMount(path, args[0], false) if err != nil { - pluginID, _ := dockervol.GetV2PluginID("hpe:latest", "") pathForManagedPlugin := "/var/lib/docker/plugins/" + pluginID + "/rootfs"+ path - - //args := []string{flag, pathForManagedPlugin, mountPoint} - //_, _, err := util.ExecCommandOutput(mountCommand, args) + util.LogDebug.Printf("pathForManagedPlugin: %s", pathForManagedPlugin) err = linux.BindMount(pathForManagedPlugin, args[0], false) if err != nil { diff --git a/common/linux/bmount.go b/common/linux/bmount.go index e912708..6f20daa 100644 --- a/common/linux/bmount.go +++ b/common/linux/bmount.go @@ -48,8 +48,7 @@ func BindMount(path, mountPoint string, rbind bool) error { args := []string{flag, path, mountPoint} out, rc, err := util.ExecCommandOutput(mountCommand, args) if err != nil { - util.LogError.Printf("BindMount failed with %d. It was called with %s %s %v. Output=%v.", rc, path, mountPoint, rbind, out) - + util.LogError.Printf("BindMount failed with %d. It was called with %s %s %v. Output=%v.", rc, path, mountPoint, rbind, out) return err } From d40aa23097d90908f4c272a8079223b7a556466e Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Thu, 24 May 2018 14:47:55 -0400 Subject: [PATCH 4/7] Optimized getV2PluginSocket to return socketPath, pluginID --- cmd/dory/dory.go | 1 + common/docker/dockervol/dockervol.go | 36 +++++++--------------------- common/k8s/flexvol/flexvol.go | 7 +++--- common/linux/bmount.go | 1 - 4 files changed, 12 insertions(+), 33 deletions(-) diff --git a/cmd/dory/dory.go b/cmd/dory/dory.go index b2a5193..12faa98 100644 --- a/cmd/dory/dory.go +++ b/cmd/dory/dory.go @@ -87,6 +87,7 @@ func main() { ListOfStorageResourceOptions: listOfStorageResourceOptions, FactorForConversion: factorForConversion, SupportsCapabilities: supportsCapabilities, + ManagedPluginID: "", } err := flexvol.Config(dockervolOptions) var mess string diff --git a/common/docker/dockervol/dockervol.go b/common/docker/dockervol/dockervol.go index b645cb1..911888e 100644 --- a/common/docker/dockervol/dockervol.go +++ b/common/docker/dockervol/dockervol.go @@ -48,6 +48,7 @@ const ( NotFound = "Unable to find" defaultSocketPath = "/run/docker/plugins/nimble.sock" + //managedPluginID = "" maxTries = 3 ) @@ -61,6 +62,7 @@ type Options struct { ListOfStorageResourceOptions []string FactorForConversion int SupportsCapabilities bool + ManagedPluginID string } //DockerVolumePlugin is the client to a specific docker volume plugin @@ -143,7 +145,7 @@ func NewDockerVolumePlugin(options *Options) (*DockerVolumePlugin, error) { var err error if !strings.HasPrefix(options.SocketPath, "/") { // this is a v2 plugin, so we need to find its socket file - options.SocketPath, err = getV2PluginSocket(options.SocketPath, "") + options.SocketPath, options.ManagedPluginID, err = getV2PluginSocket(options.SocketPath, "") } if err != nil { return nil, err @@ -394,44 +396,22 @@ func driverErrorCheck(e Errorer) error { } // name is the name of the docker volume plugin. dockerSocket is the full path to the docker socket. The default is used if an empty string is passed. -func getV2PluginSocket(name, dockerSocket string) (string, error) { +func getV2PluginSocket(name, dockerSocket string) (string, string, error) { c := dockerlt.NewDockerClient(dockerSocket) plugins, err := c.PluginsGet() if err != nil { - return "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) + return "", "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) } for _, plugin := range plugins { if strings.Compare(name, plugin.Name) == 0 || strings.Compare(fmt.Sprintf("%s:latest", name), plugin.Name) == 0 { if !plugin.Enabled { - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) + return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), "", fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) } - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), nil + return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), plugin.ID, nil } } - return "", fmt.Errorf("unable to find V2 plugin named %s", name) -} -// GetV2PluginID - name is the name of the plugin and this function returns the -// plugin ID of the managed plugin -func GetV2PluginID(name, dockerSocket string) (string, error) { - c := dockerlt.NewDockerClient(dockerSocket) - plugins, err := c.PluginsGet() - - if err != nil { - return "", fmt.Errorf("failed to get V2 plugins from docker. error=%s", err.Error()) - } - - for _, plugin := range plugins { - if strings.Compare(name, plugin.Name) == 0 || strings.Compare(fmt.Sprintf("%s:latest", name), plugin.Name) == 0 { - if !plugin.Enabled { - return fmt.Sprintf("/run/docker/plugins/%s/%s", plugin.ID, plugin.Config.Interface.Socket), fmt.Errorf("found Docker V2 Plugin named %s, but it is disabled", name) - } - - return plugin.ID, nil - } - } - - return "", fmt.Errorf("unable to find V2 plugin ID %s", name) + return "", "", fmt.Errorf("unable to find V2 plugin named %s", name) } diff --git a/common/k8s/flexvol/flexvol.go b/common/k8s/flexvol/flexvol.go index cea0f3a..6b91835 100644 --- a/common/k8s/flexvol/flexvol.go +++ b/common/k8s/flexvol/flexvol.go @@ -60,6 +60,7 @@ const ( var ( //createVolumes indicate whether the driver should create missing volumes createVolumes = true + pluginID = "" dvp *dockervol.DockerVolumePlugin ) @@ -99,6 +100,7 @@ func (ar *AttachRequest) getBestName() string { func Config(options *dockervol.Options) (err error) { dvp, err = dockervol.NewDockerVolumePlugin(options) createVolumes = options.CreateVolumes + pluginID = options.ManagedPluginID return err } @@ -228,11 +230,8 @@ func Mount(args []string) (string, error) { //Bind mount the docker path to the flexvol path err = linux.BindMount(path, args[0], false) if err != nil { - pluginID, _ := dockervol.GetV2PluginID("hpe:latest", "") pathForManagedPlugin := "/var/lib/docker/plugins/" + pluginID + "/rootfs"+ path - - //args := []string{flag, pathForManagedPlugin, mountPoint} - //_, _, err := util.ExecCommandOutput(mountCommand, args) + util.LogDebug.Printf("pathForManagedPlugin: %s", pathForManagedPlugin) err = linux.BindMount(pathForManagedPlugin, args[0], false) if err != nil { diff --git a/common/linux/bmount.go b/common/linux/bmount.go index e912708..311155b 100644 --- a/common/linux/bmount.go +++ b/common/linux/bmount.go @@ -49,7 +49,6 @@ func BindMount(path, mountPoint string, rbind bool) error { out, rc, err := util.ExecCommandOutput(mountCommand, args) if err != nil { util.LogError.Printf("BindMount failed with %d. It was called with %s %s %v. Output=%v.", rc, path, mountPoint, rbind, out) - return err } From caf556f4abd50773eadac3f65f50ce72d4a101e2 Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Thu, 24 May 2018 15:13:35 -0400 Subject: [PATCH 5/7] Removed commented line --- common/docker/dockervol/dockervol.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/docker/dockervol/dockervol.go b/common/docker/dockervol/dockervol.go index 911888e..cc31f12 100644 --- a/common/docker/dockervol/dockervol.go +++ b/common/docker/dockervol/dockervol.go @@ -48,7 +48,6 @@ const ( NotFound = "Unable to find" defaultSocketPath = "/run/docker/plugins/nimble.sock" - //managedPluginID = "" maxTries = 3 ) From d64b97132d54a73927a1c0280b6cf373ea7ca420 Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Wed, 6 Jun 2018 03:21:11 -0400 Subject: [PATCH 6/7] Unmount fix --- common/k8s/flexvol/flexvol.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/k8s/flexvol/flexvol.go b/common/k8s/flexvol/flexvol.go index 6b91835..f680b62 100644 --- a/common/k8s/flexvol/flexvol.go +++ b/common/k8s/flexvol/flexvol.go @@ -26,6 +26,7 @@ import ( "path/filepath" "regexp" "time" + "strings" ) const ( @@ -338,7 +339,8 @@ func getVolumeNameFromMountPath(k8sPath, dockerPath string) (string, error) { return "", err } for _, vol := range volumes.Volumes { - if vol.Mountpoint == dockerPath { + if (vol.Mountpoint == dockerPath || (strings.Contains(dockerPath, vol.Mountpoint) && strings.HasPrefix(dockerPath,"/var/lib/docker"))) { + util.LogDebug.Printf("dockerPath %s, volume mountpoint %s", dockerPath, vol.Mountpoint) return vol.Name, nil } } From 8b9f69486dce9142a292cfcccb3db41a16b3751a Mon Sep 17 00:00:00 2001 From: William Durairaj Date: Wed, 13 Jun 2018 10:52:43 -0400 Subject: [PATCH 7/7] Alternate approach for unmount in managed plugin usecase --- common/k8s/flexvol/flexvol.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/common/k8s/flexvol/flexvol.go b/common/k8s/flexvol/flexvol.go index f680b62..e35b684 100644 --- a/common/k8s/flexvol/flexvol.go +++ b/common/k8s/flexvol/flexvol.go @@ -339,8 +339,10 @@ func getVolumeNameFromMountPath(k8sPath, dockerPath string) (string, error) { return "", err } for _, vol := range volumes.Volumes { - if (vol.Mountpoint == dockerPath || (strings.Contains(dockerPath, vol.Mountpoint) && strings.HasPrefix(dockerPath,"/var/lib/docker"))) { - util.LogDebug.Printf("dockerPath %s, volume mountpoint %s", dockerPath, vol.Mountpoint) + + util.LogDebug.Printf("dockerPath %s, volume mountpoint %s", dockerPath, vol.Mountpoint) + if (vol.Mountpoint == dockerPath || (findStringAfterLastSlash(vol.Mountpoint) == findStringAfterLastSlash(dockerPath))){ + util.LogDebug.Printf(" returning docker volume name %s", vol.Name) return vol.Name, nil } } @@ -360,3 +362,13 @@ func findJSON(args []string, req *AttachRequest) (string, error) { } return "", err } +func findStringAfterLastSlash(s string) string { + + //s := "/var/lib/docker/plugins/a238188db964f8139af8d502a9b134b1f9522ccc27936ae5512b2f1b662f0aa5/rootfs/opt/hpe/data/hpedocker-dm-uuid-mpath-360002ac0000000000101331f00019d52" + + flds := strings.Split(s, "/") + arrayLength := len(flds) + fmt.Printf(" Length = %d, last substring %s" ,arrayLength, flds[arrayLength -1]) + return flds[arrayLength - 1] + +}