From 0909a77710c67ed48267cb0ceb9e2614e5a09600 Mon Sep 17 00:00:00 2001 From: Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:30:42 +0530 Subject: [PATCH] Disk Usage check correction (#1233) * correcting disk usage checks --- CHANGELOG.md | 1 + common/util.go | 31 +++++++++++++++- common/util_test.go | 43 +++++++++++++++++++++++ component/file_cache/cache_policy.go | 19 ++++++++-- component/file_cache/cache_policy_test.go | 11 +++++- 5 files changed, 100 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1975a23c7..6353e8da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - If `$PATH` is not populated correctly, find out correct path for `du` command. - Disable `kernel_cache` and `writeback_cache` when `direct_io` is set. - Fix FUSE CLI parameter parsing, where CLI overrides parameters provided in config file. +- [#1226](https://github.com/Azure/azure-storage-fuse/issues/1226) If max disk-cache size is not configured, check the available disk space to kick-in early eviction. - [#1230](https://github.com/Azure/azure-storage-fuse/issues/1230) Truncate file locally and then upload instead of downloading it again. ## 2.0.5 (2023-08-02) diff --git a/common/util.go b/common/util.go index 123d1e6d7..a4548ce2c 100644 --- a/common/util.go +++ b/common/util.go @@ -294,7 +294,7 @@ func NotifyMountToParent() error { var duPath []string = []string{"/usr/bin/du", "/usr/local/bin/du", "/usr/sbin/du", "/usr/local/sbin/du", "/sbin/du", "/bin/du"} var selectedDuPath string = "" -// getUsage: The current disk usage in MB +// GetUsage: The current disk usage in MB func GetUsage(path string) (float64, error) { var currSize float64 var out bytes.Buffer @@ -351,3 +351,32 @@ func GetUsage(path string) (float64, error) { return currSize, nil } + +var currentUID int = -1 + +// GetDiskUsageFromStatfs: Current disk usage of temp path +func GetDiskUsageFromStatfs(path string) (float64, float64, error) { + // We need to compute the disk usage percentage for the temp path + var stat syscall.Statfs_t + err := syscall.Statfs(path, &stat) + if err != nil { + return 0, 0, err + } + + if currentUID == -1 { + currentUID = os.Getuid() + } + + var availableSpace uint64 + if currentUID == 0 { + // Sudo has mounted + availableSpace = stat.Bfree * uint64(stat.Frsize) + } else { + // non Sudo has mounted + availableSpace = stat.Bavail * uint64(stat.Frsize) + } + + totalSpace := stat.Blocks * uint64(stat.Frsize) + usedSpace := float64(totalSpace - availableSpace) + return usedSpace, float64(usedSpace) / float64(totalSpace) * 100, nil +} diff --git a/common/util_test.go b/common/util_test.go index 580b8aa0d..ddcac17ab 100644 --- a/common/util_test.go +++ b/common/util_test.go @@ -186,3 +186,46 @@ func (suite *utilTestSuite) TestExpandPath() { expandedPath = ExpandPath(path) suite.assert.Equal(expandedPath, path) } + +func (suite *utilTestSuite) TestGetUSage() { + pwd, err := os.Getwd() + if err != nil { + return + } + + dirName := filepath.Join(pwd, "util_test") + err = os.Mkdir(dirName, 0777) + suite.assert.Nil(err) + + data := make([]byte, 1024*1024) + err = os.WriteFile(dirName+"/1.txt", data, 0777) + suite.assert.Nil(err) + + err = os.WriteFile(dirName+"/2.txt", data, 0777) + suite.assert.Nil(err) + + usage, err := GetUsage(dirName) + suite.assert.Nil(err) + suite.assert.GreaterOrEqual(int(usage), 2) + suite.assert.LessOrEqual(int(usage), 4) + + _ = os.RemoveAll(dirName) +} + +func (suite *utilTestSuite) TestGetDiskUsage() { + pwd, err := os.Getwd() + if err != nil { + return + } + + dirName := filepath.Join(pwd, "util_test", "a", "b", "c") + err = os.MkdirAll(dirName, 0777) + suite.assert.Nil(err) + + usage, usagePercent, err := GetDiskUsageFromStatfs(dirName) + suite.assert.Nil(err) + suite.assert.NotEqual(usage, 0) + suite.assert.NotEqual(usagePercent, 0) + suite.assert.NotEqual(usagePercent, 100) + _ = os.RemoveAll(filepath.Join(pwd, "util_test")) +} diff --git a/component/file_cache/cache_policy.go b/component/file_cache/cache_policy.go index 54cd95170..c3019ff40 100644 --- a/component/file_cache/cache_policy.go +++ b/component/file_cache/cache_policy.go @@ -75,12 +75,25 @@ type cachePolicy interface { // getUsagePercentage: The current cache usage as a percentage of the maxSize func getUsagePercentage(path string, maxSize float64) float64 { + var currSize float64 + var usagePercent float64 + var err error + if maxSize == 0 { - return 0 + currSize, usagePercent, err = common.GetDiskUsageFromStatfs(path) + if err != nil { + log.Err("cachePolicy::getUsagePercentage : failed to get disk usage for %s [%v]", path, err.Error) + } + } else { + // We need to compuate % usage of temp directory against configured limit + currSize, err = common.GetUsage(path) + if err != nil { + log.Err("cachePolicy::getUsagePercentage : failed to get directory usage for %s [%v]", path, err.Error) + } + + usagePercent = (currSize / float64(maxSize)) * 100 } - currSize, _ := common.GetUsage(path) - usagePercent := (currSize / float64(maxSize)) * 100 log.Debug("cachePolicy::getUsagePercentage : current cache usage : %f%%", usagePercent) fileCacheStatsCollector.UpdateStats(stats_manager.Replace, cacheUsage, fmt.Sprintf("%f MB", currSize)) diff --git a/component/file_cache/cache_policy_test.go b/component/file_cache/cache_policy_test.go index ce1db1815..9d1ce4aa5 100644 --- a/component/file_cache/cache_policy_test.go +++ b/component/file_cache/cache_policy_test.go @@ -71,17 +71,25 @@ func (suite *cachePolicyTestSuite) TestGetUsage() { f.Write(data) result, _ := common.GetUsage(cache_path) suite.assert.Equal(float64(1), math.Floor(result)) + f.Close() } func (suite *cachePolicyTestSuite) TestGetUsagePercentage() { defer suite.cleanupTest() - f, _ := os.Create(cache_path + "/test") data := make([]byte, 1024*1024) + + f, _ := os.Create(cache_path + "/test") f.Write(data) result := getUsagePercentage(cache_path, 4) // since the value might defer a little distro to distro suite.assert.GreaterOrEqual(result, float64(25)) suite.assert.LessOrEqual(result, float64(30)) + f.Close() + + result = getUsagePercentage("/", 0) + // since the value might defer a little distro to distro + suite.assert.GreaterOrEqual(result, float64(0)) + suite.assert.LessOrEqual(result, float64(90)) } func (suite *cachePolicyTestSuite) TestDeleteFile() { @@ -89,6 +97,7 @@ func (suite *cachePolicyTestSuite) TestDeleteFile() { f, _ := os.Create(cache_path + "/test") result := deleteFile(f.Name() + "not_exist") suite.assert.Equal(nil, result) + f.Close() } func TestCachePolicyTestSuite(t *testing.T) {