Disk Usage check correction (#1233)

* correcting disk usage checks

CHANGELOG.md

@@ -13,6 +13,7 @@
 - If `$PATH` is not populated correctly, find out correct path for `du` command.
 - Disable `kernel_cache` and `writeback_cache` when `direct_io` is set.
 - Fix FUSE CLI parameter parsing, where CLI overrides parameters provided in config file.
+- [#1226](https://github.com/Azure/azure-storage-fuse/issues/1226) If max disk-cache size is not configured, check the available disk space to kick-in early eviction.
 - [#1230](https://github.com/Azure/azure-storage-fuse/issues/1230) Truncate file locally and then upload instead of downloading it again.
 
 ## 2.0.5 (2023-08-02)

common/util.go

@@ -294,7 +294,7 @@ func NotifyMountToParent() error {
 var duPath []string = []string{"/usr/bin/du", "/usr/local/bin/du", "/usr/sbin/du", "/usr/local/sbin/du", "/sbin/du", "/bin/du"}
 var selectedDuPath string = ""
 
-// getUsage: The current disk usage in MB
+// GetUsage: The current disk usage in MB
 func GetUsage(path string) (float64, error) {
 	var currSize float64
 	var out bytes.Buffer
@@ -351,3 +351,32 @@ func GetUsage(path string) (float64, error) {
 
 	return currSize, nil
 }
+
+var currentUID int = -1
+
+// GetDiskUsageFromStatfs: Current disk usage of temp path
+func GetDiskUsageFromStatfs(path string) (float64, float64, error) {
+	// We need to compute the disk usage percentage for the temp path
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(path, &stat)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if currentUID == -1 {
+		currentUID = os.Getuid()
+	}
+
+	var availableSpace uint64
+	if currentUID == 0 {
+		// Sudo  has mounted
+		availableSpace = stat.Bfree * uint64(stat.Frsize)
+	} else {
+		// non Sudo has mounted
+		availableSpace = stat.Bavail * uint64(stat.Frsize)
+	}
+
+	totalSpace := stat.Blocks * uint64(stat.Frsize)
+	usedSpace := float64(totalSpace - availableSpace)
+	return usedSpace, float64(usedSpace) / float64(totalSpace) * 100, nil
+}

common/util_test.go

@@ -186,3 +186,46 @@ func (suite *utilTestSuite) TestExpandPath() {
 	expandedPath = ExpandPath(path)
 	suite.assert.Equal(expandedPath, path)
 }
+
+func (suite *utilTestSuite) TestGetUSage() {
+	pwd, err := os.Getwd()
+	if err != nil {
+		return
+	}
+
+	dirName := filepath.Join(pwd, "util_test")
+	err = os.Mkdir(dirName, 0777)
+	suite.assert.Nil(err)
+
+	data := make([]byte, 1024*1024)
+	err = os.WriteFile(dirName+"/1.txt", data, 0777)
+	suite.assert.Nil(err)
+
+	err = os.WriteFile(dirName+"/2.txt", data, 0777)
+	suite.assert.Nil(err)
+
+	usage, err := GetUsage(dirName)
+	suite.assert.Nil(err)
+	suite.assert.GreaterOrEqual(int(usage), 2)
+	suite.assert.LessOrEqual(int(usage), 4)
+
+	_ = os.RemoveAll(dirName)
+}
+
+func (suite *utilTestSuite) TestGetDiskUsage() {
+	pwd, err := os.Getwd()
+	if err != nil {
+		return
+	}
+
+	dirName := filepath.Join(pwd, "util_test", "a", "b", "c")
+	err = os.MkdirAll(dirName, 0777)
+	suite.assert.Nil(err)
+
+	usage, usagePercent, err := GetDiskUsageFromStatfs(dirName)
+	suite.assert.Nil(err)
+	suite.assert.NotEqual(usage, 0)
+	suite.assert.NotEqual(usagePercent, 0)
+	suite.assert.NotEqual(usagePercent, 100)
+	_ = os.RemoveAll(filepath.Join(pwd, "util_test"))
+}

component/file_cache/cache_policy.go

@@ -75,12 +75,25 @@ type cachePolicy interface {
 
 // getUsagePercentage:  The current cache usage as a percentage of the maxSize
 func getUsagePercentage(path string, maxSize float64) float64 {
+	var currSize float64
+	var usagePercent float64
+	var err error
+
 	if maxSize == 0 {
-		return 0
+		currSize, usagePercent, err = common.GetDiskUsageFromStatfs(path)
+		if err != nil {
+			log.Err("cachePolicy::getUsagePercentage : failed to get disk usage for %s [%v]", path, err.Error)
+		}
+	} else {
+		// We need to compuate % usage of temp directory against configured limit
+		currSize, err = common.GetUsage(path)
+		if err != nil {
+			log.Err("cachePolicy::getUsagePercentage : failed to get directory usage for %s [%v]", path, err.Error)
+		}
+
+		usagePercent = (currSize / float64(maxSize)) * 100
 	}
 
-	currSize, _ := common.GetUsage(path)
-	usagePercent := (currSize / float64(maxSize)) * 100
 	log.Debug("cachePolicy::getUsagePercentage : current cache usage : %f%%", usagePercent)
 
 	fileCacheStatsCollector.UpdateStats(stats_manager.Replace, cacheUsage, fmt.Sprintf("%f MB", currSize))

component/file_cache/cache_policy_test.go

@@ -71,24 +71,33 @@ func (suite *cachePolicyTestSuite) TestGetUsage() {
 	f.Write(data)
 	result, _ := common.GetUsage(cache_path)
 	suite.assert.Equal(float64(1), math.Floor(result))
+	f.Close()
 }
 
 func (suite *cachePolicyTestSuite) TestGetUsagePercentage() {
 	defer suite.cleanupTest()
-	f, _ := os.Create(cache_path + "/test")
 	data := make([]byte, 1024*1024)
+
+	f, _ := os.Create(cache_path + "/test")
 	f.Write(data)
 	result := getUsagePercentage(cache_path, 4)
 	// since the value might defer a little distro to distro
 	suite.assert.GreaterOrEqual(result, float64(25))
 	suite.assert.LessOrEqual(result, float64(30))
+	f.Close()
+
+	result = getUsagePercentage("/", 0)
+	// since the value might defer a little distro to distro
+	suite.assert.GreaterOrEqual(result, float64(0))
+	suite.assert.LessOrEqual(result, float64(90))
 }
 
 func (suite *cachePolicyTestSuite) TestDeleteFile() {
 	defer suite.cleanupTest()
 	f, _ := os.Create(cache_path + "/test")
 	result := deleteFile(f.Name() + "not_exist")
 	suite.assert.Equal(nil, result)
+	f.Close()
 }
 
 func TestCachePolicyTestSuite(t *testing.T) {