✨ [filesystem] Add filepath utilities to work across file-systems no matter the underlying platform (#639)

!--
Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors.
All rights reserved.
SPDX-License-Identifier: Apache-2.0
-->
### Description

- Add utilities to manage paths
- Fix bug with regards to zipping between filesystem with different path
separators


### Test Coverage

<!--
Please put an `x` in the correct box e.g. `[x]` to indicate the testing
coverage of this change.
-->

- [x]  This change is covered by existing or additional automated tests.
- [ ] Manual testing has been performed (and evidence provided) as
automated testing was not feasible.
- [ ] Additional tests are not required for this change (e.g.
documentation update).

Author: acabarbaye

.github/workflows/ci.yml

@@ -107,6 +107,11 @@ jobs:
         run: go build -v ./...
         working-directory: ${{ matrix.go-module }}
 
+      - if: ${{ startsWith(matrix.os, 'ubuntu') }}
+        name: Test [${{ matrix.go-module }} on ${{ matrix.os }}]
+        run: go test -race -cover -v -tags integration -timeout 30m -coverprofile ${{ matrix.go-module }}_coverage.out ./...
+        working-directory: ${{ matrix.go-module }}
+
 # FIXME: Run tests on Mac and Windows
 #      - if: ${{ startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macOS') }}
 #        run: go test -race -cover -v ./...

.secrets.baseline

@@ -66,10 +66,6 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
-    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -276,5 +272,5 @@
       }
     ]
   },
-  "generated_at": "2025-06-13T15:31:00Z"
+  "generated_at": "2025-06-27T16:00:59Z"
 }

changes/20250627145431.feature

@@ -0,0 +1 @@
+:sparkles: `[filesystem]` Add filepath utilities to work across file-systems no matter the underlying platform

changes/20250627161647.bugfix

@@ -0,0 +1 @@
+:bug: `[filesystem]` fix [bug](https://unix.stackexchange.com/questions/166159/convert-a-windows-created-zip-to-linux-internal-paths-issue) when zipping a package on Windows and unzipping it on a different platform such as Linux

utils/collection/parseLists_test.go

@@ -6,9 +6,7 @@ package collection
 
 import (
 	"maps"
-	"math/rand"
 	"testing"
-	"time"
 
 	"github.com/go-faker/faker/v4"
 	"github.com/stretchr/testify/assert"
@@ -18,10 +16,6 @@ import (
 	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
 )
 
-var (
-	random = rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec //causes G404: Use of weak random number generator (math/rand instead of crypto/rand) (gosec), So disable gosec as this is just for
-)
-
 func TestParseCommaSeparatedListWordsOnly(t *testing.T) {
 	t.Run("simple", func(t *testing.T) {
 		stringArray := []string{faker.Word(), faker.Word(), faker.Word(), faker.UUIDDigit(), faker.URL(), faker.Username(), faker.DomainName()}
@@ -30,14 +24,15 @@ func TestParseCommaSeparatedListWordsOnly(t *testing.T) {
 	t.Run("with whitespaces", func(t *testing.T) {
 		stringList := ""
 		var stringArray []string
-		// we don't need cryptographically secure random numbers for generating a number of elements in a list
-		lengthOfList := random.Intn(10) //nolint:gosec
-		for i := 0; i < lengthOfList; i++ {
+		lengthOfList, err := faker.RandomInt(1, 10, 1)
+		require.NoError(t, err)
+		for i := 0; i < lengthOfList[0]; i++ {
 			word := faker.Word()
 			stringList += word
 			stringArray = append(stringArray, word)
-			numSpacesToAdd := random.Intn(5) //nolint:gosec
-			for j := 0; j < numSpacesToAdd; j++ {
+			numSpacesToAdd, err := faker.RandomInt(0, 5, 1)
+			require.NoError(t, err)
+			for j := 0; j < numSpacesToAdd[0]; j++ {
 				stringList += " "
 			}
 			stringList += ","
@@ -56,14 +51,15 @@ func TestParseCommaSeparatedListWithSpacesBetweenWords(t *testing.T) {
 	t.Run("with whitespaces", func(t *testing.T) {
 		stringList := ""
 		var stringArray []string
-		// we don't need cryptographically secure random numbers for generating a number of elements in a list
-		lengthOfList := random.Intn(10) //nolint:gosec
-		for i := 0; i < lengthOfList; i++ {
+		lengthOfList, err := faker.RandomInt(1, 10, 1)
+		require.NoError(t, err)
+		for i := 0; i < lengthOfList[0]; i++ {
 			word := faker.Sentence()
 			stringList += word
 			stringArray = append(stringArray, word)
-			numSpacesToAdd := random.Intn(5) //nolint:gosec
-			for j := 0; j < numSpacesToAdd; j++ {
+			numSpacesToAdd, err := faker.RandomInt(0, 5, 1)
+			require.NoError(t, err)
+			for j := 0; j < numSpacesToAdd[0]; j++ {
 				stringList += " "
 			}
 			stringList += ","
@@ -76,20 +72,22 @@ func TestParseCommaSeparatedListWithSpacesBetweenWords(t *testing.T) {
 func TestParseCommaSeparatedListWithSpacesBetweenWordsKeepBlanks(t *testing.T) {
 	stringList := ""
 	var stringArray []string
-	// we don't need cryptographically secure random numbers for generating a number of elements in a list
-	lengthOfList := random.Intn(10) + 8 //nolint:gosec
-	for i := 0; i < lengthOfList; i++ {
+	lengthOfList, err := faker.RandomInt(8, 18, 1)
+	require.NoError(t, err)
+	for i := 0; i < lengthOfList[0]; i++ {
 		word := faker.Sentence()
 		stringList += word
 		stringArray = append(stringArray, word)
-		numSpacesToAdd := random.Intn(5) //nolint:gosec
-		for j := 0; j < numSpacesToAdd; j++ {
+		numSpacesToAdd, err := faker.RandomInt(0, 5, 1)
+		require.NoError(t, err)
+		for j := 0; j < numSpacesToAdd[0]; j++ {
 			stringList += " "
 		}
 		stringList += ","
 		if i%3 == 2 {
-			numSpacesToAdd := random.Intn(5) //nolint:gosec
-			for j := 0; j < numSpacesToAdd; j++ {
+			numSpacesToAdd, err := faker.RandomInt(0, 5, 1)
+			require.NoError(t, err)
+			for j := 0; j < numSpacesToAdd[0]; j++ {
 				stringList += " "
 			}
 			stringArray = append(stringArray, "")

utils/filesystem/embedfs_test.go

@@ -19,7 +19,7 @@ import (
 //go:embed *
 var testContent embed.FS
 
-const testFile1Content = "this is a text file with some content\n"
+const testFile1Content = "this is a text file with some content"
 
 func Test_embedFS_Exists(t *testing.T) {
 	fs, err := NewEmbedFileSystem(&testContent)
@@ -97,7 +97,7 @@ func Test_embedFS_Read(t *testing.T) {
 	t.Run("embed read", func(t *testing.T) {
 		c, err := testContent.ReadFile("testdata/embed/test.txt")
 		require.NoError(t, err)
-		assert.Equal(t, testFile1Content, string(c))
+		assert.Contains(t, string(c), testFile1Content)
 	})
 
 	t.Run("using file opening", func(t *testing.T) {
@@ -106,7 +106,7 @@ func Test_embedFS_Read(t *testing.T) {
 		defer func() { _ = f.Close() }()
 		c, err := io.ReadAll(f)
 		require.NoError(t, err)
-		assert.Equal(t, testFile1Content, string(c))
+		assert.Contains(t, string(c), testFile1Content)
 		require.NoError(t, f.Close())
 	})
 
@@ -116,14 +116,14 @@ func Test_embedFS_Read(t *testing.T) {
 		defer func() { _ = f.Close() }()
 		c, err := io.ReadAll(f)
 		require.NoError(t, err)
-		assert.Equal(t, testFile1Content, string(c))
+		assert.Contains(t, string(c), testFile1Content)
 		require.NoError(t, f.Close())
 	})
 
 	t.Run("using file read", func(t *testing.T) {
 		c, err := efs.ReadFile("testdata/embed/test.txt")
 		require.NoError(t, err)
-		assert.Equal(t, testFile1Content, string(c))
+		assert.Contains(t, string(c), testFile1Content)
 	})
 
 }

utils/filesystem/filepath.go

@@ -16,10 +16,10 @@ import (
 
 // FilepathStem returns  the final path component, without its suffix. It's similar to `stem` in python's [pathlib](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.stem)
 func FilepathStem(fp string) string {
-	return strings.TrimSuffix(filepath.Base(fp), filepath.Ext(fp))
+	return FilePathStemOnFilesystem(GetGlobalFileSystem(), fp)
 }
 
-// FilepathParents returns a list of to the logical ancestors of the path and it's similar to `parents` in python's [pathlib](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.parents)
+// FilepathParents returns a list of to the logical ancestors of the path, and it's similar to `parents` in python's [pathlib](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.parents)
 func FilepathParents(fp string) []string {
 	return FilePathParentsOnFilesystem(GetGlobalFileSystem(), fp)
 }
@@ -55,16 +55,16 @@ func FilePathJoin(fs FS, element ...string) string {
 		return ""
 	}
 
-	if fs.PathSeparator() == '/' {
+	if isSlashSeparator(fs) {
 		return path.Join(element...)
 	}
 
-	joinedPath := filepath.Join(element...)
-	if fs.PathSeparator() != platform.PathListSeparator {
-		joinedPath = strings.ReplaceAll(joinedPath, string(platform.PathListSeparator), string(fs.PathSeparator()))
+	elements := make([]string, len(element))
+	for i := range elements {
+		elements[i] = FilePathToPlatformPathSeparator(fs, element[i])
 	}
 
-	return joinedPath
+	return FilePathFromPlatformPathSeparator(fs, filepath.Join(elements...))
 }
 
 // FilePathBase has the same behaviour as filepath.Base, but can handle different filesystems.
@@ -73,15 +73,145 @@ func FilePathBase(fs FS, fp string) string {
 		return ""
 	}
 
-	if fs.PathSeparator() == '/' {
+	if isSlashSeparator(fs) {
 		return path.Base(fp)
 	}
 
-	if fs.PathSeparator() != platform.PathListSeparator {
-		fp = strings.ReplaceAll(fp, string(fs.PathSeparator()), string(platform.PathListSeparator))
+	return FilePathFromPlatformPathSeparator(fs, filepath.Base(FilePathToPlatformPathSeparator(fs, fp)))
+}
+
+// FilePathDir has the same behaviour as filepath.Dir, but can handle different filesystems.
+func FilePathDir(fs FS, fp string) string {
+	if fs == nil {
+		return ""
+	}
+
+	if isSlashSeparator(fs) {
+		return path.Dir(fp)
+	}
+	return FilePathFromPlatformPathSeparator(fs, filepath.Dir(FilePathToPlatformPathSeparator(fs, fp)))
+}
+
+// FilePathIsAbs has the same behaviour as filepath.IsAbs, but can handle different filesystems.
+func FilePathIsAbs(fs FS, fp string) bool {
+	if fs == nil {
+		return false
+	}
+	if isSlashSeparator(fs) {
+		return path.IsAbs(fp)
+	}
+	return filepath.IsAbs(FilePathToPlatformPathSeparator(fs, fp))
+}
+
+// FilePathRel has the same behaviour as filepath.FilePathRel, but can handle different filesystems.
+func FilePathRel(fs FS, basepath, targpath string) (rel string, err error) {
+	if fs == nil {
+		err = commonerrors.UndefinedVariable("filesystem specification")
+		return
+	}
+	rel, err = filepath.Rel(FilePathToPlatformPathSeparator(fs, basepath), FilePathToPlatformPathSeparator(fs, targpath))
+	if err != nil {
+		return
+	}
+	rel = FilePathFromPlatformPathSeparator(fs, rel)
+	return
+}
+
+// FilePathSplit has the same behaviour as filepath.Split, but can handle different filesystems
+func FilePathSplit(fs FS, fp string) (dir, file string) {
+	if fs == nil {
+		return
+	}
+
+	if isSlashSeparator(fs) {
+		return path.Split(fp)
+	}
+	dir, file = filepath.Split(FilePathToPlatformPathSeparator(fs, fp))
+	dir = FilePathFromPlatformPathSeparator(fs, dir)
+	return
+}
+
+// FilePathAbs tries to be similar to filepath.Abs behaviour but without using platform information or location.
+func FilePathAbs(fs FS, fp, currentDirectory string) string {
+	if FilePathIsAbs(fs, fp) {
+		return FilePathClean(fs, fp)
+	}
+	return FilePathJoin(fs, currentDirectory, fp)
+}
+
+// FilePathToPlatformPathSeparator returns the result of replacing each separator character in path with a platform path separator character
+func FilePathToPlatformPathSeparator(fs FS, path string) string {
+	if fs.PathSeparator() == platform.PathSeparator {
+		return path
+	}
+	return strings.ReplaceAll(path, string(fs.PathSeparator()), string(platform.PathSeparator))
+}
+
+// FilePathFromPlatformPathSeparator returns the result of replacing each platform path separator character in path with filesystem's path separator character
+func FilePathFromPlatformPathSeparator(fs FS, path string) string {
+	if fs.PathSeparator() == platform.PathSeparator {
+		return path
 	}
+	return strings.ReplaceAll(path, string(platform.PathSeparator), string(fs.PathSeparator()))
+}
 
-	return filepath.Base(fp)
+// FilePathToSlash is filepath.ToSlash  but using filesystem path separator rather than platform's
+func FilePathToSlash(fs FS, path string) string {
+	if isSlashSeparator(fs) {
+		return path
+	}
+	return strings.ReplaceAll(path, string(fs.PathSeparator()), "/")
+}
+
+// FilePathsToSlash just applying FilePathToSlash to a slice of path
+func FilePathsToSlash(fs FS, path ...string) (toSlashes []string) {
+	if len(path) == 0 {
+		return
+	}
+	toSlashes = make([]string, len(path))
+	for i := range path {
+		toSlashes[i] = FilePathToSlash(fs, path[i])
+	}
+	return
+}
+
+// FilePathFromSlash is filepath.FromSlash but using filesystem path separator rather than platform's
+func FilePathFromSlash(fs FS, path string) string {
+	if isSlashSeparator(fs) {
+		return path
+	}
+	return strings.ReplaceAll(path, "/", string(fs.PathSeparator()))
+}
+
+// FilePathsFromSlash just applying FilePathFromSlash to a slice of path
+func FilePathsFromSlash(fs FS, path ...string) (fromlashes []string) {
+	if len(path) == 0 {
+		return
+	}
+	fromlashes = make([]string, len(path))
+	for i := range path {
+		fromlashes[i] = FilePathFromSlash(fs, path[i])
+	}
+	return
+}
+
+// FilePathStemOnFilesystem has the same behaviour as FilePathStem, but can handle different filesystems.
+func FilePathStemOnFilesystem(fs FS, fp string) string {
+	if fs == nil {
+		return ""
+	}
+	return strings.TrimSuffix(FilePathBase(fs, fp), FilePathExt(fs, fp))
+}
+
+// FilePathExt has the same behaviour as filepath.Ext, but can handle different filesystems.
+func FilePathExt(fs FS, fp string) string {
+	if fs == nil {
+		return ""
+	}
+	if isSlashSeparator(fs) {
+		return path.Ext(fp)
+	}
+	return filepath.Ext(FilePathToPlatformPathSeparator(fs, fp))
 }
 
 // FilePathClean has the same behaviour as filepath.Clean, but can handle different filesystems.
@@ -90,11 +220,26 @@ func FilePathClean(fs FS, fp string) string {
 		return ""
 	}
 
-	if fs.PathSeparator() == '/' {
+	if isSlashSeparator(fs) {
 		return path.Clean(fp)
 	}
+	return FilePathFromPlatformPathSeparator(fs, filepath.Clean(FilePathToPlatformPathSeparator(fs, fp)))
+}
+
+// FilePathVolumeName has the same behaviour as filepath.VolumeName, but can handle different filesystems.
+func FilePathVolumeName(fs FS, fp string) string {
+	if fs == nil {
+		return ""
+	}
 
-	return filepath.Clean(fp)
+	return filepath.VolumeName(FilePathToPlatformPathSeparator(fs, fp))
+}
+
+func isSlashSeparator(fs FS) bool {
+	if fs == nil {
+		return false
+	}
+	return fs.PathSeparator() == '/'
 }
 
 // FileTreeDepth returns the depth of a file in a tree starting from root
@@ -110,7 +255,7 @@ func FileTreeDepth(fs FS, root, filePath string) (depth int64, err error) {
 	if reflection.IsEmpty(diff) {
 		return
 	}
-	diff = strings.ReplaceAll(diff, string(fs.PathSeparator()), "/")
+	diff = FilePathToSlash(fs, diff)
 	depth = int64(len(strings.Split(diff, "/")) - 1)
 	return
 }