Merge pull request #1577 from ksylvan/0705-custom-patterns-dir

Add Custom Patterns Directory Support

Author: danielmiessler

core/plugin_registry.go

@@ -31,6 +31,7 @@ import (
 	"github.com/danielmiessler/fabric/plugins/db/fsdb"
 	"github.com/danielmiessler/fabric/plugins/template"
 	"github.com/danielmiessler/fabric/plugins/tools"
+	"github.com/danielmiessler/fabric/plugins/tools/custom_patterns"
 	"github.com/danielmiessler/fabric/plugins/tools/jina"
 	"github.com/danielmiessler/fabric/plugins/tools/lang"
 	"github.com/danielmiessler/fabric/plugins/tools/youtube"
@@ -69,6 +70,7 @@ func NewPluginRegistry(db *fsdb.Db) (ret *PluginRegistry, err error) {
 		VendorManager:  ai.NewVendorsManager(),
 		VendorsAll:     ai.NewVendorsManager(),
 		PatternsLoader: tools.NewPatternsLoader(db.Patterns),
+		CustomPatterns: custom_patterns.NewCustomPatterns(),
 		YouTube:        youtube.NewYouTube(),
 		Language:       lang.NewLanguage(),
 		Jina:           jina.NewClient(),
@@ -138,6 +140,7 @@ type PluginRegistry struct {
 	VendorsAll         *ai.VendorsManager
 	Defaults           *tools.Defaults
 	PatternsLoader     *tools.PatternsLoader
+	CustomPatterns     *custom_patterns.CustomPatterns
 	YouTube            *youtube.YouTube
 	Language           *lang.Language
 	Jina               *jina.Client
@@ -151,6 +154,7 @@ func (o *PluginRegistry) SaveEnvFile() (err error) {
 
 	o.Defaults.Settings.FillEnvFileContent(&envFileContent)
 	o.PatternsLoader.SetupFillEnvFileContent(&envFileContent)
+	o.CustomPatterns.SetupFillEnvFileContent(&envFileContent)
 	o.Strategies.SetupFillEnvFileContent(&envFileContent)
 
 	for _, vendor := range o.VendorManager.Vendors {
@@ -183,7 +187,7 @@ func (o *PluginRegistry) Setup() (err error) {
 			return vendor
 		})...)
 
-	groupsPlugins.AddGroupItems("Tools", o.Defaults, o.Jina, o.Language, o.PatternsLoader, o.Strategies, o.YouTube)
+	groupsPlugins.AddGroupItems("Tools", o.CustomPatterns, o.Defaults, o.Jina, o.Language, o.PatternsLoader, o.Strategies, o.YouTube)
 
 	for {
 		groupsPlugins.Print(false)

plugins/db/fsdb/db.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/joho/godotenv"
@@ -15,10 +16,22 @@ func NewDb(dir string) (db *Db) {
 
 	db.EnvFilePath = db.FilePath(".env")
 
+	// Check for custom patterns directory from environment variable
+	customPatternsDir := os.Getenv("CUSTOM_PATTERNS_DIRECTORY")
+	if customPatternsDir != "" {
+		// Expand home directory if needed
+		if strings.HasPrefix(customPatternsDir, "~/") {
+			if homeDir, err := os.UserHomeDir(); err == nil {
+				customPatternsDir = filepath.Join(homeDir, customPatternsDir[2:])
+			}
+		}
+	}
+
 	db.Patterns = &PatternsEntity{
 		StorageEntity:          &StorageEntity{Label: "Patterns", Dir: db.FilePath("patterns"), ItemIsDir: true},
 		SystemPatternFile:      "system.md",
 		UniquePatternsFilePath: db.FilePath("unique_patterns.txt"),
+		CustomPatternsDir:      customPatternsDir,
 	}
 
 	db.Sessions = &SessionsEntity{

plugins/db/fsdb/patterns.go

@@ -16,6 +16,7 @@ type PatternsEntity struct {
 	*StorageEntity
 	SystemPatternFile      string
 	UniquePatternsFilePath string
+	CustomPatternsDir      string
 }
 
 // Pattern represents a single pattern with its metadata
@@ -43,7 +44,7 @@ func (o *PatternsEntity) GetApplyVariables(
 		}
 
 		// Use the resolved absolute path to get the pattern
-		pattern, err = o.getFromFile(absPath)
+		pattern, _ = o.getFromFile(absPath)
 	} else {
 		// Otherwise, get the pattern from the database
 		pattern, err = o.getFromDB(source)
@@ -89,6 +90,19 @@ func (o *PatternsEntity) applyVariables(
 
 // retrieves a pattern from the database by name
 func (o *PatternsEntity) getFromDB(name string) (ret *Pattern, err error) {
+	// First check custom patterns directory if it exists
+	if o.CustomPatternsDir != "" {
+		customPatternPath := filepath.Join(o.CustomPatternsDir, name, o.SystemPatternFile)
+		if pattern, customErr := os.ReadFile(customPatternPath); customErr == nil {
+			ret = &Pattern{
+				Name:    name,
+				Pattern: string(pattern),
+			}
+			return ret, nil
+		}
+	}
+
+	// Fallback to main patterns directory
 	patternPath := filepath.Join(o.Dir, name, o.SystemPatternFile)
 
 	var pattern []byte
@@ -145,6 +159,48 @@ func (o *PatternsEntity) getFromFile(pathStr string) (pattern *Pattern, err erro
 	return
 }
 
+// GetNames overrides StorageEntity.GetNames to include custom patterns directory
+func (o *PatternsEntity) GetNames() (ret []string, err error) {
+	// Get names from main patterns directory
+	mainNames, err := o.StorageEntity.GetNames()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a map to track unique pattern names (custom patterns override main ones)
+	nameMap := make(map[string]bool)
+	for _, name := range mainNames {
+		nameMap[name] = true
+	}
+
+	// Get names from custom patterns directory if it exists
+	if o.CustomPatternsDir != "" {
+		// Create a temporary StorageEntity for the custom directory
+		customStorage := &StorageEntity{
+			Dir:           o.CustomPatternsDir,
+			ItemIsDir:     o.StorageEntity.ItemIsDir,
+			FileExtension: o.StorageEntity.FileExtension,
+		}
+
+		customNames, customErr := customStorage.GetNames()
+		if customErr == nil {
+			// Add custom patterns, they will override main patterns with same name
+			for _, name := range customNames {
+				nameMap[name] = true
+			}
+		}
+		// Ignore errors from custom directory (it might not exist)
+	}
+
+	// Convert map keys back to slice
+	ret = make([]string, 0, len(nameMap))
+	for name := range nameMap {
+		ret = append(ret, name)
+	}
+
+	return ret, nil
+}
+
 // Get required for Storage interface
 func (o *PatternsEntity) Get(name string) (*Pattern, error) {
 	// Use GetPattern with no variables

plugins/db/fsdb/patterns_test.go

@@ -162,3 +162,123 @@ func TestPatternsEntity_Save(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, content, data)
 }
+
+func TestPatternsEntity_CustomPatterns(t *testing.T) {
+	// Create main patterns directory
+	mainDir, err := os.MkdirTemp("", "test-main-patterns-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(mainDir)
+
+	// Create custom patterns directory
+	customDir, err := os.MkdirTemp("", "test-custom-patterns-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(customDir)
+
+	entity := &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       mainDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+		CustomPatternsDir: customDir,
+	}
+
+	// Create a pattern in main directory
+	createTestPattern(t, &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       mainDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+	}, "main-pattern", "Main pattern content")
+
+	// Create a pattern in custom directory
+	createTestPattern(t, &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       customDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+	}, "custom-pattern", "Custom pattern content")
+
+	// Create a pattern with same name in both directories (custom should override)
+	createTestPattern(t, &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       mainDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+	}, "shared-pattern", "Main shared pattern")
+
+	createTestPattern(t, &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       customDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+	}, "shared-pattern", "Custom shared pattern")
+
+	// Test GetNames includes both directories
+	names, err := entity.GetNames()
+	require.NoError(t, err)
+	assert.Contains(t, names, "main-pattern")
+	assert.Contains(t, names, "custom-pattern")
+	assert.Contains(t, names, "shared-pattern")
+
+	// Test that custom pattern overrides main pattern
+	pattern, err := entity.getFromDB("shared-pattern")
+	require.NoError(t, err)
+	assert.Equal(t, "Custom shared pattern", pattern.Pattern)
+
+	// Test that main pattern is accessible when not overridden
+	pattern, err = entity.getFromDB("main-pattern")
+	require.NoError(t, err)
+	assert.Equal(t, "Main pattern content", pattern.Pattern)
+
+	// Test that custom pattern is accessible
+	pattern, err = entity.getFromDB("custom-pattern")
+	require.NoError(t, err)
+	assert.Equal(t, "Custom pattern content", pattern.Pattern)
+}
+
+func TestPatternsEntity_CustomPatternsEmpty(t *testing.T) {
+	// Test behavior when custom patterns directory is empty or doesn't exist
+	mainDir, err := os.MkdirTemp("", "test-main-patterns-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(mainDir)
+
+	entity := &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       mainDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+		CustomPatternsDir: "/nonexistent/directory",
+	}
+
+	// Create a pattern in main directory
+	createTestPattern(t, &PatternsEntity{
+		StorageEntity: &StorageEntity{
+			Dir:       mainDir,
+			Label:     "patterns",
+			ItemIsDir: true,
+		},
+		SystemPatternFile: "system.md",
+	}, "main-pattern", "Main pattern content")
+
+	// Test GetNames works even with nonexistent custom directory
+	names, err := entity.GetNames()
+	require.NoError(t, err)
+	assert.Contains(t, names, "main-pattern")
+
+	// Test that main pattern is accessible
+	pattern, err := entity.getFromDB("main-pattern")
+	require.NoError(t, err)
+	assert.Equal(t, "Main pattern content", pattern.Pattern)
+}

plugins/tools/custom_patterns/custom_patterns.go

@@ -0,0 +1,61 @@
+package custom_patterns
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/danielmiessler/fabric/plugins"
+)
+
+func NewCustomPatterns() (ret *CustomPatterns) {
+	label := "Custom Patterns"
+	ret = &CustomPatterns{}
+
+	ret.PluginBase = &plugins.PluginBase{
+		Name:             label,
+		SetupDescription: "Custom Patterns - Set directory for your custom patterns (optional)",
+		EnvNamePrefix:    plugins.BuildEnvVariablePrefix(label),
+		ConfigureCustom:  ret.configure,
+	}
+
+	ret.CustomPatternsDir = ret.AddSetupQuestionCustom("Directory", false,
+		"Enter the path to your custom patterns directory (leave empty to skip)")
+
+	return
+}
+
+type CustomPatterns struct {
+	*plugins.PluginBase
+	CustomPatternsDir *plugins.SetupQuestion
+}
+
+func (o *CustomPatterns) configure() error {
+	if o.CustomPatternsDir.Value != "" {
+		// Expand home directory if needed
+		if strings.HasPrefix(o.CustomPatternsDir.Value, "~/") {
+			if homeDir, err := os.UserHomeDir(); err == nil {
+				o.CustomPatternsDir.Value = filepath.Join(homeDir, o.CustomPatternsDir.Value[2:])
+			}
+		}
+
+		// Convert to absolute path
+		if absPath, err := filepath.Abs(o.CustomPatternsDir.Value); err == nil {
+			o.CustomPatternsDir.Value = absPath
+		}
+
+		// Create the directory if it doesn't exist
+		if err := os.MkdirAll(o.CustomPatternsDir.Value, 0755); err != nil {
+			// If we can't create it, clear the value to avoid errors
+			o.CustomPatternsDir.Value = ""
+		}
+	}
+
+	return nil
+}
+
+// IsConfigured returns true if a custom patterns directory has been set
+func (o *CustomPatterns) IsConfigured() bool {
+	// Check if the plugin has been configured with a directory
+	return o.CustomPatternsDir.Value != ""
+}