Add --sync-db to generate_changelog, plus many fixes

.github/workflows/update-version-and-create-tag.yml

@@ -9,6 +9,7 @@ on:
       - "**/*.md"
       - "data/strategies/**"
       - "cmd/generate_changelog/*.db"
+      - "cmd/generate_changelog/incoming/*.txt"
       - "scripts/pattern_descriptions/*.json"
       - "web/static/data/pattern_descriptions.json"
 

CHANGELOG.md

@@ -18,16 +18,6 @@
 - Enhance changelog generation to avoid duplicate commit entries by extracting PR numbers and filtering commits already included via PR files
 - Add version parameter requirement for PR processing with commit SHA tracking to prevent duplicate entries and improve formatting consistency
 
-## v1.4.262 (2025-07-21)
-
-### PR [#1640](https://github.com/danielmiessler/Fabric/pull/1640) by [ksylvan](https://github.com/ksylvan): Implement Automated Changelog System for CI/CD Integration
-
-- Add automated changelog processing for CI/CD integration with comprehensive test coverage and GitHub client validation methods
-- Implement release aggregation for incoming files with git operations for staging changes and support for version detection from nix files
-- Change push behavior from opt-out to opt-in with GitHub token authentication and automatic repository detection
-- Enhance changelog generation to avoid duplicate commit entries by extracting PR numbers and filtering commits already included via PR files
-- Add version parameter requirement for PR processing with commit SHA tracking to prevent duplicate entries and improve formatting consistency
-
 ### Direct commits
 
 - Docs: Update CHANGELOG after v1.4.261

cmd/generate_changelog/incoming/1642.txt

@@ -0,0 +1,7 @@
+### PR [#1642](https://github.com/danielmiessler/Fabric/pull/1642) by [ksylvan](https://github.com/ksylvan): Add --sync-db to `generate_changelog`, plus many fixes
+
+- Add database synchronization command with comprehensive validation and sync-db flag for database integrity validation
+- Implement version and commit existence checking methods with enhanced time parsing using RFC3339Nano fallback support
+- Improve timestamp handling and merge commit detection in changelog generator with comprehensive merge commit detection using parents
+- Add email field support to PRCommit struct for author information and improve error logging throughout changelog generation
+- Optimize merge pattern matching with lazy initialization and thread-safe pattern compilation for better performance

cmd/generate_changelog/internal/cache/cache.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/git"
@@ -201,7 +202,14 @@ func (c *Cache) GetVersions() (map[string]*git.Version, error) {
 		}
 
 		if dateStr.Valid {
-			v.Date, _ = time.Parse(time.RFC3339, dateStr.String)
+			// Try RFC3339Nano first (for nanosecond precision), then fall back to RFC3339
+			v.Date, err = time.Parse(time.RFC3339Nano, dateStr.String)
+			if err != nil {
+				v.Date, err = time.Parse(time.RFC3339, dateStr.String)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "Error parsing date '%s' for version '%s': %v. Expected format: RFC3339 or RFC3339Nano.\n", dateStr.String, v.Name, err)
+				}
+			}
 		}
 
 		if prNumbersJSON != "" {
@@ -260,6 +268,26 @@ func (c *Cache) Clear() error {
 	return nil
 }
 
+// VersionExists checks if a version already exists in the cache
+func (c *Cache) VersionExists(version string) (bool, error) {
+	var count int
+	err := c.db.QueryRow("SELECT COUNT(*) FROM versions WHERE name = ?", version).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+// CommitExists checks if a commit already exists in the cache
+func (c *Cache) CommitExists(hash string) (bool, error) {
+	var count int
+	err := c.db.QueryRow("SELECT COUNT(*) FROM commits WHERE sha = ?", hash).Scan(&count)
+	if err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
 // GetLastPRSync returns the timestamp of the last PR sync
 func (c *Cache) GetLastPRSync() (time.Time, error) {
 	var timestamp string

cmd/generate_changelog/internal/changelog/generator.go

@@ -65,7 +65,7 @@ func (g *Generator) Generate() (string, error) {
 		return "", fmt.Errorf("failed to collect data: %w", err)
 	}
 
-	if err := g.fetchPRs(); err != nil {
+	if err := g.fetchPRs(g.cfg.ForcePRSync); err != nil {
 		return "", fmt.Errorf("failed to fetch PRs: %w", err)
 	}
 
@@ -193,7 +193,7 @@ func (g *Generator) collectData() error {
 	return nil
 }
 
-func (g *Generator) fetchPRs() error {
+func (g *Generator) fetchPRs(forcePRSync bool) error {
 	// First, load all cached PRs
 	if g.cache != nil {
 		cachedPRs, err := g.cache.GetAllPRs()
@@ -229,7 +229,7 @@ func (g *Generator) fetchPRs() error {
 	}
 	// If we have never synced or it's been more than 24 hours, do a full sync
 	// Also sync if we have versions with PR numbers that aren't cached
-	needsSync := lastSync.IsZero() || time.Since(lastSync) > 24*time.Hour || g.cfg.ForcePRSync || missingPRs
+	needsSync := lastSync.IsZero() || time.Since(lastSync) > 24*time.Hour || forcePRSync || missingPRs
 
 	if !needsSync {
 		fmt.Fprintf(os.Stderr, "Using cached PR data (last sync: %s)\n", lastSync.Format("2006-01-02 15:04:05"))
@@ -697,3 +697,109 @@ func hashContent(content string) string {
 	hash := sha256.Sum256([]byte(content))
 	return fmt.Sprintf("%x", hash)
 }
+
+// SyncDatabase performs a comprehensive database synchronization and validation
+func (g *Generator) SyncDatabase() error {
+	if g.cache == nil {
+		return fmt.Errorf("cache is disabled, cannot sync database")
+	}
+
+	fmt.Fprintf(os.Stderr, "[SYNC] Starting database synchronization...\n")
+
+	// Step 1: Force PR sync (pass true explicitly)
+	fmt.Fprintf(os.Stderr, "[PR_SYNC] Forcing PR sync from GitHub...\n")
+	if err := g.fetchPRs(true); err != nil {
+		return fmt.Errorf("failed to sync PRs: %w", err)
+	}
+
+	// Step 2: Rebuild git history and verify versions/commits completeness
+	fmt.Fprintf(os.Stderr, "[VERIFY] Verifying git history and version completeness...\n")
+	if err := g.syncGitHistory(); err != nil {
+		return fmt.Errorf("failed to sync git history: %w", err)
+	}
+
+	// Step 3: Verify commit-PR mappings
+	fmt.Fprintf(os.Stderr, "[MAPPING] Verifying commit-PR mappings...\n")
+	if err := g.verifyCommitPRMappings(); err != nil {
+		return fmt.Errorf("failed to verify commit-PR mappings: %w", err)
+	}
+
+	fmt.Fprintf(os.Stderr, "[SUCCESS] Database synchronization completed successfully!\n")
+	return nil
+}
+
+// syncGitHistory walks the complete git history and ensures all versions and commits are cached
+func (g *Generator) syncGitHistory() error {
+	// Walk complete git history (reuse existing logic)
+	versions, err := g.gitWalker.WalkHistory()
+	if err != nil {
+		return fmt.Errorf("failed to walk git history: %w", err)
+	}
+
+	// Save only new versions and commits (preserve existing data)
+	var newVersions, newCommits int
+	for _, version := range versions {
+		// Only save version if it doesn't exist
+		exists, err := g.cache.VersionExists(version.Name)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to check existence of version %s: %v. This may affect the completeness of the sync operation.\n", version.Name, err)
+			continue
+		}
+		if !exists {
+			if err := g.cache.SaveVersion(version); err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to save version %s: %v\n", version.Name, err)
+			} else {
+				newVersions++
+			}
+		}
+
+		// Only save commits that don't exist
+		for _, commit := range version.Commits {
+			exists, err := g.cache.CommitExists(commit.SHA)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Warning: Failed to check commit %s existence: %v\n", commit.SHA, err)
+				continue
+			}
+			if !exists {
+				if err := g.cache.SaveCommit(commit, version.Name); err != nil {
+					fmt.Fprintf(os.Stderr, "Warning: Failed to save commit %s: %v\n", commit.SHA, err)
+				} else {
+					newCommits++
+				}
+			}
+		}
+	}
+
+	// Update last processed tag
+	if latestTag, err := g.gitWalker.GetLatestTag(); err == nil && latestTag != "" {
+		if err := g.cache.SetLastProcessedTag(latestTag); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to update last processed tag: %v\n", err)
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "   Added %d new versions and %d new commits (preserved existing data)\n", newVersions, newCommits)
+	return nil
+}
+
+// verifyCommitPRMappings ensures all PR commits have proper mappings
+func (g *Generator) verifyCommitPRMappings() error {
+	// Get all cached PRs
+	allPRs, err := g.cache.GetAllPRs()
+	if err != nil {
+		return fmt.Errorf("failed to get cached PRs: %w", err)
+	}
+
+	// Convert to slice for batch operations (reuse existing logic)
+	var prSlice []*github.PR
+	for _, pr := range allPRs {
+		prSlice = append(prSlice, pr)
+	}
+
+	// Save commit-PR mappings (reuse existing logic)
+	if err := g.cache.SaveCommitPRMappings(prSlice); err != nil {
+		return fmt.Errorf("failed to save commit-PR mappings: %w", err)
+	}
+
+	fmt.Fprintf(os.Stderr, "   Verified mappings for %d PRs\n", len(prSlice))
+	return nil
+}

cmd/generate_changelog/internal/changelog/merge_detection_test.go

@@ -0,0 +1,82 @@
+package changelog
+
+import (
+	"testing"
+	"time"
+
+	"github.com/danielmiessler/fabric/cmd/generate_changelog/internal/github"
+)
+
+func TestIsMergeCommit(t *testing.T) {
+	tests := []struct {
+		name     string
+		commit   github.PRCommit
+		expected bool
+	}{
+		{
+			name: "Regular commit with single parent",
+			commit: github.PRCommit{
+				SHA:     "abc123",
+				Message: "Fix bug in user authentication",
+				Author:  "John Doe",
+				Date:    time.Now(),
+				Parents: []string{"def456"},
+			},
+			expected: false,
+		},
+		{
+			name: "Merge commit with multiple parents",
+			commit: github.PRCommit{
+				SHA:     "abc123",
+				Message: "Merge pull request #42 from feature/auth",
+				Author:  "GitHub",
+				Date:    time.Now(),
+				Parents: []string{"def456", "ghi789"},
+			},
+			expected: true,
+		},
+		{
+			name: "Merge commit detected by message pattern only",
+			commit: github.PRCommit{
+				SHA:     "abc123",
+				Message: "Merge pull request #123 from user/feature-branch",
+				Author:  "GitHub",
+				Date:    time.Now(),
+				Parents: []string{}, // Empty parents - fallback to message detection
+			},
+			expected: true,
+		},
+		{
+			name: "Merge branch commit pattern",
+			commit: github.PRCommit{
+				SHA:     "abc123",
+				Message: "Merge branch 'feature' into main",
+				Author:  "Developer",
+				Date:    time.Now(),
+				Parents: []string{"def456"}, // Single parent but merge pattern
+			},
+			expected: true,
+		},
+		{
+			name: "Regular commit with no merge patterns",
+			commit: github.PRCommit{
+				SHA:     "abc123",
+				Message: "Add new feature for user management",
+				Author:  "Jane Doe",
+				Date:    time.Now(),
+				Parents: []string{"def456"},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isMergeCommit(tt.commit)
+			if result != tt.expected {
+				t.Errorf("isMergeCommit() = %v, expected %v for commit: %s",
+					result, tt.expected, tt.commit.Message)
+			}
+		})
+	}
+}