bug fix duplicate posts & refactor

Author: EricFrancis12

autoposter.go

@@ -47,7 +47,7 @@ func (a *AutoPoster) Run() {
 					continue
 				}
 
-				savedPosts, err := a.db.GetPostsByTwitterID(acct.TwitterID)
+				savedPosts, err := a.db.GetSavedPostsByTwitterID(acct.TwitterID)
 				if err != nil {
 					PrintErrWithTimeout(err, a.timeout)
 					continue
@@ -59,10 +59,8 @@ func (a *AutoPoster) Run() {
 					freshPosts = posts
 				} else {
 					for _, post := range posts {
-						for _, savedPost := range savedPosts {
-							if post.Url != savedPost.Url {
-								freshPosts = append(freshPosts, post)
-							}
+						if !post.InSaved(savedPosts) {
+							freshPosts = append(freshPosts, post)
 						}
 					}
 				}
@@ -82,7 +80,7 @@ func (a *AutoPoster) Run() {
 						continue
 					}
 
-					if err := a.db.InsertPost(p.ToSaved(acct.TwitterID)); err != nil {
+					if err := a.db.InsertSavedPost(p.ToSaved(acct.TwitterID)); err != nil {
 						PrintErr(err)
 					}
 

db.go

@@ -35,7 +35,7 @@ func NewDB(driverName string, dataSourceName string) (*DB, error) {
 		create table if not exists %s (
 			id text not null primary key,
 			twitter_id text not null,
-			url text not null,
+			url text not null unique,
 			title text not null
 		);
 	`, PostsTableName)
@@ -52,33 +52,33 @@ func NewDB(driverName string, dataSourceName string) (*DB, error) {
 	}, nil
 }
 
-func (db DB) InsertPost(sp SavedPost) error {
-	query := fmt.Sprintf(`
+func (db DB) InsertSavedPost(sp SavedPost) error {
+	row := db.Client.QueryRow(fmt.Sprintf(`select * from %s where url = "%s"`, PostsTableName, sp.Url))
+	if row.Scan() != sql.ErrNoRows {
+		return fmt.Errorf(`SavedPost with url "%s" already present in db`, sp.Url)
+	}
+
+	insertQuery := fmt.Sprintf(`
 		insert into %s (id, twitter_id, url, title) 
 		values (?, ?, ?, ?);
 	`, PostsTableName)
 
-	// Remove query string to ensure proper url match via autoposter
-	url, err := stripQueryString(sp.Url)
-	if err != nil {
-		return err
-	}
-
-	if _, err := db.Client.Exec(query, sp.ID, sp.TwitterID, url, sp.Title); err != nil {
+	if _, err := db.Client.Exec(insertQuery, sp.ID, sp.TwitterID, sp.Url, sp.Title); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (db DB) GetPostsByTwitterID(twitterID string) ([]SavedPost, error) {
+func (db DB) GetSavedPostsByTwitterID(twitterID string) ([]SavedPost, error) {
+	var savedPosts []SavedPost
+
 	rows, err := db.Client.Query(fmt.Sprintf(`select * from %s where twitter_id = "%s"`, PostsTableName, twitterID))
 	if err != nil {
-		return []SavedPost{}, err
+		return savedPosts, err
 	}
 	defer rows.Close()
 
-	var savedPosts []SavedPost
 	for rows.Next() {
 		sp, err := scanIntoSavedPost(rows)
 		if err != nil {

main.go

@@ -8,7 +8,7 @@ import (
 const (
 	ConfigFilePath string = "config.json"
 	// TODO: Impliment a max and min timeout to reduce chance of bot detection,
-	// and also make it configurable.
+	// and also make it configurable via flags.
 	Timeout time.Duration = time.Minute * 18
 )
 

post.go

@@ -7,11 +7,17 @@ type Post struct {
 	Title string
 }
 
-func NewPost(url string, title string) *Post {
+func NewPost(url string, title string) (*Post, error) {
+	// Remove query string to ensure proper url match in autoposter
+	url, err := stripQueryString(url)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Post{
 		Url:   url,
 		Title: title,
-	}
+	}, nil
 }
 
 func (p Post) ToSaved(twitterID string) SavedPost {
@@ -21,3 +27,17 @@ func (p Post) ToSaved(twitterID string) SavedPost {
 		Post:      p,
 	}
 }
+
+// TODO: refactor to combine In() and InSaved()
+
+func (p Post) In(posts []Post) bool {
+	return Some(posts, func(post Post) bool {
+		return post.Url == p.Url
+	})
+}
+
+func (p Post) InSaved(savedPosts []SavedPost) bool {
+	return Some(savedPosts, func(savedPost SavedPost) bool {
+		return savedPost.Url == p.Url
+	})
+}

source.go

@@ -47,7 +47,14 @@ func (r RssFeedSource) FetchPosts() ([]Post, error) {
 	}
 
 	for _, item := range feed.Items {
-		posts = append(posts, *NewPost(item.Link, item.Title))
+		post, err := NewPost(item.Link, item.Title)
+		if err != nil {
+			continue
+		}
+
+		if !post.In(posts) {
+			posts = append(posts, *post)
+		}
 	}
 
 	return posts, nil

util.go

@@ -6,6 +6,15 @@ import (
 	"time"
 )
 
+func Some[T any](arr []T, f func(T) bool) bool {
+	for _, t := range arr {
+		if f(t) {
+			return true
+		}
+	}
+	return false
+}
+
 func stripQueryString(u string) (string, error) {
 	parsedUrl, err := url.Parse(u)
 	if err != nil {
@@ -19,15 +28,15 @@ func stripQueryString(u string) (string, error) {
 
 func PrintWithTimestamp(s string) {
 	timestamp := time.Now().Format(time.UnixDate)
-	fmt.Printf("[%s] error: %s\n", timestamp, s)
+	fmt.Printf("[%s] %s\n\n", timestamp, s)
 }
 
 func PrintWithTimestampf(format string, a ...any) {
 	PrintWithTimestamp(fmt.Sprintf(format, a...))
 }
 
 func PrintErr(err error) {
-	PrintWithTimestamp(err.Error())
+	PrintWithTimestampf("Error: %s", err.Error())
 }
 
 func PrintErrWithTimeout(err error, timeout time.Duration) {