Combine rules with boolean operators.

Rules in *.list files can now be combined with boolean operators:

<django> & <guide> 25
google.com &^ /search/p 300
bing.com & (<roses> | <pansies>) 100

(&^ is AND NOT.)

Author: andybalholm

acl.go

@@ -30,11 +30,11 @@ type ACLDefinitions struct {
 	ConnectPorts      map[int][]string
 	ContentTypes      map[string][]string
 	Methods           map[string][]string
-	Referers          map[string][]string
+	Referers          map[rule][]string
 	RefererCategories map[string][]string
 	StatusCodes       map[int][]string
 	URLs              *URLMatcher
-	URLTags           map[string][]string
+	URLTags           map[rule][]string
 	UserIPs           IPMap
 	UserNames         map[string][]string
 	ServerIPs         IPMap
@@ -116,12 +116,13 @@ func (a *ACLDefinitions) AddRule(acl string, newRule []string) error {
 			a.URLs = newURLMatcher()
 		}
 		if a.Referers == nil {
-			a.Referers = make(map[string][]string)
+			a.Referers = make(map[rule][]string)
 		}
 		for _, u := range args {
 			u = strings.ToLower(u)
-			a.URLs.AddRule(rule{t: urlMatch, content: u})
-			a.Referers[u] = append(a.Referers[u], acl)
+			r := simpleRule{t: urlMatch, content: u}
+			a.URLs.AddRule(r)
+			a.Referers[r] = append(a.Referers[r], acl)
 		}
 
 	case "referer-category", "referrer-category":
@@ -171,12 +172,13 @@ func (a *ACLDefinitions) AddRule(acl string, newRule []string) error {
 			a.URLs = newURLMatcher()
 		}
 		if a.URLTags == nil {
-			a.URLTags = make(map[string][]string)
+			a.URLTags = make(map[rule][]string)
 		}
 		for _, u := range args {
 			u = strings.ToLower(u)
-			a.URLs.AddRule(rule{t: urlMatch, content: u})
-			a.URLTags[u] = append(a.URLTags[u], acl)
+			r := simpleRule{t: urlMatch, content: u}
+			a.URLs.AddRule(r)
+			a.URLTags[r] = append(a.URLTags[r], acl)
 		}
 
 	case "user-agent":
@@ -355,7 +357,7 @@ func (a *ACLDefinitions) requestACLs(r *http.Request, user string) map[string]bo
 
 	if a.URLs != nil {
 		for match := range a.URLs.MatchingRules(r.URL) {
-			for _, acl := range a.URLTags[match.content] {
+			for _, acl := range a.URLTags[match] {
 				acls[acl] = true
 			}
 		}
@@ -364,7 +366,7 @@ func (a *ACLDefinitions) requestACLs(r *http.Request, user string) map[string]bo
 			refURL, err := url.Parse(referer)
 			if err == nil {
 				for match := range a.URLs.MatchingRules(refURL) {
-					for _, acl := range a.Referers[match.content] {
+					for _, acl := range a.Referers[match] {
 						acls[acl] = true
 					}
 				}

categories.go

@@ -205,7 +205,7 @@ func loadRuleFile(c *category, filename string, multiplier float64) {
 			break
 		}
 
-		r, line, err := parseRule(line)
+		r, line, err := parseCompoundRule(line)
 		if err != nil {
 			log.Printf("Error in line %d of %s: %s", cr.LineNo, filename, err)
 			continue
@@ -220,43 +220,54 @@ func loadRuleFile(c *category, filename string, multiplier float64) {
 			w.maxPoints = int(float64(w.maxPoints) * multiplier)
 		}
 
-		if r.t == defaultRule {
+		if sr, ok := r.(simpleRule); ok && sr.t == defaultRule {
 			defaultWeight = w.points
 		} else {
 			c.weights[r] = w
 		}
 	}
 }
 
+func (cf *config) addRule(r rule) {
+	switch r := r.(type) {
+	case simpleRule:
+		switch r.t {
+		case contentPhrase:
+			cf.ContentPhraseList.addPhrase(r.content)
+		case imageHash:
+			content := r.content
+			threshold := -1
+			if dash := strings.Index(content, "-"); dash != -1 {
+				t, err := strconv.Atoi(content[dash+1:])
+				if err != nil {
+					log.Printf("%v: %v", r, err)
+					return
+				}
+				threshold = t
+				content = content[:dash]
+			}
+			h, err := dhash.Parse(content)
+			if err != nil {
+				log.Printf("%v: %v", r, err)
+				return
+			}
+			cf.ImageHashes = append(cf.ImageHashes, dhashWithThreshold{h, threshold})
+		default:
+			cf.URLRules.AddRule(r)
+		}
+	case compoundRule:
+		cf.addRule(r.left)
+		cf.addRule(r.right)
+		cf.CompoundRules = append(cf.CompoundRules, r)
+	}
+}
+
 // collectRules collects the rules from all the categories and adds
 // them to URLRules and phraseRules.
 func (cf *config) collectRules() {
 	for _, c := range cf.Categories {
-		for rule, _ := range c.weights {
-			switch rule.t {
-			case contentPhrase:
-				cf.ContentPhraseList.addPhrase(rule.content)
-			case imageHash:
-				content := rule.content
-				threshold := -1
-				if dash := strings.Index(content, "-"); dash != -1 {
-					t, err := strconv.Atoi(content[dash+1:])
-					if err != nil {
-						log.Printf("%v: %v", rule, err)
-						continue
-					}
-					threshold = t
-					content = content[:dash]
-				}
-				h, err := dhash.Parse(content)
-				if err != nil {
-					log.Printf("%v: %v", rule, err)
-					continue
-				}
-				cf.ImageHashes = append(cf.ImageHashes, dhashWithThreshold{h, threshold})
-			default:
-				cf.URLRules.AddRule(rule)
-			}
+		for r, _ := range c.weights {
+			cf.addRule(r)
 		}
 	}
 	cf.ContentPhraseList.findFallbackNodes(0, nil)
@@ -284,12 +295,39 @@ func (c *category) score(tally map[rule]int, conf *config) int {
 	return total
 }
 
+func (cf *config) applyCompoundRules(tally map[rule]int) {
+	for _, cr := range cf.CompoundRules {
+		left := tally[cr.left]
+		right := tally[cr.right]
+		combined := left
+		switch cr.op {
+		case "&":
+			if right < left {
+				combined = right
+			}
+		case "|":
+			if right > left {
+				combined = right
+			}
+		case "&^":
+			if right != 0 {
+				combined = 0
+			}
+		}
+		if combined != 0 {
+			tally[cr] = combined
+		}
+	}
+}
+
 // categoryScores returns a map containing a page's score for each category.
 func (cf *config) categoryScores(tally map[rule]int) map[string]int {
 	if len(tally) == 0 {
 		return nil
 	}
 
+	cf.applyCompoundRules(tally)
+
 	scores := make(map[string]int)
 	for _, c := range cf.Categories {
 		s := c.score(tally, cf)

classify.go

@@ -117,7 +117,7 @@ func handleClassification(w http.ResponseWriter, r *http.Request) {
 		for _, h := range conf.ImageHashes {
 			distance := dhash.Distance(hash, h.Hash)
 			if distance <= h.Threshold || h.Threshold == -1 && distance <= conf.DhashThreshold {
-				tally[rule{imageHash, h.String()}]++
+				tally[simpleRule{imageHash, h.String()}]++
 				scoresNeedUpdate = true
 			}
 		}

combinators.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+// parser combinators, inspired by nom
+
+type parser[T any] func(input string) (value T, rest string, err error)
+
+func tag(t string) parser[string] {
+	return func(input string) (string, string, error) {
+		s := strings.TrimLeft(input, " \t\r\n\f")
+		if strings.HasPrefix(s, t) {
+			return t, strings.TrimPrefix(s, t), nil
+		} else {
+			return "", input, fmt.Errorf("not found: %q", t)
+		}
+	}
+}
+
+func anyTag(tags ...string) parser[string] {
+	return func(input string) (string, string, error) {
+		s := strings.TrimLeft(input, " \t\r\n\f")
+		for _, t := range tags {
+			if strings.HasPrefix(s, t) {
+				return t, strings.TrimPrefix(s, t), nil
+			}
+		}
+		return "", input, fmt.Errorf("not found: %q", tags)
+	}
+}
+
+func alt[T any](options ...parser[T]) parser[T] {
+	return func(input string) (value T, rest string, err error) {
+		for _, p := range options {
+			value, rest, err = p(input)
+			if err == nil {
+				return
+			}
+		}
+		return
+	}
+}
+
+func delimited[T, U, V any](left parser[T], inner parser[U], right parser[V]) parser[U] {
+	return func(input string) (value U, rest string, err error) {
+		_, rest, err = left(input)
+		if err != nil {
+			return
+		}
+		value, rest, err = inner(rest)
+		if err != nil {
+			return
+		}
+		_, rest, err = right(rest)
+		return
+	}
+}

config.go

@@ -50,6 +50,7 @@ type config struct {
 	CountOnce          bool
 	Threshold          int
 	URLRules           *URLMatcher
+	CompoundRules      []compoundRule
 	MaxContentScanSize int
 	PublicSuffixes     []string
 

go.mod

@@ -1,30 +1,38 @@
 module github.com/andybalholm/redwood
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/andybalholm/brotli v1.0.5
 	github.com/andybalholm/cascadia v1.3.1
 	github.com/andybalholm/dhash v1.0.0
-	github.com/andybalholm/go-bit v1.0.1 // indirect
 	github.com/baruwa-enterprise/clamd v1.0.1
-	github.com/chzyer/readline v1.5.1 // indirect
-	github.com/dlclark/regexp2 v1.8.0 // indirect
 	github.com/dop251/goja v0.0.0-20230216180835-5937a312edda
 	github.com/dop251/goja_nodejs v0.0.0-20230207183254-2229640ea097
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
 	github.com/klauspost/compress v1.15.15
 	github.com/miekg/dns v1.1.50
 	github.com/open-ch/ja3 v1.0.1
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/qri-io/starlib v0.5.0
 	github.com/remogatto/ftpget v0.0.0-20120222025949-5c3c8286a3b0
 	go.starlark.net v0.0.0-20230128213706-3f75dec8e403
 	golang.org/x/crypto v0.6.0
 	golang.org/x/image v0.5.0
 	golang.org/x/net v0.7.0
 	golang.org/x/text v0.7.0
-	golang.org/x/tools v0.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-go 1.15
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/go-bit v1.0.1 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
+	github.com/dlclark/regexp2 v1.8.0 // indirect
+	github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
+go 1.18