Refactor and split out things into cmd and lib (#77)

* Refactor anubis to split business logic into a lib, and cmd to just be direct usage.

* Post-rebase fixes.

* Update changelog, remove unnecessary one.

* lib: refactor this

This is mostly based on my personal preferences for how Go code should
be laid out. I'm not sold on the package name "lib" (I'd call it anubis
but that would stutter), but people are probably gonna import it as
libanubis so it's likely fine.

Packages have been "flattened" to centralize implementation with area of
concern. This goes against the Java-esque style that many people like,
but I think this helps make things simple.

Most notably: the dnsbl client (which is a hack) is an internal package
until it's made more generic. Then it can be made external.

I also fixed the logic such that `go generate` works and rebased on
main.

* internal/test: run tests iff npx exists and DONT_USE_NETWORK is not set

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: install deps

Signed-off-by: Xe Iaso <me@xeiaso.net>

* .github/workflows: verbose go tests?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: sleep 2

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: nix this test so CI works

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: warmup per browser?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* internal/test: disable for now :(

Signed-off-by: Xe Iaso <me@xeiaso.net>

* lib/anubis: do not apply bot rules if address check fails

Closes #83

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Xe Iaso <me@xeiaso.net>

.github/workflows/go.yml

@@ -56,6 +56,14 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-golang-
 
+    - name: Cache playwright binaries
+      uses: actions/cache@v3
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/go.sum') }}
+
     - name: Build
       run: go build ./...
 

anubis.go

@@ -0,0 +1,19 @@
+// Package Anubis contains the version number of Anubis.
+package anubis
+
+// Version is the current version of Anubis.
+//
+// This variable is set at build time using the -X linker flag. If not set,
+// it defaults to "devel".
+var Version = "devel"
+
+// CookieName is the name of the cookie that Anubis uses in order to validate
+// access.
+const CookieName = "within.website-x-cmd-anubis-auth"
+
+// StaticPath is the location where all static Anubis assets are located.
+const StaticPath = "/.within.website/x/cmd/anubis/"
+
+// DefaultDifficulty is the default "difficulty" (number of leading zeroes)
+// that must be met by the client in order to pass the challenge.
+const DefaultDifficulty = 4

cmd/anubis/CHANGELOG.md

@@ -1,5 +0,0 @@
-# CHANGELOG
-
-## 2025-01-24
-
-- Added support for custom bot policy documentation, allowing administrators to change how Anubis works to meet their needs.

cmd/anubis/main.go

@@ -2,20 +2,10 @@ package main
 
 import (
 	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/subtle"
-	"embed"
-	"encoding/hex"
-	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"log/slog"
-	"math"
-	mrand "math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -29,22 +19,18 @@ import (
 	"time"
 
 	"github.com/TecharoHQ/anubis"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/config"
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/dnsbl"
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/xess"
-	"github.com/a-h/templ"
+	libanubis "github.com/TecharoHQ/anubis/lib"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/web"
 	"github.com/facebookgo/flagenv"
-	"github.com/golang-jwt/jwt/v5"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
 	bind                = flag.String("bind", ":8923", "network address to bind HTTP to")
 	bindNetwork         = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
-	challengeDifficulty = flag.Int("difficulty", defaultDifficulty, "difficulty of the challenge")
+	challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
 	metricsBind         = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
 	metricsBindNetwork  = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
 	socketMode          = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
@@ -54,49 +40,8 @@ var (
 	target              = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
 	healthcheck         = flag.Bool("healthcheck", false, "run a health check against Anubis")
 	debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally")
-
-	//go:embed static botPolicies.json
-	static embed.FS
-
-	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_issued",
-		Help: "The total number of challenges issued",
-	})
-
-	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_challenges_validated",
-		Help: "The total number of challenges validated",
-	})
-
-	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_dronebl_hits",
-		Help: "The total number of hits from DroneBL",
-	}, []string{"status"})
-
-	failedValidations = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "anubis_failed_validations",
-		Help: "The total number of failed validations",
-	})
-
-	timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{
-		Name:    "anubis_time_taken",
-		Help:    "The time taken for a browser to generate a response (milliseconds)",
-		Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19),
-	})
 )
 
-const (
-	cookieName        = "within.website-x-cmd-anubis-auth"
-	staticPath        = "/.within.website/x/cmd/anubis/"
-	defaultDifficulty = 4
-)
-
-//go:generate go tool github.com/a-h/templ/cmd/templ generate
-//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
-//go:generate gzip -f -k static/js/main.mjs
-//go:generate zstd -f -k --ultra -22 static/js/main.mjs
-//go:generate brotli -fZk static/js/main.mjs
-
 func doHealthCheck() error {
 	resp, err := http.Get("http://localhost" + *metricsBind + "/metrics")
 	if err != nil {
@@ -145,6 +90,34 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
+func makeReverseProxy(target string) (http.Handler, error) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse target URL: %w", err)
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
+	if u.Scheme == "unix" {
+		// clean path up so we don't use the socket path in proxied requests
+		addr := u.Path
+		u.Path = ""
+		// tell transport how to dial unix sockets
+		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			dialer := net.Dialer{}
+			return dialer.DialContext(ctx, "unix", addr)
+		}
+		// tell transport how to handle the unix url scheme
+		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
+	}
+
+	rp := httputil.NewSingleHostReverseProxy(u)
+	rp.Transport = transport
+
+	return rp, nil
+}
+
 func main() {
 	flagenv.Parse()
 	flag.Parse()
@@ -158,13 +131,18 @@ func main() {
 		return
 	}
 
-	s, err := New(*target, *policyFname)
+	rp, err := makeReverseProxy(*target)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("can't make reverse proxy: %v", err)
+	}
+
+	policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty)
+	if err != nil {
+		log.Fatalf("can't parse policy file: %v", err)
 	}
 
 	fmt.Println("Rule error IDs:")
-	for _, rule := range s.policy.Bots {
+	for _, rule := range policy.Bots {
 		if rule.Action != config.RuleDeny {
 			continue
 		}
@@ -178,25 +156,13 @@ func main() {
 	}
 	fmt.Println()
 
-	mux := http.NewServeMux()
-	xess.Mount(mux)
-
-	mux.Handle(staticPath, internal.UnchangingCache(http.StripPrefix(staticPath, http.FileServerFS(static))))
-
-	// mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding)
-
-	mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", s.makeChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", s.passChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", s.testError)
-
-	if *robotsTxt {
-		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
-
-		mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, static, "static/robots.txt")
-		})
+	s, err := libanubis.New(libanubis.Options{
+		Next:           rp,
+		Policy:         policy,
+		ServeRobotsTXT: *robotsTxt,
+	})
+	if err != nil {
+		log.Fatalf("can't construct libanubis.Server: %v", err)
 	}
 
 	wg := new(sync.WaitGroup)
@@ -209,10 +175,8 @@ func main() {
 		go metricsServer(ctx, wg.Done)
 	}
 
-	mux.HandleFunc("/", s.maybeReverseProxy)
-
 	var h http.Handler
-	h = mux
+	h = s
 	h = internal.DefaultXRealIP(*debugXRealIPDefault, h)
 	h = internal.XForwardedForToXRealIP(h)
 
@@ -267,428 +231,6 @@ func metricsServer(ctx context.Context, done func()) {
 	}
 }
 
-func sha256sum(text string) string {
-	hash := sha256.New()
-	hash.Write([]byte(text))
-	return hex.EncodeToString(hash.Sum(nil))
-}
-
-func (s *Server) challengeFor(r *http.Request, difficulty int) string {
-	fp := sha256.Sum256(s.priv.Seed())
-
-	data := fmt.Sprintf(
-		"Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
-		r.Header.Get("Accept-Language"),
-		r.Header.Get("X-Real-Ip"),
-		r.UserAgent(),
-		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
-		fp,
-		difficulty,
-	)
-	return sha256sum(data)
-}
-
-func New(target, policyFname string) (*Server, error) {
-	u, err := url.Parse(target)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse target URL: %w", err)
-	}
-
-	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate ed25519 key: %w", err)
-	}
-
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-
-	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
-	if u.Scheme == "unix" {
-		// clean path up so we don't use the socket path in proxied requests
-		addr := u.Path
-		u.Path = ""
-		// tell transport how to dial unix sockets
-		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			dialer := net.Dialer{}
-			return dialer.DialContext(ctx, "unix", addr)
-		}
-		// tell transport how to handle the unix url scheme
-		transport.RegisterProtocol("unix", unixRoundTripper{Transport: transport})
-	}
-
-	rp := httputil.NewSingleHostReverseProxy(u)
-	rp.Transport = transport
-
-	var fin io.ReadCloser
-
-	if policyFname != "" {
-		fin, err = os.Open(policyFname)
-		if err != nil {
-			return nil, fmt.Errorf("can't parse policy file %s: %w", policyFname, err)
-		}
-	} else {
-		policyFname = "(static)/botPolicies.json"
-		fin, err = static.Open("botPolicies.json")
-		if err != nil {
-			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", policyFname, err)
-		}
-	}
-
-	defer fin.Close()
-
-	policy, err := parseConfig(fin, policyFname, *challengeDifficulty)
-	if err != nil {
-		return nil, err // parseConfig sets a fancy error for us
-	}
-
-	return &Server{
-		rp:         rp,
-		priv:       priv,
-		pub:        pub,
-		policy:     policy,
-		dnsblCache: NewDecayMap[string, dnsbl.DroneBLResponse](),
-	}, nil
-}
-
-// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124
-type unixRoundTripper struct {
-	Transport *http.Transport
-}
-
-// set bare minimum stuff
-func (t unixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req = req.Clone(req.Context())
-	if req.Host == "" {
-		req.Host = "localhost"
-	}
-	req.URL.Host = req.Host // proxy error: no Host in request URL
-	req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion
-	return t.Transport.RoundTrip(req)
-}
-
-type Server struct {
-	rp         *httputil.ReverseProxy
-	priv       ed25519.PrivateKey
-	pub        ed25519.PublicKey
-	policy     *ParsedConfig
-	dnsblCache *DecayMap[string, dnsbl.DroneBLResponse]
-}
-
-func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for",
-		r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	r.Header.Add("X-Anubis-Rule", cr.Name)
-	r.Header.Add("X-Anubis-Action", string(cr.Rule))
-	lg = lg.With("check_result", cr)
-	policyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1)
-
-	ip := r.Header.Get("X-Real-Ip")
-
-	if s.policy.DNSBL && ip != "" {
-		resp, ok := s.dnsblCache.Get(ip)
-		if !ok {
-			lg.Debug("looking up ip in dnsbl")
-			resp, err := dnsbl.Lookup(ip)
-			if err != nil {
-				lg.Error("can't look up ip in dnsbl", "err", err)
-			}
-			s.dnsblCache.Set(ip, resp, 24*time.Hour)
-			droneBLHits.WithLabelValues(resp.String()).Inc()
-		}
-
-		if resp != dnsbl.AllGood {
-			lg.Info("DNSBL hit", "status", resp.String())
-			templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-			return
-		}
-	}
-
-	switch cr.Rule {
-	case config.RuleAllow:
-		lg.Debug("allowing traffic to origin (explicit)")
-		s.rp.ServeHTTP(w, r)
-		return
-	case config.RuleDeny:
-		clearCookie(w)
-		lg.Info("explicit deny")
-		if rule == nil {
-			lg.Error("rule is nil, cannot calculate checksum")
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-		hash, err := rule.Hash()
-		if err != nil {
-			lg.Error("can't calculate checksum of rule", "err", err)
-			templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-			return
-		}
-		lg.Debug("rule hash", "hash", hash)
-		templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
-		return
-	case config.RuleChallenge:
-		lg.Debug("challenge requested")
-	default:
-		clearCookie(w)
-		templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	ckie, err := r.Cookie(cookieName)
-	if err != nil {
-		lg.Debug("cookie not found", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if err := ckie.Valid(); err != nil {
-		lg.Debug("cookie is invalid", "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() {
-		lg.Debug("cookie expired", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) {
-		return s.pub, nil
-	}, jwt.WithExpirationRequired(), jwt.WithStrictDecoding())
-
-	if err != nil || !token.Valid {
-		lg.Debug("invalid token", "path", r.URL.Path, "err", err)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	if randomJitter() {
-		r.Header.Add("X-Anubis-Status", "PASS-BRIEF")
-		lg.Debug("cookie is not enrolled into secondary screening")
-		s.rp.ServeHTTP(w, r)
-		return
-	}
-
-	claims, ok := token.Claims.(jwt.MapClaims)
-	if !ok {
-		lg.Debug("invalid token claims type", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	if claims["challenge"] != challenge {
-		lg.Debug("invalid challenge", "path", r.URL.Path)
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	var nonce int
-
-	if v, ok := claims["nonce"].(float64); ok {
-		nonce = int(v)
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := sha256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 {
-		lg.Debug("invalid response", "path", r.URL.Path)
-		failedValidations.Inc()
-		clearCookie(w)
-		s.renderIndex(w, r)
-		return
-	}
-
-	slog.Debug("all checks passed")
-	r.Header.Add("X-Anubis-Status", "PASS-FULL")
-	s.rp.ServeHTTP(w, r)
-}
-
-func (s *Server) renderIndex(w http.ResponseWriter, r *http.Request) {
-	templ.Handler(
-		base("Making sure you're not a bot!", index()),
-	).ServeHTTP(w, r)
-}
-
-func (s *Server) makeChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(struct {
-			Error string `json:"error"`
-		}{
-			Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"",
-		})
-		return
-	}
-	lg = lg.With("check_result", cr)
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	json.NewEncoder(w).Encode(struct {
-		Challenge string                 `json:"challenge"`
-		Rules     *config.ChallengeRules `json:"rules"`
-	}{
-		Challenge: challenge,
-		Rules:     rule.Challenge,
-	})
-	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
-	challengesIssued.Inc()
-}
-
-func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) {
-	lg := slog.With(
-		"user_agent", r.UserAgent(),
-		"accept_language", r.Header.Get("Accept-Language"),
-		"priority", r.Header.Get("Priority"),
-		"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
-		"x-real-ip", r.Header.Get("X-Real-Ip"),
-	)
-
-	cr, rule, err := s.check(r)
-	if err != nil {
-		lg.Error("check failed", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-	lg = lg.With("check_result", cr)
-
-	nonceStr := r.FormValue("nonce")
-	if nonceStr == "" {
-		clearCookie(w)
-		lg.Debug("no nonce")
-		templ.Handler(base("Oh noes!", errorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	elapsedTimeStr := r.FormValue("elapsedTime")
-	if elapsedTimeStr == "" {
-		clearCookie(w)
-		lg.Debug("no elapsedTime")
-		templ.Handler(base("Oh noes!", errorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
-	if err != nil {
-		clearCookie(w)
-		lg.Debug("elapsedTime doesn't parse", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	lg.Info("challenge took", "elapsedTime", elapsedTime)
-	timeTaken.Observe(elapsedTime)
-
-	response := r.FormValue("response")
-	redir := r.FormValue("redir")
-
-	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
-
-	nonce, err := strconv.Atoi(nonceStr)
-	if err != nil {
-		clearCookie(w)
-		lg.Debug("nonce doesn't parse", "err", err)
-		templ.Handler(base("Oh noes!", errorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	calcString := fmt.Sprintf("%s%d", challenge, nonce)
-	calculated := sha256sum(calcString)
-
-	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
-		clearCookie(w)
-		lg.Debug("hash does not match", "got", response, "want", calculated)
-		templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
-		failedValidations.Inc()
-		return
-	}
-
-	// compare the leading zeroes
-	if !strings.HasPrefix(response, strings.Repeat("0", *challengeDifficulty)) {
-		clearCookie(w)
-		lg.Debug("difficulty check failed", "response", response, "difficulty", *challengeDifficulty)
-		templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
-		failedValidations.Inc()
-		return
-	}
-
-	// generate JWT cookie
-	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{
-		"challenge": challenge,
-		"nonce":     nonce,
-		"response":  response,
-		"iat":       time.Now().Unix(),
-		"nbf":       time.Now().Add(-1 * time.Minute).Unix(),
-		"exp":       time.Now().Add(24 * 7 * time.Hour).Unix(),
-	})
-	tokenString, err := token.SignedString(s.priv)
-	if err != nil {
-		lg.Error("failed to sign JWT", "err", err)
-		clearCookie(w)
-		templ.Handler(base("Oh noes!", errorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-		return
-	}
-
-	http.SetCookie(w, &http.Cookie{
-		Name:     cookieName,
-		Value:    tokenString,
-		Expires:  time.Now().Add(24 * 7 * time.Hour),
-		SameSite: http.SameSiteLaxMode,
-		Path:     "/",
-	})
-
-	challengesValidated.Inc()
-	lg.Debug("challenge passed, redirecting to app")
-	http.Redirect(w, r, redir, http.StatusFound)
-}
-
-func (s *Server) testError(w http.ResponseWriter, r *http.Request) {
-	err := r.FormValue("err")
-	templ.Handler(base("Oh noes!", errorPage(err)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-}
-
-func ohNoes(w http.ResponseWriter, r *http.Request, err error) {
-	slog.Error("super fatal error", "err", err)
-	templ.Handler(base("Oh noes!", errorPage("An internal server error happened")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-}
-
-func clearCookie(w http.ResponseWriter) {
-	http.SetCookie(w, &http.Cookie{
-		Name:     cookieName,
-		Value:    "",
-		Expires:  time.Now().Add(-1 * time.Hour),
-		MaxAge:   -1,
-		SameSite: http.SameSiteLaxMode,
-	})
-}
-
-func randomJitter() bool {
-	return mrand.Intn(100) > 10
-}
-
 func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) {
 	priorityList := []string{"zstd", "br", "gzip"}
 	enc2ext := map[string]string{
@@ -701,11 +243,11 @@ func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) {
 		if strings.Contains(r.Header.Get("Accept-Encoding"), enc) {
 			w.Header().Set("Content-Type", "text/javascript")
 			w.Header().Set("Content-Encoding", enc)
-			http.ServeFileFS(w, r, static, "static/js/main.mjs."+enc2ext[enc])
+			http.ServeFileFS(w, r, web.Static, "static/js/main.mjs."+enc2ext[enc])
 			return
 		}
 	}
 
 	w.Header().Set("Content-Type", "text/javascript")
-	http.ServeFileFS(w, r, static, "static/js/main.mjs")
+	http.ServeFileFS(w, r, web.Static, "static/js/main.mjs")
 }

cmd/anubis/policy.go

@@ -1,212 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log"
-	"log/slog"
-	"net"
-	"net/http"
-	"regexp"
-
-	"github.com/TecharoHQ/anubis/cmd/anubis/internal/config"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/yl2chen/cidranger"
-)
-
-var (
-	policyApplications = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "anubis_policy_results",
-		Help: "The results of each policy rule",
-	}, []string{"rule", "action"})
-)
-
-type ParsedConfig struct {
-	orig config.Config
-
-	Bots  []Bot
-	DNSBL bool
-}
-
-type Bot struct {
-	Name      string
-	UserAgent *regexp.Regexp
-	Path      *regexp.Regexp
-	Action    config.Rule `json:"action"`
-	Challenge *config.ChallengeRules
-	Ranger    cidranger.Ranger
-}
-
-func (b Bot) Hash() (string, error) {
-	var pathRex string
-	if b.Path != nil {
-		pathRex = b.Path.String()
-	}
-	var userAgentRex string
-	if b.UserAgent != nil {
-		userAgentRex = b.UserAgent.String()
-	}
-
-	return sha256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil
-}
-
-func parseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
-	var c config.Config
-	if err := json.NewDecoder(fin).Decode(&c); err != nil {
-		return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err)
-	}
-
-	if err := c.Valid(); err != nil {
-		return nil, err
-	}
-
-	var err error
-
-	result := &ParsedConfig{
-		orig: c,
-	}
-
-	for _, b := range c.Bots {
-		if berr := b.Valid(); berr != nil {
-			err = errors.Join(err, berr)
-			continue
-		}
-
-		var botParseErr error
-		parsedBot := Bot{
-			Name:   b.Name,
-			Action: b.Action,
-		}
-
-		if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 {
-			parsedBot.Ranger = cidranger.NewPCTrieRanger()
-
-			for _, cidr := range b.RemoteAddr {
-				_, rng, err := net.ParseCIDR(cidr)
-				if err != nil {
-					return nil, fmt.Errorf("[unexpected] range %s not parsing: %w", cidr, err)
-				}
-
-				parsedBot.Ranger.Insert(cidranger.NewBasicRangerEntry(*rng))
-			}
-		}
-
-		if b.UserAgentRegex != nil {
-			userAgent, err := regexp.Compile(*b.UserAgentRegex)
-			if err != nil {
-				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling user agent regexp: %w", err))
-				continue
-			} else {
-				parsedBot.UserAgent = userAgent
-			}
-		}
-
-		if b.PathRegex != nil {
-			path, err := regexp.Compile(*b.PathRegex)
-			if err != nil {
-				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling path regexp: %w", err))
-				continue
-			} else {
-				parsedBot.Path = path
-			}
-		}
-
-		if b.Challenge == nil {
-			parsedBot.Challenge = &config.ChallengeRules{
-				Difficulty: defaultDifficulty,
-				ReportAs:   defaultDifficulty,
-				Algorithm:  config.AlgorithmFast,
-			}
-		} else {
-			parsedBot.Challenge = b.Challenge
-			if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown {
-				parsedBot.Challenge.Algorithm = config.AlgorithmFast
-			}
-		}
-
-		result.Bots = append(result.Bots, parsedBot)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, err)
-	}
-
-	result.DNSBL = c.DNSBL
-
-	return result, nil
-}
-
-type CheckResult struct {
-	Name string
-	Rule config.Rule
-}
-
-func (cr CheckResult) LogValue() slog.Value {
-	return slog.GroupValue(
-		slog.String("name", cr.Name),
-		slog.String("rule", string(cr.Rule)))
-}
-
-func cr(name string, rule config.Rule) CheckResult {
-	return CheckResult{
-		Name: name,
-		Rule: rule,
-	}
-}
-
-func (s *Server) checkRemoteAddress(b Bot, addr net.IP) bool {
-	if b.Ranger == nil {
-		return false
-	}
-
-	ok, err := b.Ranger.Contains(addr)
-	if err != nil {
-		log.Panicf("[unexpected] something very funky is going on, %q does not have a calculable network number: %v", addr.String(), err)
-	}
-
-	return ok
-}
-
-// Check evaluates the list of rules, and returns the result
-func (s *Server) check(r *http.Request) (CheckResult, *Bot, error) {
-	host := r.Header.Get("X-Real-Ip")
-	if host == "" {
-		return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] X-Real-Ip header is not set")
-	}
-
-	addr := net.ParseIP(host)
-	if addr == nil {
-		return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host)
-	}
-
-	for _, b := range s.policy.Bots {
-		if b.UserAgent != nil {
-			if uaMatch := b.UserAgent.MatchString(r.UserAgent()); uaMatch || (uaMatch && s.checkRemoteAddress(b, addr)) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-
-		if b.Path != nil {
-			if pathMatch := b.Path.MatchString(r.URL.Path); pathMatch || (pathMatch && s.checkRemoteAddress(b, addr)) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-
-		if b.Ranger != nil {
-			if s.checkRemoteAddress(b, addr) {
-				return cr("bot/"+b.Name, b.Action), &b, nil
-			}
-		}
-	}
-
-	return cr("default/allow", config.RuleAllow), &Bot{
-		Challenge: &config.ChallengeRules{
-			Difficulty: defaultDifficulty,
-			ReportAs:   defaultDifficulty,
-			Algorithm:  config.AlgorithmFast,
-		},
-	}, nil
-}

data/embed.go

@@ -0,0 +1,8 @@
+package data
+
+import "embed"
+
+var (
+	//go:embed botPolicies.json
+	BotPolicies embed.FS
+)

decaymap/decaymap.go

@@ -1,17 +1,17 @@
-package main
+package decaymap
 
 import (
 	"sync"
 	"time"
 )
 
-func zilch[T any]() T {
+func Zilch[T any]() T {
 	var zero T
 	return zero
 }
 
-// DecayMap is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time.
-type DecayMap[K comparable, V any] struct {
+// Impl is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time.
+type Impl[K comparable, V any] struct {
 	data map[K]decayMapEntry[V]
 	lock sync.RWMutex
 }
@@ -21,17 +21,17 @@ type decayMapEntry[V any] struct {
 	expiry time.Time
 }
 
-// NewDecayMap creates a new DecayMap of key type K and value type V.
+// New creates a new DecayMap of key type K and value type V.
 //
 // Key types must be comparable to work with maps.
-func NewDecayMap[K comparable, V any]() *DecayMap[K, V] {
-	return &DecayMap[K, V]{
+func New[K comparable, V any]() *Impl[K, V] {
+	return &Impl[K, V]{
 		data: make(map[K]decayMapEntry[V]),
 	}
 }
 
 // expire forcibly expires a key by setting its time-to-live one second in the past.
-func (m *DecayMap[K, V]) expire(key K) bool {
+func (m *Impl[K, V]) expire(key K) bool {
 	m.lock.RLock()
 	val, ok := m.data[key]
 	m.lock.RUnlock()
@@ -51,13 +51,13 @@ func (m *DecayMap[K, V]) expire(key K) bool {
 // Get gets a value from the DecayMap by key.
 //
 // If a value has expired, forcibly delete it if it was not updated.
-func (m *DecayMap[K, V]) Get(key K) (V, bool) {
+func (m *Impl[K, V]) Get(key K) (V, bool) {
 	m.lock.RLock()
 	value, ok := m.data[key]
 	m.lock.RUnlock()
 
 	if !ok {
-		return zilch[V](), false
+		return Zilch[V](), false
 	}
 
 	if time.Now().After(value.expiry) {
@@ -69,14 +69,14 @@ func (m *DecayMap[K, V]) Get(key K) (V, bool) {
 		}
 		m.lock.Unlock()
 
-		return zilch[V](), false
+		return Zilch[V](), false
 	}
 
 	return value.Value, true
 }
 
 // Set sets a key value pair in the map.
-func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) {
+func (m *Impl[K, V]) Set(key K, value V, ttl time.Duration) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 

decaymap/decaymap_test.go

@@ -1,12 +1,12 @@
-package main
+package decaymap
 
 import (
 	"testing"
 	"time"
 )
 
-func TestDecayMap(t *testing.T) {
-	dm := NewDecayMap[string, string]()
+func TestImpl(t *testing.T) {
+	dm := New[string, string]()
 
 	dm.Set("test", "hi", 5*time.Minute)
 

doc.go

@@ -1,8 +0,0 @@
-// Package Anubis contains the version number of Anubis.
-package anubis
-
-// Version is the current version of Anubis.
-//
-// This variable is set at build time using the -X linker flag. If not set,
-// it defaults to "devel".
-var Version = "devel"

docs/docs/CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Fixed and clarified installation instructions
 - Introduced integration tests using Playwright
+- Refactor & Split up Anubis into cmd and lib.go
+- Fixed bot check to only apply if address range matches
 
 ## v1.14.2
 

internal/hash.go

@@ -0,0 +1,12 @@
+package internal
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+)
+
+func SHA256sum(text string) string {
+	hash := sha256.New()
+	hash.Write([]byte(text))
+	return hex.EncodeToString(hash.Sum(nil))
+}

internal/test/playwright_test.go

@@ -1,5 +1,3 @@
-//go:build integration
-
 // Integration tests for Anubis, using Playwright.
 //
 // These tests require an already running Anubis and Playwright server.
@@ -16,31 +14,60 @@
 package test
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/exec"
 	"testing"
 	"time"
 
+	"github.com/TecharoHQ/anubis"
+	libanubis "github.com/TecharoHQ/anubis/lib"
 	"github.com/playwright-community/playwright-go"
 )
 
 var (
-	anubisServer          = flag.String("anubis", "http://localhost:8923", "Anubis server URL")
 	serverBindAddr        = flag.String("bind", "localhost:3923", "test server bind address")
+	playwrightPort        = flag.Int("playwright-port", 3000, "Playwright port")
 	playwrightServer      = flag.String("playwright", "ws://localhost:3000", "Playwright server URL")
 	playwrightMaxTime     = flag.Duration("playwright-max-time", 5*time.Second, "maximum time for Playwright requests")
 	playwrightMaxHardTime = flag.Duration("playwright-max-hard-time", 5*time.Minute, "maximum time for hard Playwright requests")
 
 	testCases = []testCase{
-		{name: "firefox", action: actionChallenge, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"},
-		{name: "headlessChrome", action: actionDeny, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36"},
-		{name: "kagiBadIP", action: actionChallenge, isHard: true, realIP: placeholderIP, userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"},
-		{name: "kagiGoodIP", action: actionAllow, realIP: "216.18.205.234", userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"},
-		{name: "unknownAgent", action: actionAllow, realIP: placeholderIP, userAgent: "AnubisTest/0"},
+		{
+			name:      "firefox",
+			action:    actionChallenge,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0",
+		},
+		{
+			name:      "headlessChrome",
+			action:    actionDeny,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36",
+		},
+		{
+			name:      "kagiBadIP",
+			action:    actionChallenge,
+			isHard:    true,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
+		},
+		{
+			name:      "kagiGoodIP",
+			action:    actionAllow,
+			realIP:    "216.18.205.234",
+			userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
+		},
+		{
+			name:      "unknownAgent",
+			action:    actionAllow,
+			realIP:    placeholderIP,
+			userAgent: "AnubisTest/0",
+		},
 	}
 )
 
@@ -49,7 +76,8 @@ const (
 	actionDeny      action = "DENY"
 	actionChallenge action = "CHALLENGE"
 
-	placeholderIP = "fd11:5ee:bad:c0de::"
+	placeholderIP     = "fd11:5ee:bad:c0de::"
+	playwrightVersion = "1.50.1"
 )
 
 type action string
@@ -61,21 +89,143 @@ type testCase struct {
 	realIP, userAgent string
 }
 
+func doesNPXExist(t *testing.T) {
+	t.Helper()
+
+	if _, err := exec.LookPath("npx"); err != nil {
+		t.Skipf("npx not found in PATH, skipping integration smoke testing: %v", err)
+	}
+}
+
+func run(t *testing.T, command string) string {
+	t.Helper()
+
+	shPath, err := exec.LookPath("sh")
+	if err != nil {
+		t.Fatalf("[unexpected] %v", err)
+	}
+
+	t.Logf("running command: %s", command)
+
+	cmd := exec.Command(shPath, "-c", command)
+	cmd.Stdin = nil
+	cmd.Stderr = os.Stderr
+	output, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("can't run command: %v", err)
+	}
+
+	return string(output)
+}
+
+func daemonize(t *testing.T, command string) {
+	t.Helper()
+
+	shPath, err := exec.LookPath("sh")
+	if err != nil {
+		t.Fatalf("[unexpected] %v", err)
+	}
+
+	t.Logf("daemonizing command: %s", command)
+
+	cmd := exec.Command(shPath, "-c", command)
+	cmd.Stdin = nil
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("can't daemonize command: %v", err)
+	}
+
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+	})
+}
+
+func startPlaywright(t *testing.T) {
+	t.Helper()
+
+	if os.Getenv("CI") == "true" {
+		run(t, fmt.Sprintf("npx --yes playwright@%s install --with-deps", playwrightVersion))
+	} else {
+		run(t, fmt.Sprintf("npx --yes playwright@%s install", playwrightVersion))
+	}
+
+	daemonize(t, fmt.Sprintf("npx --yes playwright@%s run-server --port %d", playwrightVersion, *playwrightPort))
+
+	for true {
+		if _, err := http.Get(fmt.Sprintf("http://localhost:%d", *playwrightPort)); err != nil {
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		break
+	}
+
+	//nosleep:bypass XXX(Xe): Playwright doesn't have a good way to signal readiness. This is a HACK that will just let the tests pass.
+	time.Sleep(2 * time.Second)
+}
+
 func TestPlaywrightBrowser(t *testing.T) {
+	if os.Getenv("CI") == "true" {
+		t.Skip("XXX(Xe): This is broken in CI, will fix later")
+	}
+
+	if os.Getenv("DONT_USE_NETWORK") != "" {
+		t.Skip("test requires network egress")
+		return
+	}
+
+	doesNPXExist(t)
+	startPlaywright(t)
+
 	pw := setupPlaywright(t)
-	spawnTestServer(t)
+	anubisURL := spawnAnubis(t)
+
 	browsers := []playwright.BrowserType{pw.Chromium, pw.Firefox, pw.WebKit}
 
 	for _, typ := range browsers {
+		t.Run(typ.Name()+"/warmup", func(t *testing.T) {
+			browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{
+				ExposeNetwork: playwright.String("<loopback>"),
+			})
+			if err != nil {
+				t.Fatalf("could not connect to remote browser: %v", err)
+			}
+			defer browser.Close()
+
+			ctx, err := browser.NewContext(playwright.BrowserNewContextOptions{
+				AcceptDownloads: playwright.Bool(false),
+				ExtraHttpHeaders: map[string]string{
+					"X-Real-Ip": "127.0.0.1",
+				},
+				UserAgent: playwright.String("Sephiroth"),
+			})
+			if err != nil {
+				t.Fatalf("could not create context: %v", err)
+			}
+			defer ctx.Close()
+
+			page, err := ctx.NewPage()
+			if err != nil {
+				t.Fatalf("could not create page: %v", err)
+			}
+			defer page.Close()
+
+			timeout := 2.0
+			page.Goto(anubisURL, playwright.PageGotoOptions{
+				Timeout: &timeout,
+			})
+		})
+
 		for _, tc := range testCases {
-			name := fmt.Sprintf("%s@%s", tc.name, typ.Name())
+			name := fmt.Sprintf("%s/%s", typ.Name(), tc.name)
 			t.Run(name, func(t *testing.T) {
 				_, hasDeadline := t.Deadline()
 				if tc.isHard && hasDeadline {
 					t.Skip("skipping hard challenge with deadline")
 				}
 
-				perfomedAction := executeTestCase(t, tc, typ)
+				perfomedAction := executeTestCase(t, tc, typ, anubisURL)
 
 				if perfomedAction != tc.action {
 					t.Errorf("unexpected test result, expected %s, got %s", tc.action, perfomedAction)
@@ -97,7 +247,7 @@ func buildBrowserConnect(name string) string {
 	return u.String()
 }
 
-func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) action {
+func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType, anubisURL string) action {
 	deadline, _ := t.Deadline()
 
 	browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{
@@ -129,7 +279,7 @@ func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) acti
 	// Attempt challenge.
 
 	start := time.Now()
-	_, err = page.Goto(*anubisServer, playwright.PageGotoOptions{
+	_, err = page.Goto(anubisURL, playwright.PageGotoOptions{
 		Timeout: pwTimeout(tc, deadline),
 	})
 	if err != nil {
@@ -252,25 +402,34 @@ func setupPlaywright(t *testing.T) *playwright.Playwright {
 	return pw
 }
 
-func spawnTestServer(t *testing.T) {
+func spawnAnubis(t *testing.T) string {
 	t.Helper()
 
-	s := new(http.Server)
-	s.Addr = *serverBindAddr
-	s.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Content-Type", "text/html")
 		fmt.Fprintf(w, "<html><body><span id=anubis-test>%d</span></body></html>", time.Now().Unix())
 	})
 
-	go func() {
-		if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			t.Logf("test HTTP server terminated unexpectedly: %v", err)
-		}
-	}()
+	policy, err := libanubis.LoadPoliciesOrDefault("", anubis.DefaultDifficulty)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := libanubis.New(libanubis.Options{
+		Next:           h,
+		Policy:         policy,
+		ServeRobotsTXT: true,
+	})
+	if err != nil {
+		t.Fatalf("can't construct libanubis.Server: %v", err)
+	}
+
+	ts := httptest.NewServer(s)
+	t.Log(ts.URL)
 
 	t.Cleanup(func() {
-		if err := s.Shutdown(context.Background()); err != nil {
-			t.Fatalf("could not shutdown test server: %v", err)
-		}
+		ts.Close()
 	})
+
+	return ts.URL
 }

lib/anubis.go

@@ -0,0 +1,519 @@
+package lib
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"log/slog"
+	"math"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/a-h/templ"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/TecharoHQ/anubis"
+	"github.com/TecharoHQ/anubis/data"
+	"github.com/TecharoHQ/anubis/decaymap"
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/internal/dnsbl"
+	"github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/TecharoHQ/anubis/web"
+	"github.com/TecharoHQ/anubis/xess"
+)
+
+var (
+	challengesIssued = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "anubis_challenges_issued",
+		Help: "The total number of challenges issued",
+	})
+
+	challengesValidated = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "anubis_challenges_validated",
+		Help: "The total number of challenges validated",
+	})
+
+	droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "anubis_dronebl_hits",
+		Help: "The total number of hits from DroneBL",
+	}, []string{"status"})
+
+	failedValidations = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "anubis_failed_validations",
+		Help: "The total number of failed validations",
+	})
+
+	timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "anubis_time_taken",
+		Help:    "The time taken for a browser to generate a response (milliseconds)",
+		Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19),
+	})
+)
+
+type Options struct {
+	Next           http.Handler
+	Policy         *policy.ParsedConfig
+	ServeRobotsTXT bool
+}
+
+func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
+	var fin io.ReadCloser
+	var err error
+
+	if fname != "" {
+		fin, err = os.Open(fname)
+		if err != nil {
+			return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
+		}
+	} else {
+		fname = "(data)/botPolicies.json"
+		fin, err = data.BotPolicies.Open("botPolicies.json")
+		if err != nil {
+			return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err)
+		}
+	}
+
+	defer fin.Close()
+
+	policy, err := policy.ParseConfig(fin, fname, defaultDifficulty)
+
+	return policy, err
+}
+
+func New(opts Options) (*Server, error) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate ed25519 key: %w", err)
+	}
+
+	if err != nil {
+		return nil, err // parseConfig sets a fancy error for us
+	}
+
+	result := &Server{
+		next:       opts.Next,
+		priv:       priv,
+		pub:        pub,
+		policy:     opts.Policy,
+		DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
+	}
+
+	mux := http.NewServeMux()
+	xess.Mount(mux)
+
+	mux.Handle(anubis.StaticPath, internal.UnchangingCache(http.StripPrefix(anubis.StaticPath, http.FileServerFS(web.Static))))
+
+	if opts.ServeRobotsTXT {
+		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
+		})
+
+		mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
+		})
+	}
+
+	// mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding)
+
+	mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", result.MakeChallenge)
+	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", result.PassChallenge)
+	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", result.TestError)
+
+	mux.HandleFunc("/", result.MaybeReverseProxy)
+
+	result.mux = mux
+
+	return result, nil
+}
+
+type Server struct {
+	mux                 *http.ServeMux
+	next                http.Handler
+	priv                ed25519.PrivateKey
+	pub                 ed25519.PublicKey
+	policy              *policy.ParsedConfig
+	DNSBLCache          *decaymap.Impl[string, dnsbl.DroneBLResponse]
+	ChallengeDifficulty int
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mux.ServeHTTP(w, r)
+}
+
+func (s *Server) challengeFor(r *http.Request, difficulty int) string {
+	fp := sha256.Sum256(s.priv.Seed())
+
+	data := fmt.Sprintf(
+		"Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
+		r.Header.Get("Accept-Language"),
+		r.Header.Get("X-Real-Ip"),
+		r.UserAgent(),
+		time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
+		fp,
+		difficulty,
+	)
+	return internal.SHA256sum(data)
+}
+
+func (s *Server) MaybeReverseProxy(w http.ResponseWriter, r *http.Request) {
+	lg := slog.With(
+		"user_agent", r.UserAgent(),
+		"accept_language", r.Header.Get("Accept-Language"),
+		"priority", r.Header.Get("Priority"),
+		"x-forwarded-for",
+		r.Header.Get("X-Forwarded-For"),
+		"x-real-ip", r.Header.Get("X-Real-Ip"),
+	)
+
+	cr, rule, err := s.check(r)
+	if err != nil {
+		lg.Error("check failed", "err", err)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	r.Header.Add("X-Anubis-Rule", cr.Name)
+	r.Header.Add("X-Anubis-Action", string(cr.Rule))
+	lg = lg.With("check_result", cr)
+	policy.PolicyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1)
+
+	ip := r.Header.Get("X-Real-Ip")
+
+	if s.policy.DNSBL && ip != "" {
+		resp, ok := s.DNSBLCache.Get(ip)
+		if !ok {
+			lg.Debug("looking up ip in dnsbl")
+			resp, err := dnsbl.Lookup(ip)
+			if err != nil {
+				lg.Error("can't look up ip in dnsbl", "err", err)
+			}
+			s.DNSBLCache.Set(ip, resp, 24*time.Hour)
+			droneBLHits.WithLabelValues(resp.String()).Inc()
+		}
+
+		if resp != dnsbl.AllGood {
+			lg.Info("DNSBL hit", "status", resp.String())
+			templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
+			return
+		}
+	}
+
+	switch cr.Rule {
+	case config.RuleAllow:
+		lg.Debug("allowing traffic to origin (explicit)")
+		s.next.ServeHTTP(w, r)
+		return
+	case config.RuleDeny:
+		ClearCookie(w)
+		lg.Info("explicit deny")
+		if rule == nil {
+			lg.Error("rule is nil, cannot calculate checksum")
+			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+			return
+		}
+		hash, err := rule.Hash()
+		if err != nil {
+			lg.Error("can't calculate checksum of rule", "err", err)
+			templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+			return
+		}
+		lg.Debug("rule hash", "hash", hash)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r)
+		return
+	case config.RuleChallenge:
+		lg.Debug("challenge requested")
+	default:
+		ClearCookie(w)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	ckie, err := r.Cookie(anubis.CookieName)
+	if err != nil {
+		lg.Debug("cookie not found", "path", r.URL.Path)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	if err := ckie.Valid(); err != nil {
+		lg.Debug("cookie is invalid", "err", err)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() {
+		lg.Debug("cookie expired", "path", r.URL.Path)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) {
+		return s.pub, nil
+	}, jwt.WithExpirationRequired(), jwt.WithStrictDecoding())
+
+	if err != nil || !token.Valid {
+		lg.Debug("invalid token", "path", r.URL.Path, "err", err)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	if randomJitter() {
+		r.Header.Add("X-Anubis-Status", "PASS-BRIEF")
+		lg.Debug("cookie is not enrolled into secondary screening")
+		s.next.ServeHTTP(w, r)
+		return
+	}
+
+	claims, ok := token.Claims.(jwt.MapClaims)
+	if !ok {
+		lg.Debug("invalid token claims type", "path", r.URL.Path)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
+
+	if claims["challenge"] != challenge {
+		lg.Debug("invalid challenge", "path", r.URL.Path)
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	var nonce int
+
+	if v, ok := claims["nonce"].(float64); ok {
+		nonce = int(v)
+	}
+
+	calcString := fmt.Sprintf("%s%d", challenge, nonce)
+	calculated := internal.SHA256sum(calcString)
+
+	if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 {
+		lg.Debug("invalid response", "path", r.URL.Path)
+		failedValidations.Inc()
+		ClearCookie(w)
+		s.RenderIndex(w, r)
+		return
+	}
+
+	slog.Debug("all checks passed")
+	r.Header.Add("X-Anubis-Status", "PASS-FULL")
+	s.next.ServeHTTP(w, r)
+}
+
+func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request) {
+	templ.Handler(
+		web.Base("Making sure you're not a bot!", web.Index()),
+	).ServeHTTP(w, r)
+}
+
+func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
+	lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
+
+	cr, rule, err := s.check(r)
+	if err != nil {
+		lg.Error("check failed", "err", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		json.NewEncoder(w).Encode(struct {
+			Error string `json:"error"`
+		}{
+			Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"",
+		})
+		return
+	}
+	lg = lg.With("check_result", cr)
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
+
+	json.NewEncoder(w).Encode(struct {
+		Challenge string                 `json:"challenge"`
+		Rules     *config.ChallengeRules `json:"rules"`
+	}{
+		Challenge: challenge,
+		Rules:     rule.Challenge,
+	})
+	lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
+	challengesIssued.Inc()
+}
+
+func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
+	lg := slog.With(
+		"user_agent", r.UserAgent(),
+		"accept_language", r.Header.Get("Accept-Language"),
+		"priority", r.Header.Get("Priority"),
+		"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
+		"x-real-ip", r.Header.Get("X-Real-Ip"),
+	)
+
+	cr, rule, err := s.check(r)
+	if err != nil {
+		lg.Error("check failed", "err", err)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+	lg = lg.With("check_result", cr)
+
+	nonceStr := r.FormValue("nonce")
+	if nonceStr == "" {
+		ClearCookie(w)
+		lg.Debug("no nonce")
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	elapsedTimeStr := r.FormValue("elapsedTime")
+	if elapsedTimeStr == "" {
+		ClearCookie(w)
+		lg.Debug("no elapsedTime")
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64)
+	if err != nil {
+		ClearCookie(w)
+		lg.Debug("elapsedTime doesn't parse", "err", err)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	lg.Info("challenge took", "elapsedTime", elapsedTime)
+	timeTaken.Observe(elapsedTime)
+
+	response := r.FormValue("response")
+	redir := r.FormValue("redir")
+
+	challenge := s.challengeFor(r, rule.Challenge.Difficulty)
+
+	nonce, err := strconv.Atoi(nonceStr)
+	if err != nil {
+		ClearCookie(w)
+		lg.Debug("nonce doesn't parse", "err", err)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	calcString := fmt.Sprintf("%s%d", challenge, nonce)
+	calculated := internal.SHA256sum(calcString)
+
+	if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 {
+		ClearCookie(w)
+		lg.Debug("hash does not match", "got", response, "want", calculated)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
+		failedValidations.Inc()
+		return
+	}
+
+	// compare the leading zeroes
+	if !strings.HasPrefix(response, strings.Repeat("0", s.ChallengeDifficulty)) {
+		ClearCookie(w)
+		lg.Debug("difficulty check failed", "response", response, "difficulty", s.ChallengeDifficulty)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r)
+		failedValidations.Inc()
+		return
+	}
+
+	// generate JWT cookie
+	token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{
+		"challenge": challenge,
+		"nonce":     nonce,
+		"response":  response,
+		"iat":       time.Now().Unix(),
+		"nbf":       time.Now().Add(-1 * time.Minute).Unix(),
+		"exp":       time.Now().Add(24 * 7 * time.Hour).Unix(),
+	})
+	tokenString, err := token.SignedString(s.priv)
+	if err != nil {
+		lg.Error("failed to sign JWT", "err", err)
+		ClearCookie(w)
+		templ.Handler(web.Base("Oh noes!", web.ErrorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+		return
+	}
+
+	http.SetCookie(w, &http.Cookie{
+		Name:     anubis.CookieName,
+		Value:    tokenString,
+		Expires:  time.Now().Add(24 * 7 * time.Hour),
+		SameSite: http.SameSiteLaxMode,
+		Path:     "/",
+	})
+
+	challengesValidated.Inc()
+	lg.Debug("challenge passed, redirecting to app")
+	http.Redirect(w, r, redir, http.StatusFound)
+}
+
+func (s *Server) TestError(w http.ResponseWriter, r *http.Request) {
+	err := r.FormValue("err")
+	templ.Handler(web.Base("Oh noes!", web.ErrorPage(err)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
+}
+
+// Check evaluates the list of rules, and returns the result
+func (s *Server) check(r *http.Request) (CheckResult, *policy.Bot, error) {
+	host := r.Header.Get("X-Real-Ip")
+	if host == "" {
+		return decaymap.Zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] X-Real-Ip header is not set")
+	}
+
+	addr := net.ParseIP(host)
+	if addr == nil {
+		return decaymap.Zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host)
+	}
+
+	for _, b := range s.policy.Bots {
+		if b.UserAgent != nil {
+			if b.UserAgent.MatchString(r.UserAgent()) && s.checkRemoteAddress(b, addr) {
+				return cr("bot/"+b.Name, b.Action), &b, nil
+			}
+		}
+
+		if b.Path != nil {
+			if b.Path.MatchString(r.URL.Path) && s.checkRemoteAddress(b, addr) {
+				return cr("bot/"+b.Name, b.Action), &b, nil
+			}
+		}
+
+		if b.Ranger != nil {
+			if s.checkRemoteAddress(b, addr) {
+				return cr("bot/"+b.Name, b.Action), &b, nil
+			}
+		}
+	}
+
+	return cr("default/allow", config.RuleAllow), &policy.Bot{
+		Challenge: &config.ChallengeRules{
+			Difficulty: anubis.DefaultDifficulty,
+			ReportAs:   anubis.DefaultDifficulty,
+			Algorithm:  config.AlgorithmFast,
+		},
+	}, nil
+}
+
+func (s *Server) checkRemoteAddress(b policy.Bot, addr net.IP) bool {
+	if b.Ranger == nil {
+		return true
+	}
+
+	ok, err := b.Ranger.Contains(addr)
+	if err != nil {
+		log.Panicf("[unexpected] something very funky is going on, %q does not have a calculable network number: %v", addr.String(), err)
+	}
+
+	return ok
+}

lib/checkresult.go

@@ -0,0 +1,25 @@
+package lib
+
+import (
+	"log/slog"
+
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+)
+
+type CheckResult struct {
+	Name string
+	Rule config.Rule
+}
+
+func (cr CheckResult) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.String("name", cr.Name),
+		slog.String("rule", string(cr.Rule)))
+}
+
+func cr(name string, rule config.Rule) CheckResult {
+	return CheckResult{
+		Name: name,
+		Rule: rule,
+	}
+}

lib/http.go

@@ -0,0 +1,34 @@
+package lib
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/TecharoHQ/anubis"
+)
+
+func ClearCookie(w http.ResponseWriter) {
+	http.SetCookie(w, &http.Cookie{
+		Name:     anubis.CookieName,
+		Value:    "",
+		Expires:  time.Now().Add(-1 * time.Hour),
+		MaxAge:   -1,
+		SameSite: http.SameSiteLaxMode,
+	})
+}
+
+// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124
+type UnixRoundTripper struct {
+	Transport *http.Transport
+}
+
+// set bare minimum stuff
+func (t UnixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	if req.Host == "" {
+		req.Host = "localhost"
+	}
+	req.URL.Host = req.Host // proxy error: no Host in request URL
+	req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion
+	return t.Transport.RoundTrip(req)
+}

lib/policy/bot.go

@@ -0,0 +1,32 @@
+package policy
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/TecharoHQ/anubis/internal"
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+	"github.com/yl2chen/cidranger"
+)
+
+type Bot struct {
+	Name      string
+	UserAgent *regexp.Regexp
+	Path      *regexp.Regexp
+	Action    config.Rule `json:"action"`
+	Challenge *config.ChallengeRules
+	Ranger    cidranger.Ranger
+}
+
+func (b Bot) Hash() (string, error) {
+	var pathRex string
+	if b.Path != nil {
+		pathRex = b.Path.String()
+	}
+	var userAgentRex string
+	if b.UserAgent != nil {
+		userAgentRex = b.UserAgent.String()
+	}
+
+	return internal.SHA256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil
+}

lib/policy/config/config.go

@@ -7,6 +7,17 @@ import (
 	"regexp"
 )
 
+var (
+	ErrNoBotRulesDefined                 = errors.New("config: must define at least one (1) bot rule")
+	ErrBotMustHaveName                   = errors.New("config.Bot: must set name")
+	ErrBotMustHaveUserAgentOrPath        = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses")
+	ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both")
+	ErrUnknownAction                     = errors.New("config.Bot: unknown action")
+	ErrInvalidUserAgentRegex             = errors.New("config.Bot: invalid user agent regex")
+	ErrInvalidPathRegex                  = errors.New("config.Bot: invalid path regex")
+	ErrInvalidCIDR                       = errors.New("config.Bot: invalid CIDR")
+)
+
 type Rule string
 
 const (
@@ -24,7 +35,7 @@ const (
 	AlgorithmSlow    Algorithm = "slow"
 )
 
-type Bot struct {
+type BotConfig struct {
 	Name           string          `json:"name"`
 	UserAgentRegex *string         `json:"user_agent_regex"`
 	PathRegex      *string         `json:"path_regex"`
@@ -33,18 +44,7 @@ type Bot struct {
 	Challenge      *ChallengeRules `json:"challenge,omitempty"`
 }
 
-var (
-	ErrNoBotRulesDefined                 = errors.New("config: must define at least one (1) bot rule")
-	ErrBotMustHaveName                   = errors.New("config.Bot: must set name")
-	ErrBotMustHaveUserAgentOrPath        = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses")
-	ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both")
-	ErrUnknownAction                     = errors.New("config.Bot: unknown action")
-	ErrInvalidUserAgentRegex             = errors.New("config.Bot: invalid user agent regex")
-	ErrInvalidPathRegex                  = errors.New("config.Bot: invalid path regex")
-	ErrInvalidCIDR                       = errors.New("config.Bot: invalid CIDR")
-)
-
-func (b Bot) Valid() error {
+func (b BotConfig) Valid() error {
 	var errs []error
 
 	if b.Name == "" {
@@ -137,8 +137,8 @@ func (cr ChallengeRules) Valid() error {
 }
 
 type Config struct {
-	Bots  []Bot `json:"bots"`
-	DNSBL bool  `json:"dnsbl"`
+	Bots  []BotConfig `json:"bots"`
+	DNSBL bool        `json:"dnsbl"`
 }
 
 func (c Config) Valid() error {

lib/policy/config/config_test.go

@@ -13,12 +13,12 @@ func p[V any](v V) *V { return &v }
 func TestBotValid(t *testing.T) {
 	var tests = []struct {
 		name string
-		bot  Bot
+		bot  BotConfig
 		err  error
 	}{
 		{
 			name: "simple user agent",
-			bot: Bot{
+			bot: BotConfig{
 				Name:           "mozilla-ua",
 				Action:         RuleChallenge,
 				UserAgentRegex: p("Mozilla"),
@@ -27,7 +27,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "simple path",
-			bot: Bot{
+			bot: BotConfig{
 				Name:      "well-known-path",
 				Action:    RuleAllow,
 				PathRegex: p("^/.well-known/.*$"),
@@ -36,7 +36,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "no rule name",
-			bot: Bot{
+			bot: BotConfig{
 				Action:         RuleChallenge,
 				UserAgentRegex: p("Mozilla"),
 			},
@@ -44,7 +44,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "no rule matcher",
-			bot: Bot{
+			bot: BotConfig{
 				Name:   "broken-rule",
 				Action: RuleAllow,
 			},
@@ -52,7 +52,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "both user-agent and path",
-			bot: Bot{
+			bot: BotConfig{
 				Name:           "path-and-user-agent",
 				Action:         RuleDeny,
 				UserAgentRegex: p("Mozilla"),
@@ -62,7 +62,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "unknown action",
-			bot: Bot{
+			bot: BotConfig{
 				Name:           "Unknown action",
 				Action:         RuleUnknown,
 				UserAgentRegex: p("Mozilla"),
@@ -71,7 +71,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "invalid user agent regex",
-			bot: Bot{
+			bot: BotConfig{
 				Name:           "mozilla-ua",
 				Action:         RuleChallenge,
 				UserAgentRegex: p("a(b"),
@@ -80,7 +80,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "invalid path regex",
-			bot: Bot{
+			bot: BotConfig{
 				Name:      "mozilla-ua",
 				Action:    RuleChallenge,
 				PathRegex: p("a(b"),
@@ -89,7 +89,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "challenge difficulty too low",
-			bot: Bot{
+			bot: BotConfig{
 				Name:      "mozilla-ua",
 				Action:    RuleChallenge,
 				PathRegex: p("Mozilla"),
@@ -103,7 +103,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "challenge difficulty too high",
-			bot: Bot{
+			bot: BotConfig{
 				Name:      "mozilla-ua",
 				Action:    RuleChallenge,
 				PathRegex: p("Mozilla"),
@@ -117,7 +117,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "challenge wrong algorithm",
-			bot: Bot{
+			bot: BotConfig{
 				Name:      "mozilla-ua",
 				Action:    RuleChallenge,
 				PathRegex: p("Mozilla"),
@@ -131,7 +131,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "invalid cidr range",
-			bot: Bot{
+			bot: BotConfig{
 				Name:       "mozilla-ua",
 				Action:     RuleAllow,
 				RemoteAddr: []string{"0.0.0.0/33"},
@@ -140,7 +140,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "only filter by IP range",
-			bot: Bot{
+			bot: BotConfig{
 				Name:       "mozilla-ua",
 				Action:     RuleAllow,
 				RemoteAddr: []string{"0.0.0.0/0"},
@@ -149,7 +149,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "filter by user agent and IP range",
-			bot: Bot{
+			bot: BotConfig{
 				Name:           "mozilla-ua",
 				Action:         RuleAllow,
 				UserAgentRegex: p("Mozilla"),
@@ -159,7 +159,7 @@ func TestBotValid(t *testing.T) {
 		},
 		{
 			name: "filter by path and IP range",
-			bot: Bot{
+			bot: BotConfig{
 				Name:       "mozilla-ua",
 				Action:     RuleAllow,
 				PathRegex:  p("^.*$"),

lib/policy/policy.go

@@ -0,0 +1,122 @@
+package policy
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/yl2chen/cidranger"
+
+	"github.com/TecharoHQ/anubis/lib/policy/config"
+)
+
+var (
+	PolicyApplications = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "anubis_policy_results",
+		Help: "The results of each policy rule",
+	}, []string{"rule", "action"})
+)
+
+type ParsedConfig struct {
+	orig config.Config
+
+	Bots              []Bot
+	DNSBL             bool
+	DefaultDifficulty int
+}
+
+func NewParsedConfig(orig config.Config) *ParsedConfig {
+	return &ParsedConfig{
+		orig: orig,
+	}
+}
+
+func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
+	var c config.Config
+	if err := json.NewDecoder(fin).Decode(&c); err != nil {
+		return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err)
+	}
+
+	if err := c.Valid(); err != nil {
+		return nil, err
+	}
+
+	var err error
+
+	result := NewParsedConfig(c)
+	result.DefaultDifficulty = defaultDifficulty
+
+	for _, b := range c.Bots {
+		if berr := b.Valid(); berr != nil {
+			err = errors.Join(err, berr)
+			continue
+		}
+
+		var botParseErr error
+		parsedBot := Bot{
+			Name:   b.Name,
+			Action: b.Action,
+		}
+
+		if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 {
+			parsedBot.Ranger = cidranger.NewPCTrieRanger()
+
+			for _, cidr := range b.RemoteAddr {
+				_, rng, err := net.ParseCIDR(cidr)
+				if err != nil {
+					return nil, fmt.Errorf("[unexpected] range %s not parsing: %w", cidr, err)
+				}
+
+				parsedBot.Ranger.Insert(cidranger.NewBasicRangerEntry(*rng))
+			}
+		}
+
+		if b.UserAgentRegex != nil {
+			userAgent, err := regexp.Compile(*b.UserAgentRegex)
+			if err != nil {
+				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling user agent regexp: %w", err))
+				continue
+			} else {
+				parsedBot.UserAgent = userAgent
+			}
+		}
+
+		if b.PathRegex != nil {
+			path, err := regexp.Compile(*b.PathRegex)
+			if err != nil {
+				botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling path regexp: %w", err))
+				continue
+			} else {
+				parsedBot.Path = path
+			}
+		}
+
+		if b.Challenge == nil {
+			parsedBot.Challenge = &config.ChallengeRules{
+				Difficulty: defaultDifficulty,
+				ReportAs:   defaultDifficulty,
+				Algorithm:  config.AlgorithmFast,
+			}
+		} else {
+			parsedBot.Challenge = b.Challenge
+			if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown {
+				parsedBot.Challenge.Algorithm = config.AlgorithmFast
+			}
+		}
+
+		result.Bots = append(result.Bots, parsedBot)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, err)
+	}
+
+	result.DNSBL = c.DNSBL
+
+	return result, nil
+}

lib/policy/policy_test.go

@@ -1,25 +1,28 @@
-package main
+package policy
 
 import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/TecharoHQ/anubis"
+	"github.com/TecharoHQ/anubis/data"
 )
 
 func TestDefaultPolicyMustParse(t *testing.T) {
-	fin, err := static.Open("botPolicies.json")
+	fin, err := data.BotPolicies.Open("botPolicies.json")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer fin.Close()
 
-	if _, err := parseConfig(fin, "botPolicies.json", defaultDifficulty); err != nil {
+	if _, err := ParseConfig(fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil {
 		t.Fatalf("can't parse config: %v", err)
 	}
 }
 
 func TestGoodConfigs(t *testing.T) {
-	finfos, err := os.ReadDir("internal/config/testdata/good")
+	finfos, err := os.ReadDir("config/testdata/good")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -27,13 +30,13 @@ func TestGoodConfigs(t *testing.T) {
 	for _, st := range finfos {
 		st := st
 		t.Run(st.Name(), func(t *testing.T) {
-			fin, err := os.Open(filepath.Join("internal", "config", "testdata", "good", st.Name()))
+			fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
 			if err != nil {
 				t.Fatal(err)
 			}
 			defer fin.Close()
 
-			if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err != nil {
+			if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
 				t.Fatal(err)
 			}
 		})
@@ -41,7 +44,7 @@ func TestGoodConfigs(t *testing.T) {
 }
 
 func TestBadConfigs(t *testing.T) {
-	finfos, err := os.ReadDir("internal/config/testdata/bad")
+	finfos, err := os.ReadDir("config/testdata/bad")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -49,13 +52,13 @@ func TestBadConfigs(t *testing.T) {
 	for _, st := range finfos {
 		st := st
 		t.Run(st.Name(), func(t *testing.T) {
-			fin, err := os.Open(filepath.Join("internal", "config", "testdata", "bad", st.Name()))
+			fin, err := os.Open(filepath.Join("config", "testdata", "bad", st.Name()))
 			if err != nil {
 				t.Fatal(err)
 			}
 			defer fin.Close()
 
-			if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err == nil {
+			if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err == nil {
 				t.Fatal(err)
 			} else {
 				t.Log(err)

lib/random.go

@@ -0,0 +1,9 @@
+package lib
+
+import (
+	"math/rand"
+)
+
+func randomJitter() bool {
+	return rand.Intn(100) > 10
+}

web/embed.go

@@ -0,0 +1,14 @@
+package web
+
+import "embed"
+
+//go:generate go tool github.com/a-h/templ/cmd/templ generate
+//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs
+//go:generate gzip -f -k static/js/main.mjs
+//go:generate zstd -f -k --ultra -22 static/js/main.mjs
+//go:generate brotli -fZk static/js/main.mjs
+
+var (
+	//go:embed static
+	Static embed.FS
+)

web/index.go

@@ -0,0 +1,15 @@
+package web
+
+import "github.com/a-h/templ"
+
+func Base(title string, body templ.Component) templ.Component {
+	return base(title, body)
+}
+
+func Index() templ.Component {
+	return index()
+}
+
+func ErrorPage(msg string) templ.Component {
+	return errorPage(msg)
+}

web/index.templ

@@ -1,4 +1,4 @@
-package main
+package web
 
 import (
 	"github.com/TecharoHQ/anubis"

web/index_templ.go

@@ -1,7 +1,7 @@
 // Code generated by templ - DO NOT EDIT.
 
 // templ: version: v0.3.833
-package main
+package web
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.
 

xess/xess_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.833
+// templ: version: v0.3.850
 package xess
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.