diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2a25740..54d833c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -56,6 +56,14 @@ jobs: restore-keys: | ${{ runner.os }}-golang- + - name: Cache playwright binaries + uses: actions/cache@v3 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('**/go.sum') }} + - name: Build run: go build ./... diff --git a/anubis.go b/anubis.go new file mode 100644 index 0000000..b184a45 --- /dev/null +++ b/anubis.go @@ -0,0 +1,19 @@ +// Package Anubis contains the version number of Anubis. +package anubis + +// Version is the current version of Anubis. +// +// This variable is set at build time using the -X linker flag. If not set, +// it defaults to "devel". +var Version = "devel" + +// CookieName is the name of the cookie that Anubis uses in order to validate +// access. +const CookieName = "within.website-x-cmd-anubis-auth" + +// StaticPath is the location where all static Anubis assets are located. +const StaticPath = "/.within.website/x/cmd/anubis/" + +// DefaultDifficulty is the default "difficulty" (number of leading zeroes) +// that must be met by the client in order to pass the challenge. +const DefaultDifficulty = 4 diff --git a/cmd/anubis/CHANGELOG.md b/cmd/anubis/CHANGELOG.md deleted file mode 100644 index 612bec1..0000000 --- a/cmd/anubis/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# CHANGELOG - -## 2025-01-24 - -- Added support for custom bot policy documentation, allowing administrators to change how Anubis works to meet their needs. diff --git a/cmd/anubis/main.go b/cmd/anubis/main.go index e27e02f..e493931 100644 --- a/cmd/anubis/main.go +++ b/cmd/anubis/main.go @@ -2,20 +2,10 @@ package main import ( "context" - "crypto/ed25519" - "crypto/rand" - "crypto/sha256" - "crypto/subtle" - "embed" - "encoding/hex" - "encoding/json" "flag" "fmt" - "io" "log" "log/slog" - "math" - mrand "math/rand" "net" "net/http" "net/http/httputil" @@ -29,22 +19,18 @@ import ( "time" "github.com/TecharoHQ/anubis" - "github.com/TecharoHQ/anubis/cmd/anubis/internal/config" - "github.com/TecharoHQ/anubis/cmd/anubis/internal/dnsbl" "github.com/TecharoHQ/anubis/internal" - "github.com/TecharoHQ/anubis/xess" - "github.com/a-h/templ" + libanubis "github.com/TecharoHQ/anubis/lib" + "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/web" "github.com/facebookgo/flagenv" - "github.com/golang-jwt/jwt/v5" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( bind = flag.String("bind", ":8923", "network address to bind HTTP to") bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp") - challengeDifficulty = flag.Int("difficulty", defaultDifficulty, "difficulty of the challenge") + challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge") metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to") metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to") socketMode = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.") @@ -54,49 +40,8 @@ var ( target = flag.String("target", "http://localhost:3923", "target to reverse proxy to") healthcheck = flag.Bool("healthcheck", false, "run a health check against Anubis") debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally") - - //go:embed static botPolicies.json - static embed.FS - - challengesIssued = promauto.NewCounter(prometheus.CounterOpts{ - Name: "anubis_challenges_issued", - Help: "The total number of challenges issued", - }) - - challengesValidated = promauto.NewCounter(prometheus.CounterOpts{ - Name: "anubis_challenges_validated", - Help: "The total number of challenges validated", - }) - - droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "anubis_dronebl_hits", - Help: "The total number of hits from DroneBL", - }, []string{"status"}) - - failedValidations = promauto.NewCounter(prometheus.CounterOpts{ - Name: "anubis_failed_validations", - Help: "The total number of failed validations", - }) - - timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{ - Name: "anubis_time_taken", - Help: "The time taken for a browser to generate a response (milliseconds)", - Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19), - }) ) -const ( - cookieName = "within.website-x-cmd-anubis-auth" - staticPath = "/.within.website/x/cmd/anubis/" - defaultDifficulty = 4 -) - -//go:generate go tool github.com/a-h/templ/cmd/templ generate -//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs -//go:generate gzip -f -k static/js/main.mjs -//go:generate zstd -f -k --ultra -22 static/js/main.mjs -//go:generate brotli -fZk static/js/main.mjs - func doHealthCheck() error { resp, err := http.Get("http://localhost" + *metricsBind + "/metrics") if err != nil { @@ -145,6 +90,34 @@ func setupListener(network string, address string) (net.Listener, string) { return listener, formattedAddress } +func makeReverseProxy(target string) (http.Handler, error) { + u, err := url.Parse(target) + if err != nil { + return nil, fmt.Errorf("failed to parse target URL: %w", err) + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + + // https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124 + if u.Scheme == "unix" { + // clean path up so we don't use the socket path in proxied requests + addr := u.Path + u.Path = "" + // tell transport how to dial unix sockets + transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := net.Dialer{} + return dialer.DialContext(ctx, "unix", addr) + } + // tell transport how to handle the unix url scheme + transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport}) + } + + rp := httputil.NewSingleHostReverseProxy(u) + rp.Transport = transport + + return rp, nil +} + func main() { flagenv.Parse() flag.Parse() @@ -158,13 +131,18 @@ func main() { return } - s, err := New(*target, *policyFname) + rp, err := makeReverseProxy(*target) if err != nil { - log.Fatal(err) + log.Fatalf("can't make reverse proxy: %v", err) + } + + policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty) + if err != nil { + log.Fatalf("can't parse policy file: %v", err) } fmt.Println("Rule error IDs:") - for _, rule := range s.policy.Bots { + for _, rule := range policy.Bots { if rule.Action != config.RuleDeny { continue } @@ -178,25 +156,13 @@ func main() { } fmt.Println() - mux := http.NewServeMux() - xess.Mount(mux) - - mux.Handle(staticPath, internal.UnchangingCache(http.StripPrefix(staticPath, http.FileServerFS(static)))) - - // mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding) - - mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", s.makeChallenge) - mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", s.passChallenge) - mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", s.testError) - - if *robotsTxt { - mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { - http.ServeFileFS(w, r, static, "static/robots.txt") - }) - - mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) { - http.ServeFileFS(w, r, static, "static/robots.txt") - }) + s, err := libanubis.New(libanubis.Options{ + Next: rp, + Policy: policy, + ServeRobotsTXT: *robotsTxt, + }) + if err != nil { + log.Fatalf("can't construct libanubis.Server: %v", err) } wg := new(sync.WaitGroup) @@ -209,10 +175,8 @@ func main() { go metricsServer(ctx, wg.Done) } - mux.HandleFunc("/", s.maybeReverseProxy) - var h http.Handler - h = mux + h = s h = internal.DefaultXRealIP(*debugXRealIPDefault, h) h = internal.XForwardedForToXRealIP(h) @@ -267,428 +231,6 @@ func metricsServer(ctx context.Context, done func()) { } } -func sha256sum(text string) string { - hash := sha256.New() - hash.Write([]byte(text)) - return hex.EncodeToString(hash.Sum(nil)) -} - -func (s *Server) challengeFor(r *http.Request, difficulty int) string { - fp := sha256.Sum256(s.priv.Seed()) - - data := fmt.Sprintf( - "Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d", - r.Header.Get("Accept-Language"), - r.Header.Get("X-Real-Ip"), - r.UserAgent(), - time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339), - fp, - difficulty, - ) - return sha256sum(data) -} - -func New(target, policyFname string) (*Server, error) { - u, err := url.Parse(target) - if err != nil { - return nil, fmt.Errorf("failed to parse target URL: %w", err) - } - - pub, priv, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - return nil, fmt.Errorf("failed to generate ed25519 key: %w", err) - } - - transport := http.DefaultTransport.(*http.Transport).Clone() - - // https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124 - if u.Scheme == "unix" { - // clean path up so we don't use the socket path in proxied requests - addr := u.Path - u.Path = "" - // tell transport how to dial unix sockets - transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := net.Dialer{} - return dialer.DialContext(ctx, "unix", addr) - } - // tell transport how to handle the unix url scheme - transport.RegisterProtocol("unix", unixRoundTripper{Transport: transport}) - } - - rp := httputil.NewSingleHostReverseProxy(u) - rp.Transport = transport - - var fin io.ReadCloser - - if policyFname != "" { - fin, err = os.Open(policyFname) - if err != nil { - return nil, fmt.Errorf("can't parse policy file %s: %w", policyFname, err) - } - } else { - policyFname = "(static)/botPolicies.json" - fin, err = static.Open("botPolicies.json") - if err != nil { - return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", policyFname, err) - } - } - - defer fin.Close() - - policy, err := parseConfig(fin, policyFname, *challengeDifficulty) - if err != nil { - return nil, err // parseConfig sets a fancy error for us - } - - return &Server{ - rp: rp, - priv: priv, - pub: pub, - policy: policy, - dnsblCache: NewDecayMap[string, dnsbl.DroneBLResponse](), - }, nil -} - -// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124 -type unixRoundTripper struct { - Transport *http.Transport -} - -// set bare minimum stuff -func (t unixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req = req.Clone(req.Context()) - if req.Host == "" { - req.Host = "localhost" - } - req.URL.Host = req.Host // proxy error: no Host in request URL - req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion - return t.Transport.RoundTrip(req) -} - -type Server struct { - rp *httputil.ReverseProxy - priv ed25519.PrivateKey - pub ed25519.PublicKey - policy *ParsedConfig - dnsblCache *DecayMap[string, dnsbl.DroneBLResponse] -} - -func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) { - lg := slog.With( - "user_agent", r.UserAgent(), - "accept_language", r.Header.Get("Accept-Language"), - "priority", r.Header.Get("Priority"), - "x-forwarded-for", - r.Header.Get("X-Forwarded-For"), - "x-real-ip", r.Header.Get("X-Real-Ip"), - ) - - cr, rule, err := s.check(r) - if err != nil { - lg.Error("check failed", "err", err) - templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - r.Header.Add("X-Anubis-Rule", cr.Name) - r.Header.Add("X-Anubis-Action", string(cr.Rule)) - lg = lg.With("check_result", cr) - policyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1) - - ip := r.Header.Get("X-Real-Ip") - - if s.policy.DNSBL && ip != "" { - resp, ok := s.dnsblCache.Get(ip) - if !ok { - lg.Debug("looking up ip in dnsbl") - resp, err := dnsbl.Lookup(ip) - if err != nil { - lg.Error("can't look up ip in dnsbl", "err", err) - } - s.dnsblCache.Set(ip, resp, 24*time.Hour) - droneBLHits.WithLabelValues(resp.String()).Inc() - } - - if resp != dnsbl.AllGood { - lg.Info("DNSBL hit", "status", resp.String()) - templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r) - return - } - } - - switch cr.Rule { - case config.RuleAllow: - lg.Debug("allowing traffic to origin (explicit)") - s.rp.ServeHTTP(w, r) - return - case config.RuleDeny: - clearCookie(w) - lg.Info("explicit deny") - if rule == nil { - lg.Error("rule is nil, cannot calculate checksum") - templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - hash, err := rule.Hash() - if err != nil { - lg.Error("can't calculate checksum of rule", "err", err) - templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - lg.Debug("rule hash", "hash", hash) - templ.Handler(base("Oh noes!", errorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r) - return - case config.RuleChallenge: - lg.Debug("challenge requested") - default: - clearCookie(w) - templ.Handler(base("Oh noes!", errorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - ckie, err := r.Cookie(cookieName) - if err != nil { - lg.Debug("cookie not found", "path", r.URL.Path) - clearCookie(w) - s.renderIndex(w, r) - return - } - - if err := ckie.Valid(); err != nil { - lg.Debug("cookie is invalid", "err", err) - clearCookie(w) - s.renderIndex(w, r) - return - } - - if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() { - lg.Debug("cookie expired", "path", r.URL.Path) - clearCookie(w) - s.renderIndex(w, r) - return - } - - token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) { - return s.pub, nil - }, jwt.WithExpirationRequired(), jwt.WithStrictDecoding()) - - if err != nil || !token.Valid { - lg.Debug("invalid token", "path", r.URL.Path, "err", err) - clearCookie(w) - s.renderIndex(w, r) - return - } - - if randomJitter() { - r.Header.Add("X-Anubis-Status", "PASS-BRIEF") - lg.Debug("cookie is not enrolled into secondary screening") - s.rp.ServeHTTP(w, r) - return - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - lg.Debug("invalid token claims type", "path", r.URL.Path) - clearCookie(w) - s.renderIndex(w, r) - return - } - challenge := s.challengeFor(r, rule.Challenge.Difficulty) - - if claims["challenge"] != challenge { - lg.Debug("invalid challenge", "path", r.URL.Path) - clearCookie(w) - s.renderIndex(w, r) - return - } - - var nonce int - - if v, ok := claims["nonce"].(float64); ok { - nonce = int(v) - } - - calcString := fmt.Sprintf("%s%d", challenge, nonce) - calculated := sha256sum(calcString) - - if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 { - lg.Debug("invalid response", "path", r.URL.Path) - failedValidations.Inc() - clearCookie(w) - s.renderIndex(w, r) - return - } - - slog.Debug("all checks passed") - r.Header.Add("X-Anubis-Status", "PASS-FULL") - s.rp.ServeHTTP(w, r) -} - -func (s *Server) renderIndex(w http.ResponseWriter, r *http.Request) { - templ.Handler( - base("Making sure you're not a bot!", index()), - ).ServeHTTP(w, r) -} - -func (s *Server) makeChallenge(w http.ResponseWriter, r *http.Request) { - lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip")) - - cr, rule, err := s.check(r) - if err != nil { - lg.Error("check failed", "err", err) - w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(struct { - Error string `json:"error"` - }{ - Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"", - }) - return - } - lg = lg.With("check_result", cr) - challenge := s.challengeFor(r, rule.Challenge.Difficulty) - - json.NewEncoder(w).Encode(struct { - Challenge string `json:"challenge"` - Rules *config.ChallengeRules `json:"rules"` - }{ - Challenge: challenge, - Rules: rule.Challenge, - }) - lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr) - challengesIssued.Inc() -} - -func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) { - lg := slog.With( - "user_agent", r.UserAgent(), - "accept_language", r.Header.Get("Accept-Language"), - "priority", r.Header.Get("Priority"), - "x-forwarded-for", r.Header.Get("X-Forwarded-For"), - "x-real-ip", r.Header.Get("X-Real-Ip"), - ) - - cr, rule, err := s.check(r) - if err != nil { - lg.Error("check failed", "err", err) - templ.Handler(base("Oh noes!", errorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - lg = lg.With("check_result", cr) - - nonceStr := r.FormValue("nonce") - if nonceStr == "" { - clearCookie(w) - lg.Debug("no nonce") - templ.Handler(base("Oh noes!", errorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - elapsedTimeStr := r.FormValue("elapsedTime") - if elapsedTimeStr == "" { - clearCookie(w) - lg.Debug("no elapsedTime") - templ.Handler(base("Oh noes!", errorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64) - if err != nil { - clearCookie(w) - lg.Debug("elapsedTime doesn't parse", "err", err) - templ.Handler(base("Oh noes!", errorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - lg.Info("challenge took", "elapsedTime", elapsedTime) - timeTaken.Observe(elapsedTime) - - response := r.FormValue("response") - redir := r.FormValue("redir") - - challenge := s.challengeFor(r, rule.Challenge.Difficulty) - - nonce, err := strconv.Atoi(nonceStr) - if err != nil { - clearCookie(w) - lg.Debug("nonce doesn't parse", "err", err) - templ.Handler(base("Oh noes!", errorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - calcString := fmt.Sprintf("%s%d", challenge, nonce) - calculated := sha256sum(calcString) - - if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 { - clearCookie(w) - lg.Debug("hash does not match", "got", response, "want", calculated) - templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) - failedValidations.Inc() - return - } - - // compare the leading zeroes - if !strings.HasPrefix(response, strings.Repeat("0", *challengeDifficulty)) { - clearCookie(w) - lg.Debug("difficulty check failed", "response", response, "difficulty", *challengeDifficulty) - templ.Handler(base("Oh noes!", errorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) - failedValidations.Inc() - return - } - - // generate JWT cookie - token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{ - "challenge": challenge, - "nonce": nonce, - "response": response, - "iat": time.Now().Unix(), - "nbf": time.Now().Add(-1 * time.Minute).Unix(), - "exp": time.Now().Add(24 * 7 * time.Hour).Unix(), - }) - tokenString, err := token.SignedString(s.priv) - if err != nil { - lg.Error("failed to sign JWT", "err", err) - clearCookie(w) - templ.Handler(base("Oh noes!", errorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) - return - } - - http.SetCookie(w, &http.Cookie{ - Name: cookieName, - Value: tokenString, - Expires: time.Now().Add(24 * 7 * time.Hour), - SameSite: http.SameSiteLaxMode, - Path: "/", - }) - - challengesValidated.Inc() - lg.Debug("challenge passed, redirecting to app") - http.Redirect(w, r, redir, http.StatusFound) -} - -func (s *Server) testError(w http.ResponseWriter, r *http.Request) { - err := r.FormValue("err") - templ.Handler(base("Oh noes!", errorPage(err)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) -} - -func ohNoes(w http.ResponseWriter, r *http.Request, err error) { - slog.Error("super fatal error", "err", err) - templ.Handler(base("Oh noes!", errorPage("An internal server error happened")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) -} - -func clearCookie(w http.ResponseWriter) { - http.SetCookie(w, &http.Cookie{ - Name: cookieName, - Value: "", - Expires: time.Now().Add(-1 * time.Hour), - MaxAge: -1, - SameSite: http.SameSiteLaxMode, - }) -} - -func randomJitter() bool { - return mrand.Intn(100) > 10 -} - func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) { priorityList := []string{"zstd", "br", "gzip"} enc2ext := map[string]string{ @@ -701,11 +243,11 @@ func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) { if strings.Contains(r.Header.Get("Accept-Encoding"), enc) { w.Header().Set("Content-Type", "text/javascript") w.Header().Set("Content-Encoding", enc) - http.ServeFileFS(w, r, static, "static/js/main.mjs."+enc2ext[enc]) + http.ServeFileFS(w, r, web.Static, "static/js/main.mjs."+enc2ext[enc]) return } } w.Header().Set("Content-Type", "text/javascript") - http.ServeFileFS(w, r, static, "static/js/main.mjs") + http.ServeFileFS(w, r, web.Static, "static/js/main.mjs") } diff --git a/cmd/anubis/policy.go b/cmd/anubis/policy.go deleted file mode 100644 index 5de5e72..0000000 --- a/cmd/anubis/policy.go +++ /dev/null @@ -1,212 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "log" - "log/slog" - "net" - "net/http" - "regexp" - - "github.com/TecharoHQ/anubis/cmd/anubis/internal/config" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/yl2chen/cidranger" -) - -var ( - policyApplications = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "anubis_policy_results", - Help: "The results of each policy rule", - }, []string{"rule", "action"}) -) - -type ParsedConfig struct { - orig config.Config - - Bots []Bot - DNSBL bool -} - -type Bot struct { - Name string - UserAgent *regexp.Regexp - Path *regexp.Regexp - Action config.Rule `json:"action"` - Challenge *config.ChallengeRules - Ranger cidranger.Ranger -} - -func (b Bot) Hash() (string, error) { - var pathRex string - if b.Path != nil { - pathRex = b.Path.String() - } - var userAgentRex string - if b.UserAgent != nil { - userAgentRex = b.UserAgent.String() - } - - return sha256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil -} - -func parseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { - var c config.Config - if err := json.NewDecoder(fin).Decode(&c); err != nil { - return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err) - } - - if err := c.Valid(); err != nil { - return nil, err - } - - var err error - - result := &ParsedConfig{ - orig: c, - } - - for _, b := range c.Bots { - if berr := b.Valid(); berr != nil { - err = errors.Join(err, berr) - continue - } - - var botParseErr error - parsedBot := Bot{ - Name: b.Name, - Action: b.Action, - } - - if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 { - parsedBot.Ranger = cidranger.NewPCTrieRanger() - - for _, cidr := range b.RemoteAddr { - _, rng, err := net.ParseCIDR(cidr) - if err != nil { - return nil, fmt.Errorf("[unexpected] range %s not parsing: %w", cidr, err) - } - - parsedBot.Ranger.Insert(cidranger.NewBasicRangerEntry(*rng)) - } - } - - if b.UserAgentRegex != nil { - userAgent, err := regexp.Compile(*b.UserAgentRegex) - if err != nil { - botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling user agent regexp: %w", err)) - continue - } else { - parsedBot.UserAgent = userAgent - } - } - - if b.PathRegex != nil { - path, err := regexp.Compile(*b.PathRegex) - if err != nil { - botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling path regexp: %w", err)) - continue - } else { - parsedBot.Path = path - } - } - - if b.Challenge == nil { - parsedBot.Challenge = &config.ChallengeRules{ - Difficulty: defaultDifficulty, - ReportAs: defaultDifficulty, - Algorithm: config.AlgorithmFast, - } - } else { - parsedBot.Challenge = b.Challenge - if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown { - parsedBot.Challenge.Algorithm = config.AlgorithmFast - } - } - - result.Bots = append(result.Bots, parsedBot) - } - - if err != nil { - return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, err) - } - - result.DNSBL = c.DNSBL - - return result, nil -} - -type CheckResult struct { - Name string - Rule config.Rule -} - -func (cr CheckResult) LogValue() slog.Value { - return slog.GroupValue( - slog.String("name", cr.Name), - slog.String("rule", string(cr.Rule))) -} - -func cr(name string, rule config.Rule) CheckResult { - return CheckResult{ - Name: name, - Rule: rule, - } -} - -func (s *Server) checkRemoteAddress(b Bot, addr net.IP) bool { - if b.Ranger == nil { - return false - } - - ok, err := b.Ranger.Contains(addr) - if err != nil { - log.Panicf("[unexpected] something very funky is going on, %q does not have a calculable network number: %v", addr.String(), err) - } - - return ok -} - -// Check evaluates the list of rules, and returns the result -func (s *Server) check(r *http.Request) (CheckResult, *Bot, error) { - host := r.Header.Get("X-Real-Ip") - if host == "" { - return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] X-Real-Ip header is not set") - } - - addr := net.ParseIP(host) - if addr == nil { - return zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host) - } - - for _, b := range s.policy.Bots { - if b.UserAgent != nil { - if uaMatch := b.UserAgent.MatchString(r.UserAgent()); uaMatch || (uaMatch && s.checkRemoteAddress(b, addr)) { - return cr("bot/"+b.Name, b.Action), &b, nil - } - } - - if b.Path != nil { - if pathMatch := b.Path.MatchString(r.URL.Path); pathMatch || (pathMatch && s.checkRemoteAddress(b, addr)) { - return cr("bot/"+b.Name, b.Action), &b, nil - } - } - - if b.Ranger != nil { - if s.checkRemoteAddress(b, addr) { - return cr("bot/"+b.Name, b.Action), &b, nil - } - } - } - - return cr("default/allow", config.RuleAllow), &Bot{ - Challenge: &config.ChallengeRules{ - Difficulty: defaultDifficulty, - ReportAs: defaultDifficulty, - Algorithm: config.AlgorithmFast, - }, - }, nil -} diff --git a/cmd/anubis/botPolicies.json b/data/botPolicies.json similarity index 100% rename from cmd/anubis/botPolicies.json rename to data/botPolicies.json diff --git a/data/embed.go b/data/embed.go new file mode 100644 index 0000000..5a5f4d2 --- /dev/null +++ b/data/embed.go @@ -0,0 +1,8 @@ +package data + +import "embed" + +var ( + //go:embed botPolicies.json + BotPolicies embed.FS +) diff --git a/cmd/anubis/decaymap.go b/decaymap/decaymap.go similarity index 67% rename from cmd/anubis/decaymap.go rename to decaymap/decaymap.go index dcd2952..edcbd1a 100644 --- a/cmd/anubis/decaymap.go +++ b/decaymap/decaymap.go @@ -1,17 +1,17 @@ -package main +package decaymap import ( "sync" "time" ) -func zilch[T any]() T { +func Zilch[T any]() T { var zero T return zero } -// DecayMap is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time. -type DecayMap[K comparable, V any] struct { +// Impl is a lazy key->value map. It's a wrapper around a map and a mutex. If values exceed their time-to-live, they are pruned at Get time. +type Impl[K comparable, V any] struct { data map[K]decayMapEntry[V] lock sync.RWMutex } @@ -21,17 +21,17 @@ type decayMapEntry[V any] struct { expiry time.Time } -// NewDecayMap creates a new DecayMap of key type K and value type V. +// New creates a new DecayMap of key type K and value type V. // // Key types must be comparable to work with maps. -func NewDecayMap[K comparable, V any]() *DecayMap[K, V] { - return &DecayMap[K, V]{ +func New[K comparable, V any]() *Impl[K, V] { + return &Impl[K, V]{ data: make(map[K]decayMapEntry[V]), } } // expire forcibly expires a key by setting its time-to-live one second in the past. -func (m *DecayMap[K, V]) expire(key K) bool { +func (m *Impl[K, V]) expire(key K) bool { m.lock.RLock() val, ok := m.data[key] m.lock.RUnlock() @@ -51,13 +51,13 @@ func (m *DecayMap[K, V]) expire(key K) bool { // Get gets a value from the DecayMap by key. // // If a value has expired, forcibly delete it if it was not updated. -func (m *DecayMap[K, V]) Get(key K) (V, bool) { +func (m *Impl[K, V]) Get(key K) (V, bool) { m.lock.RLock() value, ok := m.data[key] m.lock.RUnlock() if !ok { - return zilch[V](), false + return Zilch[V](), false } if time.Now().After(value.expiry) { @@ -69,14 +69,14 @@ func (m *DecayMap[K, V]) Get(key K) (V, bool) { } m.lock.Unlock() - return zilch[V](), false + return Zilch[V](), false } return value.Value, true } // Set sets a key value pair in the map. -func (m *DecayMap[K, V]) Set(key K, value V, ttl time.Duration) { +func (m *Impl[K, V]) Set(key K, value V, ttl time.Duration) { m.lock.Lock() defer m.lock.Unlock() diff --git a/cmd/anubis/decaymap_test.go b/decaymap/decaymap_test.go similarity index 83% rename from cmd/anubis/decaymap_test.go rename to decaymap/decaymap_test.go index 73e0626..c930e08 100644 --- a/cmd/anubis/decaymap_test.go +++ b/decaymap/decaymap_test.go @@ -1,12 +1,12 @@ -package main +package decaymap import ( "testing" "time" ) -func TestDecayMap(t *testing.T) { - dm := NewDecayMap[string, string]() +func TestImpl(t *testing.T) { + dm := New[string, string]() dm.Set("test", "hi", 5*time.Minute) diff --git a/doc.go b/doc.go deleted file mode 100644 index 2593bad..0000000 --- a/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Package Anubis contains the version number of Anubis. -package anubis - -// Version is the current version of Anubis. -// -// This variable is set at build time using the -X linker flag. If not set, -// it defaults to "devel". -var Version = "devel" diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index f0b9dc4..e0743c0 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed and clarified installation instructions - Introduced integration tests using Playwright +- Refactor & Split up Anubis into cmd and lib.go +- Fixed bot check to only apply if address range matches ## v1.14.2 diff --git a/cmd/anubis/internal/dnsbl/dnsbl.go b/internal/dnsbl/dnsbl.go similarity index 100% rename from cmd/anubis/internal/dnsbl/dnsbl.go rename to internal/dnsbl/dnsbl.go diff --git a/cmd/anubis/internal/dnsbl/dnsbl_test.go b/internal/dnsbl/dnsbl_test.go similarity index 100% rename from cmd/anubis/internal/dnsbl/dnsbl_test.go rename to internal/dnsbl/dnsbl_test.go diff --git a/cmd/anubis/internal/dnsbl/droneblresponse_string.go b/internal/dnsbl/droneblresponse_string.go similarity index 100% rename from cmd/anubis/internal/dnsbl/droneblresponse_string.go rename to internal/dnsbl/droneblresponse_string.go diff --git a/internal/hash.go b/internal/hash.go new file mode 100644 index 0000000..818ad55 --- /dev/null +++ b/internal/hash.go @@ -0,0 +1,12 @@ +package internal + +import ( + "crypto/sha256" + "encoding/hex" +) + +func SHA256sum(text string) string { + hash := sha256.New() + hash.Write([]byte(text)) + return hex.EncodeToString(hash.Sum(nil)) +} diff --git a/internal/test/playwright_test.go b/internal/test/playwright_test.go index 156d7df..b8b6cac 100644 --- a/internal/test/playwright_test.go +++ b/internal/test/playwright_test.go @@ -1,5 +1,3 @@ -//go:build integration - // Integration tests for Anubis, using Playwright. // // These tests require an already running Anubis and Playwright server. @@ -16,31 +14,60 @@ package test import ( - "context" "flag" "fmt" "net/http" + "net/http/httptest" "net/url" "os" + "os/exec" "testing" "time" + "github.com/TecharoHQ/anubis" + libanubis "github.com/TecharoHQ/anubis/lib" "github.com/playwright-community/playwright-go" ) var ( - anubisServer = flag.String("anubis", "http://localhost:8923", "Anubis server URL") serverBindAddr = flag.String("bind", "localhost:3923", "test server bind address") + playwrightPort = flag.Int("playwright-port", 3000, "Playwright port") playwrightServer = flag.String("playwright", "ws://localhost:3000", "Playwright server URL") playwrightMaxTime = flag.Duration("playwright-max-time", 5*time.Second, "maximum time for Playwright requests") playwrightMaxHardTime = flag.Duration("playwright-max-hard-time", 5*time.Minute, "maximum time for hard Playwright requests") testCases = []testCase{ - {name: "firefox", action: actionChallenge, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"}, - {name: "headlessChrome", action: actionDeny, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36"}, - {name: "kagiBadIP", action: actionChallenge, isHard: true, realIP: placeholderIP, userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"}, - {name: "kagiGoodIP", action: actionAllow, realIP: "216.18.205.234", userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"}, - {name: "unknownAgent", action: actionAllow, realIP: placeholderIP, userAgent: "AnubisTest/0"}, + { + name: "firefox", + action: actionChallenge, + realIP: placeholderIP, + userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0", + }, + { + name: "headlessChrome", + action: actionDeny, + realIP: placeholderIP, + userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36", + }, + { + name: "kagiBadIP", + action: actionChallenge, + isHard: true, + realIP: placeholderIP, + userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", + }, + { + name: "kagiGoodIP", + action: actionAllow, + realIP: "216.18.205.234", + userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", + }, + { + name: "unknownAgent", + action: actionAllow, + realIP: placeholderIP, + userAgent: "AnubisTest/0", + }, } ) @@ -49,7 +76,8 @@ const ( actionDeny action = "DENY" actionChallenge action = "CHALLENGE" - placeholderIP = "fd11:5ee:bad:c0de::" + placeholderIP = "fd11:5ee:bad:c0de::" + playwrightVersion = "1.50.1" ) type action string @@ -61,21 +89,143 @@ type testCase struct { realIP, userAgent string } +func doesNPXExist(t *testing.T) { + t.Helper() + + if _, err := exec.LookPath("npx"); err != nil { + t.Skipf("npx not found in PATH, skipping integration smoke testing: %v", err) + } +} + +func run(t *testing.T, command string) string { + t.Helper() + + shPath, err := exec.LookPath("sh") + if err != nil { + t.Fatalf("[unexpected] %v", err) + } + + t.Logf("running command: %s", command) + + cmd := exec.Command(shPath, "-c", command) + cmd.Stdin = nil + cmd.Stderr = os.Stderr + output, err := cmd.Output() + if err != nil { + t.Fatalf("can't run command: %v", err) + } + + return string(output) +} + +func daemonize(t *testing.T, command string) { + t.Helper() + + shPath, err := exec.LookPath("sh") + if err != nil { + t.Fatalf("[unexpected] %v", err) + } + + t.Logf("daemonizing command: %s", command) + + cmd := exec.Command(shPath, "-c", command) + cmd.Stdin = nil + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + if err := cmd.Start(); err != nil { + t.Fatalf("can't daemonize command: %v", err) + } + + t.Cleanup(func() { + cmd.Process.Kill() + }) +} + +func startPlaywright(t *testing.T) { + t.Helper() + + if os.Getenv("CI") == "true" { + run(t, fmt.Sprintf("npx --yes playwright@%s install --with-deps", playwrightVersion)) + } else { + run(t, fmt.Sprintf("npx --yes playwright@%s install", playwrightVersion)) + } + + daemonize(t, fmt.Sprintf("npx --yes playwright@%s run-server --port %d", playwrightVersion, *playwrightPort)) + + for true { + if _, err := http.Get(fmt.Sprintf("http://localhost:%d", *playwrightPort)); err != nil { + time.Sleep(500 * time.Millisecond) + continue + } + break + } + + //nosleep:bypass XXX(Xe): Playwright doesn't have a good way to signal readiness. This is a HACK that will just let the tests pass. + time.Sleep(2 * time.Second) +} + func TestPlaywrightBrowser(t *testing.T) { + if os.Getenv("CI") == "true" { + t.Skip("XXX(Xe): This is broken in CI, will fix later") + } + + if os.Getenv("DONT_USE_NETWORK") != "" { + t.Skip("test requires network egress") + return + } + + doesNPXExist(t) + startPlaywright(t) + pw := setupPlaywright(t) - spawnTestServer(t) + anubisURL := spawnAnubis(t) + browsers := []playwright.BrowserType{pw.Chromium, pw.Firefox, pw.WebKit} for _, typ := range browsers { + t.Run(typ.Name()+"/warmup", func(t *testing.T) { + browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{ + ExposeNetwork: playwright.String(""), + }) + if err != nil { + t.Fatalf("could not connect to remote browser: %v", err) + } + defer browser.Close() + + ctx, err := browser.NewContext(playwright.BrowserNewContextOptions{ + AcceptDownloads: playwright.Bool(false), + ExtraHttpHeaders: map[string]string{ + "X-Real-Ip": "127.0.0.1", + }, + UserAgent: playwright.String("Sephiroth"), + }) + if err != nil { + t.Fatalf("could not create context: %v", err) + } + defer ctx.Close() + + page, err := ctx.NewPage() + if err != nil { + t.Fatalf("could not create page: %v", err) + } + defer page.Close() + + timeout := 2.0 + page.Goto(anubisURL, playwright.PageGotoOptions{ + Timeout: &timeout, + }) + }) + for _, tc := range testCases { - name := fmt.Sprintf("%s@%s", tc.name, typ.Name()) + name := fmt.Sprintf("%s/%s", typ.Name(), tc.name) t.Run(name, func(t *testing.T) { _, hasDeadline := t.Deadline() if tc.isHard && hasDeadline { t.Skip("skipping hard challenge with deadline") } - perfomedAction := executeTestCase(t, tc, typ) + perfomedAction := executeTestCase(t, tc, typ, anubisURL) if perfomedAction != tc.action { t.Errorf("unexpected test result, expected %s, got %s", tc.action, perfomedAction) @@ -97,7 +247,7 @@ func buildBrowserConnect(name string) string { return u.String() } -func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) action { +func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType, anubisURL string) action { deadline, _ := t.Deadline() browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{ @@ -129,7 +279,7 @@ func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) acti // Attempt challenge. start := time.Now() - _, err = page.Goto(*anubisServer, playwright.PageGotoOptions{ + _, err = page.Goto(anubisURL, playwright.PageGotoOptions{ Timeout: pwTimeout(tc, deadline), }) if err != nil { @@ -252,25 +402,34 @@ func setupPlaywright(t *testing.T) *playwright.Playwright { return pw } -func spawnTestServer(t *testing.T) { +func spawnAnubis(t *testing.T) string { t.Helper() - s := new(http.Server) - s.Addr = *serverBindAddr - s.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "text/html") fmt.Fprintf(w, "%d", time.Now().Unix()) }) - go func() { - if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed { - t.Logf("test HTTP server terminated unexpectedly: %v", err) - } - }() + policy, err := libanubis.LoadPoliciesOrDefault("", anubis.DefaultDifficulty) + if err != nil { + t.Fatal(err) + } + + s, err := libanubis.New(libanubis.Options{ + Next: h, + Policy: policy, + ServeRobotsTXT: true, + }) + if err != nil { + t.Fatalf("can't construct libanubis.Server: %v", err) + } + + ts := httptest.NewServer(s) + t.Log(ts.URL) t.Cleanup(func() { - if err := s.Shutdown(context.Background()); err != nil { - t.Fatalf("could not shutdown test server: %v", err) - } + ts.Close() }) + + return ts.URL } diff --git a/lib/anubis.go b/lib/anubis.go new file mode 100644 index 0000000..5953a48 --- /dev/null +++ b/lib/anubis.go @@ -0,0 +1,519 @@ +package lib + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/json" + "fmt" + "io" + "log" + "log/slog" + "math" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/a-h/templ" + "github.com/golang-jwt/jwt/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/data" + "github.com/TecharoHQ/anubis/decaymap" + "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/internal/dnsbl" + "github.com/TecharoHQ/anubis/lib/policy" + "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/TecharoHQ/anubis/web" + "github.com/TecharoHQ/anubis/xess" +) + +var ( + challengesIssued = promauto.NewCounter(prometheus.CounterOpts{ + Name: "anubis_challenges_issued", + Help: "The total number of challenges issued", + }) + + challengesValidated = promauto.NewCounter(prometheus.CounterOpts{ + Name: "anubis_challenges_validated", + Help: "The total number of challenges validated", + }) + + droneBLHits = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "anubis_dronebl_hits", + Help: "The total number of hits from DroneBL", + }, []string{"status"}) + + failedValidations = promauto.NewCounter(prometheus.CounterOpts{ + Name: "anubis_failed_validations", + Help: "The total number of failed validations", + }) + + timeTaken = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "anubis_time_taken", + Help: "The time taken for a browser to generate a response (milliseconds)", + Buckets: prometheus.ExponentialBucketsRange(1, math.Pow(2, 18), 19), + }) +) + +type Options struct { + Next http.Handler + Policy *policy.ParsedConfig + ServeRobotsTXT bool +} + +func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { + var fin io.ReadCloser + var err error + + if fname != "" { + fin, err = os.Open(fname) + if err != nil { + return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err) + } + } else { + fname = "(data)/botPolicies.json" + fin, err = data.BotPolicies.Open("botPolicies.json") + if err != nil { + return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err) + } + } + + defer fin.Close() + + policy, err := policy.ParseConfig(fin, fname, defaultDifficulty) + + return policy, err +} + +func New(opts Options) (*Server, error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate ed25519 key: %w", err) + } + + if err != nil { + return nil, err // parseConfig sets a fancy error for us + } + + result := &Server{ + next: opts.Next, + priv: priv, + pub: pub, + policy: opts.Policy, + DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](), + } + + mux := http.NewServeMux() + xess.Mount(mux) + + mux.Handle(anubis.StaticPath, internal.UnchangingCache(http.StripPrefix(anubis.StaticPath, http.FileServerFS(web.Static)))) + + if opts.ServeRobotsTXT { + mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { + http.ServeFileFS(w, r, web.Static, "static/robots.txt") + }) + + mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) { + http.ServeFileFS(w, r, web.Static, "static/robots.txt") + }) + } + + // mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding) + + mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", result.MakeChallenge) + mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", result.PassChallenge) + mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", result.TestError) + + mux.HandleFunc("/", result.MaybeReverseProxy) + + result.mux = mux + + return result, nil +} + +type Server struct { + mux *http.ServeMux + next http.Handler + priv ed25519.PrivateKey + pub ed25519.PublicKey + policy *policy.ParsedConfig + DNSBLCache *decaymap.Impl[string, dnsbl.DroneBLResponse] + ChallengeDifficulty int +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +func (s *Server) challengeFor(r *http.Request, difficulty int) string { + fp := sha256.Sum256(s.priv.Seed()) + + data := fmt.Sprintf( + "Accept-Language=%s,X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d", + r.Header.Get("Accept-Language"), + r.Header.Get("X-Real-Ip"), + r.UserAgent(), + time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339), + fp, + difficulty, + ) + return internal.SHA256sum(data) +} + +func (s *Server) MaybeReverseProxy(w http.ResponseWriter, r *http.Request) { + lg := slog.With( + "user_agent", r.UserAgent(), + "accept_language", r.Header.Get("Accept-Language"), + "priority", r.Header.Get("Priority"), + "x-forwarded-for", + r.Header.Get("X-Forwarded-For"), + "x-real-ip", r.Header.Get("X-Real-Ip"), + ) + + cr, rule, err := s.check(r) + if err != nil { + lg.Error("check failed", "err", err) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"maybeReverseProxy\"")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + r.Header.Add("X-Anubis-Rule", cr.Name) + r.Header.Add("X-Anubis-Action", string(cr.Rule)) + lg = lg.With("check_result", cr) + policy.PolicyApplications.WithLabelValues(cr.Name, string(cr.Rule)).Add(1) + + ip := r.Header.Get("X-Real-Ip") + + if s.policy.DNSBL && ip != "" { + resp, ok := s.DNSBLCache.Get(ip) + if !ok { + lg.Debug("looking up ip in dnsbl") + resp, err := dnsbl.Lookup(ip) + if err != nil { + lg.Error("can't look up ip in dnsbl", "err", err) + } + s.DNSBLCache.Set(ip, resp, 24*time.Hour) + droneBLHits.WithLabelValues(resp.String()).Inc() + } + + if resp != dnsbl.AllGood { + lg.Info("DNSBL hit", "status", resp.String()) + templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r) + return + } + } + + switch cr.Rule { + case config.RuleAllow: + lg.Debug("allowing traffic to origin (explicit)") + s.next.ServeHTTP(w, r) + return + case config.RuleDeny: + ClearCookie(w) + lg.Info("explicit deny") + if rule == nil { + lg.Error("rule is nil, cannot calculate checksum") + templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + hash, err := rule.Hash() + if err != nil { + lg.Error("can't calculate checksum of rule", "err", err) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + lg.Debug("rule hash", "hash", hash) + templ.Handler(web.Base("Oh noes!", web.ErrorPage(fmt.Sprintf("Access Denied: error code %s", hash))), templ.WithStatus(http.StatusOK)).ServeHTTP(w, r) + return + case config.RuleChallenge: + lg.Debug("challenge requested") + default: + ClearCookie(w) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + ckie, err := r.Cookie(anubis.CookieName) + if err != nil { + lg.Debug("cookie not found", "path", r.URL.Path) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + if err := ckie.Valid(); err != nil { + lg.Debug("cookie is invalid", "err", err) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() { + lg.Debug("cookie expired", "path", r.URL.Path) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + token, err := jwt.ParseWithClaims(ckie.Value, jwt.MapClaims{}, func(token *jwt.Token) (interface{}, error) { + return s.pub, nil + }, jwt.WithExpirationRequired(), jwt.WithStrictDecoding()) + + if err != nil || !token.Valid { + lg.Debug("invalid token", "path", r.URL.Path, "err", err) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + if randomJitter() { + r.Header.Add("X-Anubis-Status", "PASS-BRIEF") + lg.Debug("cookie is not enrolled into secondary screening") + s.next.ServeHTTP(w, r) + return + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + lg.Debug("invalid token claims type", "path", r.URL.Path) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + challenge := s.challengeFor(r, rule.Challenge.Difficulty) + + if claims["challenge"] != challenge { + lg.Debug("invalid challenge", "path", r.URL.Path) + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + var nonce int + + if v, ok := claims["nonce"].(float64); ok { + nonce = int(v) + } + + calcString := fmt.Sprintf("%s%d", challenge, nonce) + calculated := internal.SHA256sum(calcString) + + if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 { + lg.Debug("invalid response", "path", r.URL.Path) + failedValidations.Inc() + ClearCookie(w) + s.RenderIndex(w, r) + return + } + + slog.Debug("all checks passed") + r.Header.Add("X-Anubis-Status", "PASS-FULL") + s.next.ServeHTTP(w, r) +} + +func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request) { + templ.Handler( + web.Base("Making sure you're not a bot!", web.Index()), + ).ServeHTTP(w, r) +} + +func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) { + lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip")) + + cr, rule, err := s.check(r) + if err != nil { + lg.Error("check failed", "err", err) + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(struct { + Error string `json:"error"` + }{ + Error: "Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"makeChallenge\"", + }) + return + } + lg = lg.With("check_result", cr) + challenge := s.challengeFor(r, rule.Challenge.Difficulty) + + json.NewEncoder(w).Encode(struct { + Challenge string `json:"challenge"` + Rules *config.ChallengeRules `json:"rules"` + }{ + Challenge: challenge, + Rules: rule.Challenge, + }) + lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr) + challengesIssued.Inc() +} + +func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) { + lg := slog.With( + "user_agent", r.UserAgent(), + "accept_language", r.Header.Get("Accept-Language"), + "priority", r.Header.Get("Priority"), + "x-forwarded-for", r.Header.Get("X-Forwarded-For"), + "x-real-ip", r.Header.Get("X-Real-Ip"), + ) + + cr, rule, err := s.check(r) + if err != nil { + lg.Error("check failed", "err", err) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("Internal Server Error: administrator has misconfigured Anubis. Please contact the administrator and ask them to look for the logs around \"passChallenge\".")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + lg = lg.With("check_result", cr) + + nonceStr := r.FormValue("nonce") + if nonceStr == "" { + ClearCookie(w) + lg.Debug("no nonce") + templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + elapsedTimeStr := r.FormValue("elapsedTime") + if elapsedTimeStr == "" { + ClearCookie(w) + lg.Debug("no elapsedTime") + templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64) + if err != nil { + ClearCookie(w) + lg.Debug("elapsedTime doesn't parse", "err", err) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + lg.Info("challenge took", "elapsedTime", elapsedTime) + timeTaken.Observe(elapsedTime) + + response := r.FormValue("response") + redir := r.FormValue("redir") + + challenge := s.challengeFor(r, rule.Challenge.Difficulty) + + nonce, err := strconv.Atoi(nonceStr) + if err != nil { + ClearCookie(w) + lg.Debug("nonce doesn't parse", "err", err) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + calcString := fmt.Sprintf("%s%d", challenge, nonce) + calculated := internal.SHA256sum(calcString) + + if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 { + ClearCookie(w) + lg.Debug("hash does not match", "got", response, "want", calculated) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) + failedValidations.Inc() + return + } + + // compare the leading zeroes + if !strings.HasPrefix(response, strings.Repeat("0", s.ChallengeDifficulty)) { + ClearCookie(w) + lg.Debug("difficulty check failed", "response", response, "difficulty", s.ChallengeDifficulty) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) + failedValidations.Inc() + return + } + + // generate JWT cookie + token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, jwt.MapClaims{ + "challenge": challenge, + "nonce": nonce, + "response": response, + "iat": time.Now().Unix(), + "nbf": time.Now().Add(-1 * time.Minute).Unix(), + "exp": time.Now().Add(24 * 7 * time.Hour).Unix(), + }) + tokenString, err := token.SignedString(s.priv) + if err != nil { + lg.Error("failed to sign JWT", "err", err) + ClearCookie(w) + templ.Handler(web.Base("Oh noes!", web.ErrorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) + return + } + + http.SetCookie(w, &http.Cookie{ + Name: anubis.CookieName, + Value: tokenString, + Expires: time.Now().Add(24 * 7 * time.Hour), + SameSite: http.SameSiteLaxMode, + Path: "/", + }) + + challengesValidated.Inc() + lg.Debug("challenge passed, redirecting to app") + http.Redirect(w, r, redir, http.StatusFound) +} + +func (s *Server) TestError(w http.ResponseWriter, r *http.Request) { + err := r.FormValue("err") + templ.Handler(web.Base("Oh noes!", web.ErrorPage(err)), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) +} + +// Check evaluates the list of rules, and returns the result +func (s *Server) check(r *http.Request) (CheckResult, *policy.Bot, error) { + host := r.Header.Get("X-Real-Ip") + if host == "" { + return decaymap.Zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] X-Real-Ip header is not set") + } + + addr := net.ParseIP(host) + if addr == nil { + return decaymap.Zilch[CheckResult](), nil, fmt.Errorf("[misconfiguration] %q is not an IP address", host) + } + + for _, b := range s.policy.Bots { + if b.UserAgent != nil { + if b.UserAgent.MatchString(r.UserAgent()) && s.checkRemoteAddress(b, addr) { + return cr("bot/"+b.Name, b.Action), &b, nil + } + } + + if b.Path != nil { + if b.Path.MatchString(r.URL.Path) && s.checkRemoteAddress(b, addr) { + return cr("bot/"+b.Name, b.Action), &b, nil + } + } + + if b.Ranger != nil { + if s.checkRemoteAddress(b, addr) { + return cr("bot/"+b.Name, b.Action), &b, nil + } + } + } + + return cr("default/allow", config.RuleAllow), &policy.Bot{ + Challenge: &config.ChallengeRules{ + Difficulty: anubis.DefaultDifficulty, + ReportAs: anubis.DefaultDifficulty, + Algorithm: config.AlgorithmFast, + }, + }, nil +} + +func (s *Server) checkRemoteAddress(b policy.Bot, addr net.IP) bool { + if b.Ranger == nil { + return true + } + + ok, err := b.Ranger.Contains(addr) + if err != nil { + log.Panicf("[unexpected] something very funky is going on, %q does not have a calculable network number: %v", addr.String(), err) + } + + return ok +} diff --git a/lib/checkresult.go b/lib/checkresult.go new file mode 100644 index 0000000..3803df2 --- /dev/null +++ b/lib/checkresult.go @@ -0,0 +1,25 @@ +package lib + +import ( + "log/slog" + + "github.com/TecharoHQ/anubis/lib/policy/config" +) + +type CheckResult struct { + Name string + Rule config.Rule +} + +func (cr CheckResult) LogValue() slog.Value { + return slog.GroupValue( + slog.String("name", cr.Name), + slog.String("rule", string(cr.Rule))) +} + +func cr(name string, rule config.Rule) CheckResult { + return CheckResult{ + Name: name, + Rule: rule, + } +} diff --git a/lib/http.go b/lib/http.go new file mode 100644 index 0000000..1284523 --- /dev/null +++ b/lib/http.go @@ -0,0 +1,34 @@ +package lib + +import ( + "net/http" + "time" + + "github.com/TecharoHQ/anubis" +) + +func ClearCookie(w http.ResponseWriter) { + http.SetCookie(w, &http.Cookie{ + Name: anubis.CookieName, + Value: "", + Expires: time.Now().Add(-1 * time.Hour), + MaxAge: -1, + SameSite: http.SameSiteLaxMode, + }) +} + +// https://github.com/oauth2-proxy/oauth2-proxy/blob/master/pkg/upstream/http.go#L124 +type UnixRoundTripper struct { + Transport *http.Transport +} + +// set bare minimum stuff +func (t UnixRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + if req.Host == "" { + req.Host = "localhost" + } + req.URL.Host = req.Host // proxy error: no Host in request URL + req.URL.Scheme = "http" // make http.Transport happy and avoid an infinite recursion + return t.Transport.RoundTrip(req) +} diff --git a/lib/policy/bot.go b/lib/policy/bot.go new file mode 100644 index 0000000..d9ca135 --- /dev/null +++ b/lib/policy/bot.go @@ -0,0 +1,32 @@ +package policy + +import ( + "fmt" + "regexp" + + "github.com/TecharoHQ/anubis/internal" + "github.com/TecharoHQ/anubis/lib/policy/config" + "github.com/yl2chen/cidranger" +) + +type Bot struct { + Name string + UserAgent *regexp.Regexp + Path *regexp.Regexp + Action config.Rule `json:"action"` + Challenge *config.ChallengeRules + Ranger cidranger.Ranger +} + +func (b Bot) Hash() (string, error) { + var pathRex string + if b.Path != nil { + pathRex = b.Path.String() + } + var userAgentRex string + if b.UserAgent != nil { + userAgentRex = b.UserAgent.String() + } + + return internal.SHA256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex)), nil +} diff --git a/cmd/anubis/internal/config/config.go b/lib/policy/config/config.go similarity index 97% rename from cmd/anubis/internal/config/config.go rename to lib/policy/config/config.go index 56975af..67eddbf 100644 --- a/cmd/anubis/internal/config/config.go +++ b/lib/policy/config/config.go @@ -7,6 +7,17 @@ import ( "regexp" ) +var ( + ErrNoBotRulesDefined = errors.New("config: must define at least one (1) bot rule") + ErrBotMustHaveName = errors.New("config.Bot: must set name") + ErrBotMustHaveUserAgentOrPath = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses") + ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both") + ErrUnknownAction = errors.New("config.Bot: unknown action") + ErrInvalidUserAgentRegex = errors.New("config.Bot: invalid user agent regex") + ErrInvalidPathRegex = errors.New("config.Bot: invalid path regex") + ErrInvalidCIDR = errors.New("config.Bot: invalid CIDR") +) + type Rule string const ( @@ -24,7 +35,7 @@ const ( AlgorithmSlow Algorithm = "slow" ) -type Bot struct { +type BotConfig struct { Name string `json:"name"` UserAgentRegex *string `json:"user_agent_regex"` PathRegex *string `json:"path_regex"` @@ -33,18 +44,7 @@ type Bot struct { Challenge *ChallengeRules `json:"challenge,omitempty"` } -var ( - ErrNoBotRulesDefined = errors.New("config: must define at least one (1) bot rule") - ErrBotMustHaveName = errors.New("config.Bot: must set name") - ErrBotMustHaveUserAgentOrPath = errors.New("config.Bot: must set either user_agent_regex, path_regex, or remote_addresses") - ErrBotMustHaveUserAgentOrPathNotBoth = errors.New("config.Bot: must set either user_agent_regex, path_regex, and not both") - ErrUnknownAction = errors.New("config.Bot: unknown action") - ErrInvalidUserAgentRegex = errors.New("config.Bot: invalid user agent regex") - ErrInvalidPathRegex = errors.New("config.Bot: invalid path regex") - ErrInvalidCIDR = errors.New("config.Bot: invalid CIDR") -) - -func (b Bot) Valid() error { +func (b BotConfig) Valid() error { var errs []error if b.Name == "" { @@ -137,8 +137,8 @@ func (cr ChallengeRules) Valid() error { } type Config struct { - Bots []Bot `json:"bots"` - DNSBL bool `json:"dnsbl"` + Bots []BotConfig `json:"bots"` + DNSBL bool `json:"dnsbl"` } func (c Config) Valid() error { diff --git a/cmd/anubis/internal/config/config_test.go b/lib/policy/config/config_test.go similarity index 94% rename from cmd/anubis/internal/config/config_test.go rename to lib/policy/config/config_test.go index 9865860..a169087 100644 --- a/cmd/anubis/internal/config/config_test.go +++ b/lib/policy/config/config_test.go @@ -13,12 +13,12 @@ func p[V any](v V) *V { return &v } func TestBotValid(t *testing.T) { var tests = []struct { name string - bot Bot + bot BotConfig err error }{ { name: "simple user agent", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, UserAgentRegex: p("Mozilla"), @@ -27,7 +27,7 @@ func TestBotValid(t *testing.T) { }, { name: "simple path", - bot: Bot{ + bot: BotConfig{ Name: "well-known-path", Action: RuleAllow, PathRegex: p("^/.well-known/.*$"), @@ -36,7 +36,7 @@ func TestBotValid(t *testing.T) { }, { name: "no rule name", - bot: Bot{ + bot: BotConfig{ Action: RuleChallenge, UserAgentRegex: p("Mozilla"), }, @@ -44,7 +44,7 @@ func TestBotValid(t *testing.T) { }, { name: "no rule matcher", - bot: Bot{ + bot: BotConfig{ Name: "broken-rule", Action: RuleAllow, }, @@ -52,7 +52,7 @@ func TestBotValid(t *testing.T) { }, { name: "both user-agent and path", - bot: Bot{ + bot: BotConfig{ Name: "path-and-user-agent", Action: RuleDeny, UserAgentRegex: p("Mozilla"), @@ -62,7 +62,7 @@ func TestBotValid(t *testing.T) { }, { name: "unknown action", - bot: Bot{ + bot: BotConfig{ Name: "Unknown action", Action: RuleUnknown, UserAgentRegex: p("Mozilla"), @@ -71,7 +71,7 @@ func TestBotValid(t *testing.T) { }, { name: "invalid user agent regex", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, UserAgentRegex: p("a(b"), @@ -80,7 +80,7 @@ func TestBotValid(t *testing.T) { }, { name: "invalid path regex", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, PathRegex: p("a(b"), @@ -89,7 +89,7 @@ func TestBotValid(t *testing.T) { }, { name: "challenge difficulty too low", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, PathRegex: p("Mozilla"), @@ -103,7 +103,7 @@ func TestBotValid(t *testing.T) { }, { name: "challenge difficulty too high", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, PathRegex: p("Mozilla"), @@ -117,7 +117,7 @@ func TestBotValid(t *testing.T) { }, { name: "challenge wrong algorithm", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleChallenge, PathRegex: p("Mozilla"), @@ -131,7 +131,7 @@ func TestBotValid(t *testing.T) { }, { name: "invalid cidr range", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleAllow, RemoteAddr: []string{"0.0.0.0/33"}, @@ -140,7 +140,7 @@ func TestBotValid(t *testing.T) { }, { name: "only filter by IP range", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleAllow, RemoteAddr: []string{"0.0.0.0/0"}, @@ -149,7 +149,7 @@ func TestBotValid(t *testing.T) { }, { name: "filter by user agent and IP range", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleAllow, UserAgentRegex: p("Mozilla"), @@ -159,7 +159,7 @@ func TestBotValid(t *testing.T) { }, { name: "filter by path and IP range", - bot: Bot{ + bot: BotConfig{ Name: "mozilla-ua", Action: RuleAllow, PathRegex: p("^.*$"), diff --git a/cmd/anubis/internal/config/testdata/bad/badregexes.json b/lib/policy/config/testdata/bad/badregexes.json similarity index 100% rename from cmd/anubis/internal/config/testdata/bad/badregexes.json rename to lib/policy/config/testdata/bad/badregexes.json diff --git a/cmd/anubis/internal/config/testdata/bad/invalid.json b/lib/policy/config/testdata/bad/invalid.json similarity index 100% rename from cmd/anubis/internal/config/testdata/bad/invalid.json rename to lib/policy/config/testdata/bad/invalid.json diff --git a/cmd/anubis/internal/config/testdata/bad/nobots.json b/lib/policy/config/testdata/bad/nobots.json similarity index 100% rename from cmd/anubis/internal/config/testdata/bad/nobots.json rename to lib/policy/config/testdata/bad/nobots.json diff --git a/cmd/anubis/internal/config/testdata/good/allow_everyone.json b/lib/policy/config/testdata/good/allow_everyone.json similarity index 100% rename from cmd/anubis/internal/config/testdata/good/allow_everyone.json rename to lib/policy/config/testdata/good/allow_everyone.json diff --git a/cmd/anubis/internal/config/testdata/good/challengemozilla.json b/lib/policy/config/testdata/good/challengemozilla.json similarity index 100% rename from cmd/anubis/internal/config/testdata/good/challengemozilla.json rename to lib/policy/config/testdata/good/challengemozilla.json diff --git a/cmd/anubis/internal/config/testdata/good/everything_blocked.json b/lib/policy/config/testdata/good/everything_blocked.json similarity index 100% rename from cmd/anubis/internal/config/testdata/good/everything_blocked.json rename to lib/policy/config/testdata/good/everything_blocked.json diff --git a/lib/policy/policy.go b/lib/policy/policy.go new file mode 100644 index 0000000..51b23ff --- /dev/null +++ b/lib/policy/policy.go @@ -0,0 +1,122 @@ +package policy + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "regexp" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/yl2chen/cidranger" + + "github.com/TecharoHQ/anubis/lib/policy/config" +) + +var ( + PolicyApplications = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "anubis_policy_results", + Help: "The results of each policy rule", + }, []string{"rule", "action"}) +) + +type ParsedConfig struct { + orig config.Config + + Bots []Bot + DNSBL bool + DefaultDifficulty int +} + +func NewParsedConfig(orig config.Config) *ParsedConfig { + return &ParsedConfig{ + orig: orig, + } +} + +func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { + var c config.Config + if err := json.NewDecoder(fin).Decode(&c); err != nil { + return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err) + } + + if err := c.Valid(); err != nil { + return nil, err + } + + var err error + + result := NewParsedConfig(c) + result.DefaultDifficulty = defaultDifficulty + + for _, b := range c.Bots { + if berr := b.Valid(); berr != nil { + err = errors.Join(err, berr) + continue + } + + var botParseErr error + parsedBot := Bot{ + Name: b.Name, + Action: b.Action, + } + + if b.RemoteAddr != nil && len(b.RemoteAddr) > 0 { + parsedBot.Ranger = cidranger.NewPCTrieRanger() + + for _, cidr := range b.RemoteAddr { + _, rng, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("[unexpected] range %s not parsing: %w", cidr, err) + } + + parsedBot.Ranger.Insert(cidranger.NewBasicRangerEntry(*rng)) + } + } + + if b.UserAgentRegex != nil { + userAgent, err := regexp.Compile(*b.UserAgentRegex) + if err != nil { + botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling user agent regexp: %w", err)) + continue + } else { + parsedBot.UserAgent = userAgent + } + } + + if b.PathRegex != nil { + path, err := regexp.Compile(*b.PathRegex) + if err != nil { + botParseErr = errors.Join(botParseErr, fmt.Errorf("while compiling path regexp: %w", err)) + continue + } else { + parsedBot.Path = path + } + } + + if b.Challenge == nil { + parsedBot.Challenge = &config.ChallengeRules{ + Difficulty: defaultDifficulty, + ReportAs: defaultDifficulty, + Algorithm: config.AlgorithmFast, + } + } else { + parsedBot.Challenge = b.Challenge + if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown { + parsedBot.Challenge.Algorithm = config.AlgorithmFast + } + } + + result.Bots = append(result.Bots, parsedBot) + } + + if err != nil { + return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, err) + } + + result.DNSBL = c.DNSBL + + return result, nil +} diff --git a/cmd/anubis/policy_test.go b/lib/policy/policy_test.go similarity index 52% rename from cmd/anubis/policy_test.go rename to lib/policy/policy_test.go index cf8ef2e..16ca9c7 100644 --- a/cmd/anubis/policy_test.go +++ b/lib/policy/policy_test.go @@ -1,25 +1,28 @@ -package main +package policy import ( "os" "path/filepath" "testing" + + "github.com/TecharoHQ/anubis" + "github.com/TecharoHQ/anubis/data" ) func TestDefaultPolicyMustParse(t *testing.T) { - fin, err := static.Open("botPolicies.json") + fin, err := data.BotPolicies.Open("botPolicies.json") if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := parseConfig(fin, "botPolicies.json", defaultDifficulty); err != nil { + if _, err := ParseConfig(fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil { t.Fatalf("can't parse config: %v", err) } } func TestGoodConfigs(t *testing.T) { - finfos, err := os.ReadDir("internal/config/testdata/good") + finfos, err := os.ReadDir("config/testdata/good") if err != nil { t.Fatal(err) } @@ -27,13 +30,13 @@ func TestGoodConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - fin, err := os.Open(filepath.Join("internal", "config", "testdata", "good", st.Name())) + fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err != nil { + if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err != nil { t.Fatal(err) } }) @@ -41,7 +44,7 @@ func TestGoodConfigs(t *testing.T) { } func TestBadConfigs(t *testing.T) { - finfos, err := os.ReadDir("internal/config/testdata/bad") + finfos, err := os.ReadDir("config/testdata/bad") if err != nil { t.Fatal(err) } @@ -49,13 +52,13 @@ func TestBadConfigs(t *testing.T) { for _, st := range finfos { st := st t.Run(st.Name(), func(t *testing.T) { - fin, err := os.Open(filepath.Join("internal", "config", "testdata", "bad", st.Name())) + fin, err := os.Open(filepath.Join("config", "testdata", "bad", st.Name())) if err != nil { t.Fatal(err) } defer fin.Close() - if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err == nil { + if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err == nil { t.Fatal(err) } else { t.Log(err) diff --git a/lib/random.go b/lib/random.go new file mode 100644 index 0000000..79cded4 --- /dev/null +++ b/lib/random.go @@ -0,0 +1,9 @@ +package lib + +import ( + "math/rand" +) + +func randomJitter() bool { + return rand.Intn(100) > 10 +} diff --git a/main b/main new file mode 100755 index 0000000..3c67613 Binary files /dev/null and b/main differ diff --git a/web/embed.go b/web/embed.go new file mode 100644 index 0000000..6f5902f --- /dev/null +++ b/web/embed.go @@ -0,0 +1,14 @@ +package web + +import "embed" + +//go:generate go tool github.com/a-h/templ/cmd/templ generate +//go:generate esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs +//go:generate gzip -f -k static/js/main.mjs +//go:generate zstd -f -k --ultra -22 static/js/main.mjs +//go:generate brotli -fZk static/js/main.mjs + +var ( + //go:embed static + Static embed.FS +) diff --git a/web/index.go b/web/index.go new file mode 100644 index 0000000..7057cc8 --- /dev/null +++ b/web/index.go @@ -0,0 +1,15 @@ +package web + +import "github.com/a-h/templ" + +func Base(title string, body templ.Component) templ.Component { + return base(title, body) +} + +func Index() templ.Component { + return index() +} + +func ErrorPage(msg string) templ.Component { + return errorPage(msg) +} diff --git a/cmd/anubis/index.templ b/web/index.templ similarity index 99% rename from cmd/anubis/index.templ rename to web/index.templ index 79ae005..ca6086c 100644 --- a/cmd/anubis/index.templ +++ b/web/index.templ @@ -1,4 +1,4 @@ -package main +package web import ( "github.com/TecharoHQ/anubis" diff --git a/cmd/anubis/index_templ.go b/web/index_templ.go similarity index 99% rename from cmd/anubis/index_templ.go rename to web/index_templ.go index c3755c5..e647132 100644 --- a/cmd/anubis/index_templ.go +++ b/web/index_templ.go @@ -1,7 +1,7 @@ // Code generated by templ - DO NOT EDIT. // templ: version: v0.3.833 -package main +package web //lint:file-ignore SA4006 This context is only used if a nested component is present. diff --git a/cmd/anubis/js/main.mjs b/web/js/main.mjs similarity index 100% rename from cmd/anubis/js/main.mjs rename to web/js/main.mjs diff --git a/cmd/anubis/js/proof-of-work-slow.mjs b/web/js/proof-of-work-slow.mjs similarity index 100% rename from cmd/anubis/js/proof-of-work-slow.mjs rename to web/js/proof-of-work-slow.mjs diff --git a/cmd/anubis/js/proof-of-work.mjs b/web/js/proof-of-work.mjs similarity index 100% rename from cmd/anubis/js/proof-of-work.mjs rename to web/js/proof-of-work.mjs diff --git a/cmd/anubis/js/video.mjs b/web/js/video.mjs similarity index 100% rename from cmd/anubis/js/video.mjs rename to web/js/video.mjs diff --git a/cmd/anubis/static/img/happy.webp b/web/static/img/happy.webp similarity index 100% rename from cmd/anubis/static/img/happy.webp rename to web/static/img/happy.webp diff --git a/cmd/anubis/static/img/pensive.webp b/web/static/img/pensive.webp similarity index 100% rename from cmd/anubis/static/img/pensive.webp rename to web/static/img/pensive.webp diff --git a/cmd/anubis/static/img/sad.webp b/web/static/img/sad.webp similarity index 100% rename from cmd/anubis/static/img/sad.webp rename to web/static/img/sad.webp diff --git a/cmd/anubis/static/js/main.mjs b/web/static/js/main.mjs similarity index 100% rename from cmd/anubis/static/js/main.mjs rename to web/static/js/main.mjs diff --git a/cmd/anubis/static/js/main.mjs.br b/web/static/js/main.mjs.br similarity index 100% rename from cmd/anubis/static/js/main.mjs.br rename to web/static/js/main.mjs.br diff --git a/cmd/anubis/static/js/main.mjs.gz b/web/static/js/main.mjs.gz similarity index 95% rename from cmd/anubis/static/js/main.mjs.gz rename to web/static/js/main.mjs.gz index 0e8d735..c88b342 100644 Binary files a/cmd/anubis/static/js/main.mjs.gz and b/web/static/js/main.mjs.gz differ diff --git a/cmd/anubis/static/js/main.mjs.map b/web/static/js/main.mjs.map similarity index 100% rename from cmd/anubis/static/js/main.mjs.map rename to web/static/js/main.mjs.map diff --git a/cmd/anubis/static/js/main.mjs.zst b/web/static/js/main.mjs.zst similarity index 100% rename from cmd/anubis/static/js/main.mjs.zst rename to web/static/js/main.mjs.zst diff --git a/cmd/anubis/static/robots.txt b/web/static/robots.txt similarity index 100% rename from cmd/anubis/static/robots.txt rename to web/static/robots.txt diff --git a/cmd/anubis/static/testdata/black.mp4 b/web/static/testdata/black.mp4 similarity index 100% rename from cmd/anubis/static/testdata/black.mp4 rename to web/static/testdata/black.mp4 diff --git a/xess/xess_templ.go b/xess/xess_templ.go index a01a382..d8cab57 100644 --- a/xess/xess_templ.go +++ b/xess/xess_templ.go @@ -1,6 +1,6 @@ // Code generated by templ - DO NOT EDIT. -// templ: version: v0.3.833 +// templ: version: v0.3.850 package xess //lint:file-ignore SA4006 This context is only used if a nested component is present.