cmd/anubis: configurable difficulty per-bot rule (#53)

Closes #30

Introduces the "challenge" field in bot rule definitions:

```json
{
  "name": "generic-bot-catchall",
  "user_agent_regex": "(?i:bot|crawler)",
  "action": "CHALLENGE",
  "challenge": {
    "difficulty": 16,
    "report_as": 4,
    "algorithm": "slow"
  }
}
```

This makes Anubis return a challenge page for every user agent with
"bot" or "crawler" in it (case-insensitively) with difficulty 16 using
the old "slow" algorithm but reporting in the client as difficulty 4.

This is useful when you want to make certain clients in particular
suffer.

Additional validation and testing logic has been added to make sure
that users do not define "impossible" challenge settings.

If no algorithm is specified, Anubis defaults to the "fast" algorithm.

Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-03-21 13:48:00 -04:00 committed by GitHub
parent 90049001e9
commit d3e509517c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 311 additions and 46 deletions

View file

@ -65,6 +65,16 @@
"user_agent_regex": "HeadlessChromium",
"action": "DENY"
},
{
"name": "generic-bot-catchall",
"user_agent_regex": "(?i:bot|crawler)",
"action": "CHALLENGE",
"challenge": {
"difficulty": 16,
"report_as": 4,
"algorithm": "slow"
}
},
{
"name": "generic-browser",
"user_agent_regex": "Mozilla",
@ -72,4 +82,4 @@
}
],
"dnsbl": true
}
}

View file

@ -9,17 +9,26 @@ import (
type Rule string
const (
RuleUnknown = ""
RuleAllow = "ALLOW"
RuleDeny = "DENY"
RuleChallenge = "CHALLENGE"
RuleUnknown Rule = ""
RuleAllow Rule = "ALLOW"
RuleDeny Rule = "DENY"
RuleChallenge Rule = "CHALLENGE"
)
type Algorithm string
const (
AlgorithmUnknown Algorithm = ""
AlgorithmFast Algorithm = "fast"
AlgorithmSlow Algorithm = "slow"
)
type Bot struct {
Name string `json:"name"`
UserAgentRegex *string `json:"user_agent_regex"`
PathRegex *string `json:"path_regex"`
Action Rule `json:"action"`
Name string `json:"name"`
UserAgentRegex *string `json:"user_agent_regex"`
PathRegex *string `json:"path_regex"`
Action Rule `json:"action"`
Challenge *ChallengeRules `json:"challenge,omitempty"`
}
var (
@ -66,6 +75,12 @@ func (b Bot) Valid() error {
errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownAction, b.Action))
}
if b.Action == RuleChallenge && b.Challenge != nil {
if err := b.Challenge.Valid(); err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return fmt.Errorf("config: bot entry for %q is not valid:\n%w", b.Name, errors.Join(errs...))
}
@ -73,6 +88,43 @@ func (b Bot) Valid() error {
return nil
}
type ChallengeRules struct {
Difficulty int `json:"difficulty"`
ReportAs int `json:"report_as"`
Algorithm Algorithm `json:"algorithm"`
}
var (
ErrChallengeRuleHasWrongAlgorithm = errors.New("config.Bot.ChallengeRules: algorithm is invalid")
ErrChallengeDifficultyTooLow = errors.New("config.Bot.ChallengeRules: difficulty is too low (must be >= 1)")
ErrChallengeDifficultyTooHigh = errors.New("config.Bot.ChallengeRules: difficulty is too high (must be <= 64)")
)
func (cr ChallengeRules) Valid() error {
var errs []error
if cr.Difficulty < 1 {
errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooLow, cr.Difficulty))
}
if cr.Difficulty > 64 {
errs = append(errs, fmt.Errorf("%w, got: %d", ErrChallengeDifficultyTooHigh, cr.Difficulty))
}
switch cr.Algorithm {
case AlgorithmFast, AlgorithmSlow, AlgorithmUnknown:
// do nothing, it's all good
default:
errs = append(errs, fmt.Errorf("%w: %q", ErrChallengeRuleHasWrongAlgorithm, cr.Algorithm))
}
if len(errs) != 0 {
return fmt.Errorf("config: challenge rules entry is not valid:\n%w", errors.Join(errs...))
}
return nil
}
type Config struct {
Bots []Bot `json:"bots"`
DNSBL bool `json:"dnsbl"`

View file

@ -87,6 +87,48 @@ func TestBotValid(t *testing.T) {
},
err: ErrInvalidPathRegex,
},
{
name: "challenge difficulty too low",
bot: Bot{
Name: "mozilla-ua",
Action: RuleChallenge,
PathRegex: p("Mozilla"),
Challenge: &ChallengeRules{
Difficulty: 0,
ReportAs: 4,
Algorithm: "fast",
},
},
err: ErrChallengeDifficultyTooLow,
},
{
name: "challenge difficulty too high",
bot: Bot{
Name: "mozilla-ua",
Action: RuleChallenge,
PathRegex: p("Mozilla"),
Challenge: &ChallengeRules{
Difficulty: 420,
ReportAs: 4,
Algorithm: "fast",
},
},
err: ErrChallengeDifficultyTooHigh,
},
{
name: "challenge wrong algorithm",
bot: Bot{
Name: "mozilla-ua",
Action: RuleChallenge,
PathRegex: p("Mozilla"),
Challenge: &ChallengeRules{
Difficulty: 420,
ReportAs: 4,
Algorithm: "high quality rips",
},
},
err: ErrChallengeRuleHasWrongAlgorithm,
},
}
for _, cs := range tests {

View file

@ -1,5 +1,11 @@
import { process } from './proof-of-work.mjs';
import { testVideo } from './video.mjs';
import processFast from "./proof-of-work.mjs";
import processSlow from "./proof-of-work-slow.mjs";
import { testVideo } from "./video.mjs";
const algorithms = {
"fast": processFast,
"slow": processSlow,
}
// from Xeact
const u = (url = "", params = {}) => {
@ -37,7 +43,7 @@ const imageURL = (mood, cacheBuster) =>
status.innerHTML = 'Calculating...';
const { challenge, difficulty } = await fetch("/.within.website/x/cmd/anubis/api/make-challenge", { method: "POST" })
const { challenge, rules } = await fetch("/.within.website/x/cmd/anubis/api/make-challenge", { method: "POST" })
.then(r => {
if (!r.ok) {
throw new Error("Failed to fetch config");
@ -47,16 +53,26 @@ const imageURL = (mood, cacheBuster) =>
.catch(err => {
title.innerHTML = "Oh no!";
status.innerHTML = `Failed to fetch config: ${err.message}`;
image.src = imageURL("sad");
image.src = imageURL("sad", anubisVersion);
spinner.innerHTML = "";
spinner.style.display = "none";
throw err;
});
status.innerHTML = `Calculating...<br/>Difficulty: ${difficulty}`;
const process = algorithms[rules.algorithm];
if (!process) {
title.innerHTML = "Oh no!";
status.innerHTML = `Failed to resolve check algorithm. You may want to reload the page.`;
image.src = imageURL("sad", anubisVersion);
spinner.innerHTML = "";
spinner.style.display = "none";
return;
}
status.innerHTML = `Calculating...<br/>Difficulty: ${rules.report_as}`;
const t0 = Date.now();
const { hash, nonce } = await process(challenge, difficulty);
const { hash, nonce } = await process(challenge, rules.difficulty);
const t1 = Date.now();
console.log({ hash, nonce });

View file

@ -0,0 +1,63 @@
// https://dev.to/ratmd/simple-proof-of-work-in-javascript-3kgm
export default function process(data, difficulty = 5, _threads = 1) {
console.debug("slow algo");
return new Promise((resolve, reject) => {
let webWorkerURL = URL.createObjectURL(new Blob([
'(', processTask(), ')()'
], { type: 'application/javascript' }));
let worker = new Worker(webWorkerURL);
worker.onmessage = (event) => {
worker.terminate();
resolve(event.data);
};
worker.onerror = (event) => {
worker.terminate();
reject();
};
worker.postMessage({
data,
difficulty
});
URL.revokeObjectURL(webWorkerURL);
});
}
function processTask() {
return function () {
const sha256 = (text) => {
const encoded = new TextEncoder().encode(text);
return crypto.subtle.digest("SHA-256", encoded.buffer)
.then((result) =>
Array.from(new Uint8Array(result))
.map((c) => c.toString(16).padStart(2, "0"))
.join(""),
);
};
addEventListener('message', async (event) => {
let data = event.data.data;
let difficulty = event.data.difficulty;
let hash;
let nonce = 0;
do {
hash = await sha256(data + nonce++);
} while (hash.substring(0, difficulty) !== Array(difficulty + 1).join('0'));
nonce -= 1; // last nonce was post-incremented
postMessage({
hash,
data,
difficulty,
nonce,
});
});
}.toString();
}

View file

@ -1,6 +1,5 @@
// https://dev.to/ratmd/simple-proof-of-work-in-javascript-3kgm
export function process(data, difficulty = 5, threads = (navigator.hardwareConcurrency || 1)) {
export default function process(data, difficulty = 5, threads = (navigator.hardwareConcurrency || 1)) {
console.debug("fast algo");
return new Promise((resolve, reject) => {
let webWorkerURL = URL.createObjectURL(new Blob([
'(', processTask(), ')()'

View file

@ -44,7 +44,7 @@ import (
var (
bind = flag.String("bind", ":8923", "network address to bind HTTP to")
bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
challengeDifficulty = flag.Int("difficulty", 4, "difficulty of the challenge")
challengeDifficulty = flag.Int("difficulty", defaultDifficulty, "difficulty of the challenge")
metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
socketMode = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
@ -85,8 +85,9 @@ var (
)
const (
cookieName = "within.website-x-cmd-anubis-auth"
staticPath = "/.within.website/x/cmd/anubis/"
cookieName = "within.website-x-cmd-anubis-auth"
staticPath = "/.within.website/x/cmd/anubis/"
defaultDifficulty = 4
)
//go:generate go tool github.com/a-h/templ/cmd/templ generate
@ -261,7 +262,7 @@ func sha256sum(text string) (string, error) {
return hex.EncodeToString(hash.Sum(nil)), nil
}
func (s *Server) challengeFor(r *http.Request) string {
func (s *Server) challengeFor(r *http.Request, difficulty int) string {
fp := sha256.Sum256(s.priv.Seed())
data := fmt.Sprintf(
@ -271,7 +272,7 @@ func (s *Server) challengeFor(r *http.Request) string {
r.UserAgent(),
time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
fp,
*challengeDifficulty,
difficulty,
)
result, _ := sha256sum(data)
return result
@ -324,7 +325,7 @@ func New(target, policyFname string) (*Server, error) {
defer fin.Close()
policy, err := parseConfig(fin, policyFname)
policy, err := parseConfig(fin, policyFname, *challengeDifficulty)
if err != nil {
return nil, err // parseConfig sets a fancy error for us
}
@ -485,7 +486,9 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) {
return
}
if claims["challenge"] != s.challengeFor(r) {
challenge := s.challengeFor(r, rule.Challenge.Difficulty)
if claims["challenge"] != challenge {
lg.Debug("invalid challenge", "path", r.URL.Path)
clearCookie(w)
s.renderIndex(w, r)
@ -498,7 +501,7 @@ func (s *Server) maybeReverseProxy(w http.ResponseWriter, r *http.Request) {
nonce = int(v)
}
calcString := fmt.Sprintf("%s%d", s.challengeFor(r), nonce)
calcString := fmt.Sprintf("%s%d", challenge, nonce)
calculated, err := sha256sum(calcString)
if err != nil {
lg.Error("failed to calculate sha256sum", "path", r.URL.Path, "err", err)
@ -527,24 +530,32 @@ func (s *Server) renderIndex(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) makeChallenge(w http.ResponseWriter, r *http.Request) {
challenge := s.challengeFor(r)
difficulty := *challengeDifficulty
cr, rule := s.check(r)
challenge := s.challengeFor(r, rule.Challenge.Difficulty)
lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
json.NewEncoder(w).Encode(struct {
Challenge string `json:"challenge"`
Difficulty int `json:"difficulty"`
Challenge string `json:"challenge"`
Rules *config.ChallengeRules `json:"rules"`
}{
Challenge: challenge,
Difficulty: difficulty,
Challenge: challenge,
Rules: rule.Challenge,
})
lg.Debug("made challenge", "challenge", challenge, "difficulty", difficulty)
lg.Debug("made challenge", "challenge", challenge, "rules", rule.Challenge, "cr", cr)
challengesIssued.Inc()
}
func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) {
lg := slog.With("user_agent", r.UserAgent(), "accept_language", r.Header.Get("Accept-Language"), "priority", r.Header.Get("Priority"), "x-forwarded-for", r.Header.Get("X-Forwarded-For"), "x-real-ip", r.Header.Get("X-Real-Ip"))
cr, rule := s.check(r)
lg := slog.With(
"user_agent", r.UserAgent(),
"accept_language", r.Header.Get("Accept-Language"),
"priority", r.Header.Get("Priority"),
"x-forwarded-for", r.Header.Get("X-Forwarded-For"),
"x-real-ip", r.Header.Get("X-Real-Ip"),
"cr", cr,
)
nonceStr := r.FormValue("nonce")
if nonceStr == "" {
@ -576,7 +587,7 @@ func (s *Server) passChallenge(w http.ResponseWriter, r *http.Request) {
response := r.FormValue("response")
redir := r.FormValue("redir")
challenge := s.challengeFor(r)
challenge := s.challengeFor(r, rule.Challenge.Difficulty)
nonce, err := strconv.Atoi(nonceStr)
if err != nil {

View file

@ -32,7 +32,8 @@ type Bot struct {
Name string
UserAgent *regexp.Regexp
Path *regexp.Regexp
Action config.Rule `json:"action"`
Action config.Rule
Challenge *config.ChallengeRules
}
func (b Bot) Hash() (string, error) {
@ -48,7 +49,7 @@ func (b Bot) Hash() (string, error) {
return sha256sum(fmt.Sprintf("%s::%s::%s", b.Name, pathRex, userAgentRex))
}
func parseConfig(fin io.Reader, fname string) (*ParsedConfig, error) {
func parseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
var c config.Config
if err := json.NewDecoder(fin).Decode(&c); err != nil {
return nil, fmt.Errorf("can't parse policy config JSON %s: %w", fname, err)
@ -96,6 +97,19 @@ func parseConfig(fin io.Reader, fname string) (*ParsedConfig, error) {
}
}
if b.Challenge == nil {
parsedBot.Challenge = &config.ChallengeRules{
Difficulty: defaultDifficulty,
ReportAs: defaultDifficulty,
Algorithm: config.AlgorithmFast,
}
} else {
parsedBot.Challenge = b.Challenge
if parsedBot.Challenge.Algorithm == config.AlgorithmUnknown {
parsedBot.Challenge.Algorithm = config.AlgorithmFast
}
}
result.Bots = append(result.Bots, parsedBot)
}
@ -142,5 +156,11 @@ func (s *Server) check(r *http.Request) (CheckResult, *Bot) {
}
}
return cr("default/allow", config.RuleAllow), nil
return cr("default/allow", config.RuleAllow), &Bot{
Challenge: &config.ChallengeRules{
Difficulty: defaultDifficulty,
ReportAs: defaultDifficulty,
Algorithm: config.AlgorithmFast,
},
}
}

View file

@ -13,7 +13,7 @@ func TestDefaultPolicyMustParse(t *testing.T) {
}
defer fin.Close()
if _, err := parseConfig(fin, "botPolicies.json"); err != nil {
if _, err := parseConfig(fin, "botPolicies.json", defaultDifficulty); err != nil {
t.Fatalf("can't parse config: %v", err)
}
}
@ -33,7 +33,7 @@ func TestGoodConfigs(t *testing.T) {
}
defer fin.Close()
if _, err := parseConfig(fin, fin.Name()); err != nil {
if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err != nil {
t.Fatal(err)
}
})
@ -55,7 +55,7 @@ func TestBadConfigs(t *testing.T) {
}
defer fin.Close()
if _, err := parseConfig(fin, fin.Name()); err == nil {
if _, err := parseConfig(fin, fin.Name(), defaultDifficulty); err == nil {
t.Fatal(err)
} else {
t.Log(err)

View file

@ -1,2 +1,2 @@
(()=>{function m(n,s=5,e=(navigator.hardwareConcurrency||1)){return new Promise((t,l)=>{let a=URL.createObjectURL(new Blob(["(",g(),")()"],{type:"application/javascript"})),r=[];for(let d=0;d<e;d++){let i=new Worker(a);i.onmessage=c=>{r.forEach(u=>u.terminate()),i.terminate(),t(c.data)},i.onerror=c=>{i.terminate(),l()},i.postMessage({data:n,difficulty:s,nonce:d,threads:e}),r.push(i)}URL.revokeObjectURL(a)})}function g(){return function(){let n=e=>{let t=new TextEncoder().encode(e);return crypto.subtle.digest("SHA-256",t.buffer)};function s(e){return Array.from(e).map(t=>t.toString(16).padStart(2,"0")).join("")}addEventListener("message",async e=>{let t=e.data.data,l=e.data.difficulty,a,r=e.data.nonce,d=e.data.threads;for(;;){let i=await n(t+r),c=new Uint8Array(i),u=!0;for(let o=0;o<l;o++){let f=Math.floor(o/2),p=o%2;if((c[f]>>(p===0?4:0)&15)!==0){u=!1;break}}if(u){a=s(c),console.log(a);break}r+=d}postMessage({hash:a,data:t,difficulty:l,nonce:r})})}.toString()}var w=(n="",s={})=>{let e=new URL(n,window.location.href);return Object.entries(s).forEach(t=>{let[l,a]=t;e.searchParams.set(l,a)}),e.toString()},h=(n,s)=>w(`/.within.website/x/cmd/anubis/static/img/${n}.webp`,{cacheBuster:s});(async()=>{let n=document.getElementById("status"),s=document.getElementById("image"),e=document.getElementById("title"),t=document.getElementById("spinner"),l=JSON.parse(document.getElementById("anubis_version").textContent);n.innerHTML="Calculating...";let{challenge:a,difficulty:r}=await fetch("/.within.website/x/cmd/anubis/api/make-challenge",{method:"POST"}).then(o=>{if(!o.ok)throw new Error("Failed to fetch config");return o.json()}).catch(o=>{throw e.innerHTML="Oh no!",n.innerHTML=`Failed to fetch config: ${o.message}`,s.src=h("sad"),t.innerHTML="",t.style.display="none",o});n.innerHTML=`Calculating...<br/>Difficulty: ${r}`;let d=Date.now(),{hash:i,nonce:c}=await m(a,r),u=Date.now();console.log({hash:i,nonce:c}),e.innerHTML="Success!",n.innerHTML=`Done! Took ${u-d}ms, ${c} iterations`,s.src=h("happy",l),t.innerHTML="",t.style.display="none",setTimeout(()=>{let o=window.location.href;window.location.href=w("/.within.website/x/cmd/anubis/api/pass-challenge",{response:i,nonce:c,redir:o,elapsedTime:u-d})},250)})();})();
(()=>{function p(r,n=5,t=navigator.hardwareConcurrency||1){return console.debug("fast algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",y(),")()"],{type:"application/javascript"})),a=[];for(let i=0;i<t;i++){let c=new Worker(s);c.onmessage=d=>{a.forEach(u=>u.terminate()),c.terminate(),e(d.data)},c.onerror=d=>{c.terminate(),o()},c.postMessage({data:r,difficulty:n,nonce:i,threads:t}),a.push(c)}URL.revokeObjectURL(s)})}function y(){return function(){let r=t=>{let e=new TextEncoder().encode(t);return crypto.subtle.digest("SHA-256",e.buffer)};function n(t){return Array.from(t).map(e=>e.toString(16).padStart(2,"0")).join("")}addEventListener("message",async t=>{let e=t.data.data,o=t.data.difficulty,s,a=t.data.nonce,i=t.data.threads;for(;;){let c=await r(e+a),d=new Uint8Array(c),u=!0;for(let m=0;m<o;m++){let l=Math.floor(m/2),g=m%2;if((d[l]>>(g===0?4:0)&15)!==0){u=!1;break}}if(u){s=n(d),console.log(s);break}a+=i}postMessage({hash:s,data:e,difficulty:o,nonce:a})})}.toString()}function f(r,n=5,t=1){return console.debug("slow algo"),new Promise((e,o)=>{let s=URL.createObjectURL(new Blob(["(",b(),")()"],{type:"application/javascript"})),a=new Worker(s);a.onmessage=i=>{a.terminate(),e(i.data)},a.onerror=i=>{a.terminate(),o()},a.postMessage({data:r,difficulty:n}),URL.revokeObjectURL(s)})}function b(){return function(){let r=n=>{let t=new TextEncoder().encode(n);return crypto.subtle.digest("SHA-256",t.buffer).then(e=>Array.from(new Uint8Array(e)).map(o=>o.toString(16).padStart(2,"0")).join(""))};addEventListener("message",async n=>{let t=n.data.data,e=n.data.difficulty,o,s=0;do o=await r(t+s++);while(o.substring(0,e)!==Array(e+1).join("0"));s-=1,postMessage({hash:o,data:t,difficulty:e,nonce:s})})}.toString()}var L={fast:p,slow:f},w=(r="",n={})=>{let t=new URL(r,window.location.href);return Object.entries(n).forEach(e=>{let[o,s]=e;t.searchParams.set(o,s)}),t.toString()},h=(r,n)=>w(`/.within.website/x/cmd/anubis/static/img/${r}.webp`,{cacheBuster:n});(async()=>{let r=document.getElementById("status"),n=document.getElementById("image"),t=document.getElementById("title"),e=document.getElementById("spinner"),o=JSON.parse(document.getElementById("anubis_version").textContent);r.innerHTML="Calculating...";let{challenge:s,rules:a}=await fetch("/.within.website/x/cmd/anubis/api/make-challenge",{method:"POST"}).then(l=>{if(!l.ok)throw new Error("Failed to fetch config");return l.json()}).catch(l=>{throw t.innerHTML="Oh no!",r.innerHTML=`Failed to fetch config: ${l.message}`,n.src=h("sad",o),e.innerHTML="",e.style.display="none",l}),i=L[a.algorithm];if(!i){t.innerHTML="Oh no!",r.innerHTML="Failed to resolve check algorithm. You may want to reload the page.",n.src=h("sad",o),e.innerHTML="",e.style.display="none";return}r.innerHTML=`Calculating...<br/>Difficulty: ${a.report_as}`;let c=Date.now(),{hash:d,nonce:u}=await i(s,a.difficulty),m=Date.now();console.log({hash:d,nonce:u}),t.innerHTML="Success!",r.innerHTML=`Done! Took ${m-c}ms, ${u} iterations`,n.src=h("happy",o),e.innerHTML="",e.style.display="none",setTimeout(()=>{let l=window.location.href;window.location.href=w("/.within.website/x/cmd/anubis/api/pass-challenge",{response:d,nonce:u,redir:l,elapsedTime:m-c})},250)})();})();
//# sourceMappingURL=main.mjs.map

Binary file not shown.

Binary file not shown.

File diff suppressed because one or more lines are too long

Binary file not shown.