z0x/anubis
Code Activity

cmd/anubis: allow setting key bytes in flag/envvar (#97)

Browse source 
* cmd/anubis: allow setting key bytes in flag/envvar

Docs are updated to generate a random key on load and when people press
the recycle button.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* review feedback fixups

Signed-off-by: Xe Iaso <me@xeiaso.net>

* Update cmd/anubis/main.go

Signed-off-by: Xe Iaso <me@xeiaso.net>

* Apply suggestions from code review

Co-authored-by: Ryan Cao <70191398+ryanccn@users.noreply.github.com>
Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Ryan Cao <70191398+ryanccn@users.noreply.github.com>
This commit is contained in:
Xe Iaso 2025-03-25 17:02:48 -04:00 • committed by GitHub
parent f29a200f09
commit 4155719422
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 143 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

57
cmd/anubis/main.go
View file

@ -2,6 +2,9 @@ package main
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/hex"
"flag"
"fmt"
"log"
@ -28,20 +31,34 @@ import (
)
var (
bind = flag.String("bind", ":8923", "network address to bind HTTP to")
bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
socketMode = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
robotsTxt = flag.Bool("serve-robots-txt", false, "serve a robots.txt file that disallows all robots")
policyFname = flag.String("policy-fname", "", "full path to anubis policy document (defaults to a sensible built-in policy)")
slogLevel = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
target = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
healthcheck = flag.Bool("healthcheck", false, "run a health check against Anubis")
debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally")
bind = flag.String("bind", ":8923", "network address to bind HTTP to")
bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
ed25519PrivateKeyHex = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
socketMode = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
robotsTxt = flag.Bool("serve-robots-txt", false, "serve a robots.txt file that disallows all robots")
policyFname = flag.String("policy-fname", "", "full path to anubis policy document (defaults to a sensible built-in policy)")
slogLevel = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
target = flag.String("target", "http://localhost:3923", "target to reverse proxy to")
healthcheck = flag.Bool("healthcheck", false, "run a health check against Anubis")
debugXRealIPDefault = flag.String("debug-x-real-ip-default", "", "If set, replace empty X-Real-Ip headers with this value, useful only for debugging Anubis and running it locally")
)
func keyFromHex(value string) (ed25519.PrivateKey, error) {
keyBytes, err := hex.DecodeString(value)
if err != nil {
return nil, fmt.Errorf("supplied key is not hex-encoded: %w", err)
}
if len(keyBytes) != ed25519.SeedSize {
return nil, fmt.Errorf("supplied key is not %d bytes long, got %d bytes", ed25519.SeedSize, len(keyBytes))
}
return ed25519.NewKeyFromSeed(keyBytes), nil
}
func doHealthCheck() error {
resp, err := http.Get("http://localhost" + *metricsBind + "/metrics")
if err != nil {
@ -156,10 +173,26 @@ func main() {
}
fmt.Println()
var priv ed25519.PrivateKey
if *ed25519PrivateKeyHex == "" {
_, priv, err = ed25519.GenerateKey(rand.Reader)
if err != nil {
log.Fatalf("failed to generate ed25519 key: %v", err)
}
slog.Warn("generating random key, Anubis will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation")
} else {
priv, err = keyFromHex(*ed25519PrivateKeyHex)
if err != nil {
log.Fatalf("failed to parse and validate ED25519_PRIVATE_KEY_HEX: %v", err)
}
}
s, err := libanubis.New(libanubis.Options{
Next: rp,
Policy: policy,
ServeRobotsTXT: *robotsTxt,
PrivateKey: priv,
})
if err != nil {
log.Fatalf("can't construct libanubis.Server: %v", err)

1
docs/docs/CHANGELOG.md
View file

@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
- ed25519 signing keys for Anubis can be stored in the flag `--ed25519-private-key-hex` or envvar `ED25519_PRIVATE_KEY_HEX`; if one is not provided when Anubis starts, a new one is generated and logged
- Fixed and clarified installation instructions
- Introduced integration tests using Playwright
- Refactor & Split up Anubis into cmd and lib.go

55
docs/docs/admin/installation.mdx
View file

@ -2,8 +2,28 @@
title: Setting up Anubis
---
import RandomKey from "@site/src/components/RandomKey";
Anubis is meant to sit between your reverse proxy (such as Nginx or Caddy) and your target service. One instance of Anubis must be used per service you are protecting.
<center>
```mermaid
---
title: With Anubis installed
---
flowchart LR
LB(Load balancer /
TLS terminator)
Anubis(Anubis)
App(App)
LB --> Anubis --> App
```
</center>
Anubis is shipped in the Docker repo [`ghcr.io/techarohq/anubis`](https://github.com/TecharoHQ/anubis/pkgs/container/anubis). The following tags exist for your convenience:
| Tag | Meaning |
@ -21,17 +41,30 @@ Anubis has very minimal system requirements. I suspect that 128Mi of ram may be
Anubis uses these environment variables for configuration:
| Environment Variable | Default value | Explanation |
| :--------------------- | :------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `BIND` | `:8923` | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock` |
| `BIND_NETWORK` | `tcp` | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports. |
| `DIFFICULTY` | `5` | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses. |
| `METRICS_BIND` | `:9090` | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information. |
| `METRICS_BIND_NETWORK` | `tcp` | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information. |
| `SOCKET_MODE` | `0770` | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets. |
| `POLICY_FNAME` | unset | The file containing [bot policy configuration](./policies.md). See the bot policy documentation for more details. If unset, the default bot policy configuration is used. |
| `SERVE_ROBOTS_TXT` | `false` | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file. |
| `TARGET` | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`. |
| Environment Variable | Default value | Explanation |
| :------------------------ | :---------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `BIND` | `:8923` | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock` |
| `BIND_NETWORK` | `tcp` | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports. |
| `DIFFICULTY` | `5` | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses. |
| `ED25519_PRIVATE_KEY_HEX` | | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details. |
| `METRICS_BIND` | `:9090` | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information. |
| `METRICS_BIND_NETWORK` | `tcp` | The address family that the Anubis metrics server listens on. See `BIND_NETWORK` for more information. |
| `SOCKET_MODE` | `0770` | _Only used when at least one of the `*_BIND_NETWORK` variables are set to `unix`._ The socket mode (permissions) for Unix domain sockets. |
| `POLICY_FNAME` | unset | The file containing [bot policy configuration](./policies.md). See the bot policy documentation for more details. If unset, the default bot policy configuration is used. |
| `SERVE_ROBOTS_TXT` | `false` | If set `true`, Anubis will serve a default `robots.txt` file that disallows all known AI scrapers by name and then additionally disallows every scraper. This is useful if facts and circumstances make it difficult to change the underlying service to serve such a `robots.txt` file. |
| `TARGET` | `http://localhost:3923` | The URL of the service that Anubis should forward valid requests to. Supports Unix domain sockets, set this to a URI like so: `unix:///path/to/socket.sock`. |
### Key generation
To generate an ed25519 private key, you can use this command:
```text
openssl rand -hex 32
```
Alternatively here is a key generated by your browser:
<RandomKey />
## Docker compose

2
docs/manifest/deployment.yaml
View file

@ -22,7 +22,7 @@ spec:
ports:
- containerPort: 80
- name: anubis
image: ghcr.io/techarohq/anubis:latest
image: ghcr.io/techarohq/anubis:main
imagePullPolicy: Always
env:
- name: "BIND"

42
docs/src/components/RandomKey/index.tsx Normal file
View file

@ -0,0 +1,42 @@
import { useState, useCallback } from "react";
import Code from "@theme/CodeInline";
import BrowserOnly from "@docusaurus/BrowserOnly";
// https://www.xaymar.com/articles/2020/12/08/fastest-uint8array-to-hex-string-conversion-in-javascript/
function toHex(buffer) {
return Array.prototype.map
.call(buffer, (x) => ("00" + x.toString(16)).slice(-2))
.join("");
}
export const genRandomKey = (): String => {
const array = new Uint8Array(32);
self.crypto.getRandomValues(array);
return toHex(array);
};
export default function RandomKey() {
return (
<BrowserOnly fallback={<div>Loading...</div>}>
{() => {
const [key, setKey] = useState<String>(genRandomKey());
const genRandomKeyCb = useCallback(() => {
setKey(genRandomKey());
});
return (
<span>
<Code>{key}</Code>
<span style={{ marginLeft: "0.25rem", marginRight: "0.25rem" }} />
<button
onClick={() => {
genRandomKeyCb();
}}
>
♻️
</button>
</span>
);
}}
</BrowserOnly>
);
}

15
lib/anubis.go
View file

@ -66,6 +66,7 @@ type Options struct {
Next http.Handler
Policy *policy.ParsedConfig
ServeRobotsTXT bool
PrivateKey ed25519.PrivateKey
}
func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
@ -93,15 +94,19 @@ func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedC
}
func New(opts Options) (*Server, error) {
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return nil, fmt.Errorf("failed to generate ed25519 key: %w", err)
if opts.PrivateKey == nil {
slog.Debug("opts.PrivateKey not set, generating a new one")
_, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return nil, fmt.Errorf("lib: can't generate private key: %v", err)
}
opts.PrivateKey = priv
}
result := &Server{
next: opts.Next,
priv: priv,
pub: pub,
priv: opts.PrivateKey,
pub: opts.PrivateKey.Public().(ed25519.PublicKey),
policy: opts.Policy,
DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
}