head	1.21;
access;
symbols
	pkgsrc-2026Q1:1.19.0.2
	pkgsrc-2026Q1-base:1.19
	pkgsrc-2025Q4:1.14.0.2
	pkgsrc-2025Q4-base:1.14
	pkgsrc-2025Q3:1.10.0.2
	pkgsrc-2025Q3-base:1.10
	pkgsrc-2025Q2:1.6.0.2
	pkgsrc-2025Q2-base:1.6;
locks; strict;
comment	@# @;


1.21
date	2026.05.07.18.50.10;	author bsiegert;	state Exp;
branches;
next	1.20;
commitid	MMrI2EXGAt3HJUEG;

1.20
date	2026.04.08.05.45.12;	author bsiegert;	state Exp;
branches;
next	1.19;
commitid	FFJx9trbRMg7k7BG;

1.19
date	2026.03.06.21.08.06;	author bsiegert;	state Exp;
branches
	1.19.2.1;
next	1.18;
commitid	AQtIeR1eh5qvuXwG;

1.18
date	2026.02.24.08.45.41;	author bsiegert;	state Exp;
branches;
next	1.17;
commitid	IrpU9bGVDwSIHBvG;

1.17
date	2026.01.26.19.49.20;	author bsiegert;	state Exp;
branches;
next	1.16;
commitid	yzzTb7tKWPopjWrG;

1.16
date	2026.01.15.19.54.58;	author bsiegert;	state Exp;
branches;
next	1.15;
commitid	Zuo9UmMrkrU4HwqG;

1.15
date	2026.01.10.17.38.34;	author bsiegert;	state Exp;
branches;
next	1.14;
commitid	efSZOf0nz9kr6SpG;

1.14
date	2025.12.02.19.25.24;	author bsiegert;	state Exp;
branches;
next	1.13;
commitid	269raRGxaHJBWRkG;

1.13
date	2025.11.28.17.38.31;	author ryoon;	state Exp;
branches;
next	1.12;
commitid	Q8YMZXWiyqj5ulkG;

1.12
date	2025.10.16.18.00.04;	author bsiegert;	state Exp;
branches;
next	1.11;
commitid	RAGpkHmyeYCUYOeG;

1.11
date	2025.10.08.06.54.40;	author bsiegert;	state Exp;
branches;
next	1.10;
commitid	5p46uH9RJXSFyJdG;

1.10
date	2025.09.11.21.56.23;	author bsiegert;	state Exp;
branches;
next	1.9;
commitid	UkOmEsFYtBY1qlaG;

1.9
date	2025.09.06.13.17.13;	author bsiegert;	state Exp;
branches;
next	1.8;
commitid	cojusOdnRBFGHE9G;

1.8
date	2025.08.31.10.03.02;	author bsiegert;	state Exp;
branches;
next	1.7;
commitid	SdH7Ppgppxg5PR8G;

1.7
date	2025.07.29.15.16.10;	author ryoon;	state Exp;
branches;
next	1.6;
commitid	32iMNL5we8UnCE4G;

1.6
date	2025.06.06.14.01.41;	author bsiegert;	state Exp;
branches;
next	1.5;
commitid	JbNcx43ItkrmUPXF;

1.5
date	2025.05.24.14.41.18;	author ryoon;	state Exp;
branches;
next	1.4;
commitid	FM9aBbhxisLTxaWF;

1.4
date	2025.05.22.15.14.14;	author ryoon;	state Exp;
branches;
next	1.3;
commitid	zsgsyMlh22CdNUVF;

1.3
date	2025.05.20.15.16.41;	author ryoon;	state Exp;
branches;
next	1.2;
commitid	R99TIVwjDQU4SEVF;

1.2
date	2025.05.13.17.33.14;	author bsiegert;	state Exp;
branches;
next	1.1;
commitid	yU4s2P7bBZCTQLUF;

1.1
date	2025.05.09.19.16.40;	author bsiegert;	state Exp;
branches;
next	;
commitid	3h5ktYIvC7IlygUF;

1.19.2.1
date	2026.04.22.14.32.19;	author maya;	state Exp;
branches;
next	;
commitid	iDHLLUhOplH6NXCG;


desc
@@


1.21
log
@Revbump all Go packages after go126 security update
@
text
@# $NetBSD: Makefile,v 1.20 2026/04/08 05:45:12 bsiegert Exp $

DISTNAME=	anubis-src-vendor-npm-1.24.0
PKGNAME=	${DISTNAME:S/anubis-src-vendor-npm-/anubis-/}
PKGREVISION=	5
CATEGORIES=	www
MASTER_SITES=	${MASTER_SITE_GITHUB:=TecharoHQ/}
GITHUB_RELEASE=	v${PKGVERSION_NOREV}

MAINTAINER=	bsiegert@@NetBSD.org
HOMEPAGE=	https://anubis.techaro.lol/
COMMENT=	Proof-of-work check to stop AI bots
LICENSE=	mit

USE_LANGUAGES=		c # Go
GOFLAGS+=		-ldflags="-X github.com/TecharoHQ/anubis.Version=v${PKGVERSION_NOREV}"
GO_BUILD_PATTERN=	./cmd/anubis

EGDIR=			share/examples/anubis
INSTALLATION_DIRS=	${EGDIR}
PKG_SYSCONFSUBDIR=	anubis
CONF_FILES+=		${EGDIR}/default.env ${PKG_SYSCONFDIR}/default.env

APACHE_USER?=		www
APACHE_GROUP?=		www
PKG_USERS_VARS+=	APACHE_USER
PKG_GROUPS_VARS+=	APACHE_GROUP
PKG_GROUPS=		${APACHE_GROUP}
PKG_USERS=		${APACHE_USER}:${APACHE_GROUP}

FILES_SUBST+=		APACHE_USER=${APACHE_USER}
FILES_SUBST+=		APACHE_GROUP=${APACHE_GROUP}

RCD_SCRIPTS+=		anubis

post-install:
	${INSTALL_DATA} ${WRKSRC}/run/default.env ${DESTDIR}${PREFIX}/share/examples/anubis


.include "go-modules.mk"

.include "../../lang/go/go-module.mk"
.include "../../mk/bsd.pkg.mk"
@


1.20
log
@Revbump all Go packages after security update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.19 2026/03/06 21:08:06 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	4
@


1.19
log
@Revbump all Go packages after go126 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.18 2026/02/24 08:45:41 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	3
@


1.19.2.1
log
@Revbump all Go packages after go126 security fix
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.19 2026/03/06 21:08:06 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	4
@


1.18
log
@Revbump all Go packages after default version bump
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.17 2026/01/26 19:49:20 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	2
@


1.17
log
@Move non-pattern Go flags to GOFLAGS.

GO_BUILD_PATTERN is only for patterns, as announced on tech-pkg@@.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.16 2026/01/15 19:54:58 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	1
@


1.16
log
@Revbump all Go packages after go125 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.15 2026/01/10 17:38:34 bsiegert Exp $
d16 2
a17 2
GO_BUILD_PATTERN+=	-ldflags="-X github.com/TecharoHQ/anubis.Version=v${PKGVERSION_NOREV}"
GO_BUILD_PATTERN+=	./cmd/anubis
@


1.15
log
@anubis: update to 1.24.0

Anubis is back and better than ever! Lots of minor fixes with some big
ones interspersed.

- Fix panic when validating challenges after privacy-mode browsers strip
  headers and the follow-up request matches an ALLOW threshold.
- Expose WEIGHT rule matches as Prometheus metrics.
- Allow more OCI registry clients based on feedback.
- Expose services directory in the embedded (data) filesystem.
- Add Ukrainian locale (#1044).
- Allow Renovate as an OCI registry client.
- Properly handle 4in6 addresses so that IP matching works with those
  addresses.
- Add support to simple Valkey/Redis cluster mode
- Open Graph passthrough now reuses the configured target Host/SNI/TLS
  settings, so metadata fetches succeed when the upstream certificate
  differs from the public domain. (1283)
- Stabilize the CVE-2025-24369 regression test by always submitting an
  invalid proof instead of relying on random POW failures.
- Refine the check that ensures the presence of the Accept header to
  avoid breaking docker clients.
- Removed rules intended to reward actual browsers due to abuse in the wild.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.14 2025/12/02 19:25:24 bsiegert Exp $
d5 1
@


1.14
log
@Revbump all Go packages after go125 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.13 2025/11/28 17:38:31 ryoon Exp $
d3 1
a3 1
DISTNAME=	anubis-src-vendor-npm-1.23.1
a4 1
PKGREVISION=	1
@


1.13
log
@www/anubis: Update to 1.23.1

Changelog:
## v1.23.1: Lyse Hext - Echo 1

- Fix `SERVE_ROBOTS_TXT` setting after the double slash fix broke it.

### Potentially breaking changes

#### Remove default Tencent Cloud block rule

v1.23.0 added a default rule to block Tencent Cloud. After an email from their abuse team where they promised to take action to clean up their reputation, I have removed the default block rule. If this network causes you problems, please contact [abuse@@tencent.com](mailto:abuse@@tencent.com) and supply the following information:

- Time of abusive requests.
- IP address, User-Agent header, or other unique identifiers that can help the abuse team educate the customer about their misbehaving infrastructure.
- Does the abusive IP address request robots.txt? If not, be sure to include that information.
- A brief description of the impact to your system such as high system load, pages not rendering, or database system crashes. This helps the provider establish the fact that their customer is causing you measurable harm.
- Context as to what your service is, what it does, and why they should care.

Mention that you are using Anubis or BotStopper to protect your services. If they do not respond to you, please [contact me](https://xeiaso.net/contact) as soon as possible.

#### Docker / OCI registry clients

Anubis v1.23.0 accidentally blocked Docker / OCI registry clients. In order to explicitly allow them, add an import for `(data)/clients/docker-client.yaml`:

```yaml
bots:
  - import: (data)/meta/default-config.yaml
  - import: (data)/clients/docker-client.yaml
```

This is technically a regression as these clients used to work in Anubis v1.22.0, however it is allowable to make this opt-in as most websites do not expect to be serving Docker / OCI registry client traffic.

## v1.23.0: Lyse Hext

- Add default tencent cloud DENY rule.
- Added `(data)/meta/default-config.yaml` for importing the entire default configuration at once.
- Add `-custom-real-ip-header` flag to get the original request IP from a different header than `x-real-ip`.
- Add `contentLength` variable to bot expressions.
- Add `COOKIE_SAME_SITE_MODE` to force anubis cookies SameSite value, and downgrade automatically from `None` to `Lax` if cookie is insecure.
- Fix lock convoy problem in decaymap ([#1103](https://github.com/TecharoHQ/anubis/issues/1103)).
- Fix lock convoy problem in bbolt by implementing the actor pattern ([#1103](https://github.com/TecharoHQ/anubis/issues/1103)).
- Remove bbolt actorify implementation due to causing production issues.
- Document missing environment variables in installation guide: `SLOG_LEVEL`, `COOKIE_PREFIX`, `FORCED_LANGUAGE`, and `TARGET_DISABLE_KEEPALIVE` ([#1086](https://github.com/TecharoHQ/anubis/pull/1086)).
- Add validation warning when persistent storage is used without setting signing keys.
- Fixed `robots2policy` to properly group consecutive user agents into `any:` instead of only processing the last one ([#925](https://github.com/TecharoHQ/anubis/pull/925)).
- Make the `fast` algorithm prefer purejs when running in an insecure context.
- Add the [`s3api` storage backend](./admin/policies.mdx#s3api) to allow Anubis to use S3 API compatible object storage as its storage backend.
- Fix a "stutter" in the cookie name prefix so the auth cookie is named `techaro.lol-anubis-auth` instead of `techaro.lol-anubis-auth-auth`.
- Make `cmd/containerbuild` support commas for separating elements of the `--docker-tags` argument as well as newlines.
- Add the `DIFFICULTY_IN_JWT` option, which allows one to add the `difficulty` field in the JWT claims which indicates the difficulty of the token ([#1063](https://github.com/TecharoHQ/anubis/pull/1063)).
- Ported the client-side JS to TypeScript to avoid egregious errors in the future.
- Fixes concurrency problems with very old browsers ([#1082](https://github.com/TecharoHQ/anubis/issues/1082)).
- Randomly use the Refresh header instead of the meta refresh tag in the metarefresh challenge.
- Update OpenRC service to truncate the runtime directory before starting Anubis.
- Make the git client profile more strictly match how the git client behaves.
- Make the default configuration reward users using normal browsers.
- Allow multiple consecutive slashes in a row in application paths ([#754](https://github.com/TecharoHQ/anubis/issues/754)).
- Add option to set `targetSNI` to special keyword 'auto' to indicate that it should be automatically set to the request Host name ([424](https://github.com/TecharoHQ/anubis/issues/424)).
- The Preact challenge has been removed from the default configuration. It will be deprecated in the future.
- An open redirect when in subrequest mode has been fixed.

### Potentially breaking changes

#### Multiple checks at once has and-like semantics instead of or-like semantics

Anubis lets you stack multiple checks at once with blocks like this:

```yaml
name: allow-prometheus
action: ALLOW
user_agent_regex: ^prometheus-probe$
remote_addresses:
  - 192.168.2.0/24
```

Previously, this only returned ALLOW if _any one_ of the conditions matched. This behaviour has changed to only return ALLOW if _all_ of the conditions match. I expect this to have some issues with user configs, however this fix is grave enough that it's worth the risk of breaking configs. If this bites you, please let me know so we can make an escape hatch.

### Better error messages

In order to make it easier for legitimate clients to debug issues with their browser configuration and Anubis, Anubis will emit internal error detail in base 64 so that administrators can chase down issues. Future versions of this may also include a variant that encrypts the error detail messages.

### Bug Fixes

Sometimes the enhanced temporal assurance in [#1038](https://github.com/TecharoHQ/anubis/pull/1038) and [#1068](https://github.com/TecharoHQ/anubis/pull/1068) could backfire because Chromium and its ilk randomize the amount of time they wait in order to avoid a timing side channel attack. This has been fixed by both increasing the amount of time a client has to wait for the metarefresh and preact challenges as well as making the server side logic more permissive.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.12 2025/10/16 18:00:04 bsiegert Exp $
d5 1
@


1.12
log
@Revbump all Go packages after go125 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.11 2025/10/08 06:54:40 bsiegert Exp $
d3 1
a3 1
DISTNAME=	anubis-src-vendor-npm-1.22.0
a4 1
PKGREVISION=	2
@


1.11
log
@Revbump all Go packages after go125 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.10 2025/09/11 21:56:23 bsiegert Exp $
d5 1
a5 1
PKGREVISION=	1
@


1.10
log
@anubis: update to 1.22.0

In this release, we finally fix the odd number of CPU cores bug, pave the way
for lighter weight challenges, make Anubis more adaptable, and more.

Big ticket items

Proof of React challenge

A new "proof of React" has been added. It runs a simple app in React that has
several chained hooks. It is much more lightweight than the proof of work
check.

Smaller features

- The segments function was added for splitting a path into its
  slash-separated segments.
- Added possibility to disable HTTP keep-alive to support backends not
  properly handling it.
- When issuing a challenge, Anubis stores information about that challenge
  into the store. That stored information is later used to validate challenge
  responses. This works around nondeterminism in bot rules.
- One of the biggest sources of lag in Firefox has been eliminated: the use of
  WebCrypto. Now whenever Anubis detects the client is using Firefox (or Pale
  Moon), it will swap over to a pure-JS implementation of SHA-256 for speed.
- Proof of work solving has had a complete overhaul and rethink based on
  feedback from browser engine developers, frontend experts, and overall
  performance profiling.
- Optimize the performance of the pure-JS Anubis solver.
- Web Workers are stored as dedicated JavaScript files in
  static/js/workers/*.mjs.
- Pave the way for non-SHA256 solver methods and eventually one that uses
  WebAssembly (or WebAssembly code compiled to JS for those that disable
  WebAssembly).
- Legacy JavaScript code has been eliminated.
- When parsing Open Graph tags, add any URLs found in the responses to a
  temporary "allow cache" so that social preview images work.
- The hard dependency on WebCrypto has been removed, allowing a proof of work
  challenge to work over plain (unencrypted) HTTP.
- The Anubis version number is put in the footer of every page.
- Add a default block rule for Huawei Cloud.
- Add a default block rule for Alibaba Cloud.
- Added support to use Traefik forwardAuth middleware.
- Add X-Request-URI support so that Subrequest Authentication has path
  support.

Fixes

Odd numbers of CPU cores are properly supported

Some phones have an odd number of CPU cores. This caused interesting issues.
This was fixed by using Math.trunc to convert the number of CPU cores back
into an integer.

Smaller fixes

- A standard library HTTP server log message about HTTP pipelining not working
  has been filtered out of Anubis' logs. There is no action that can be taken
  about it.
- Added a missing link to the Caddy installation environment in the
  installation documentation.
- Downstream consumers can change the default log/slog#Logger instance that
  Anubis uses by setting opts.Logger to your slog instance of choice (#864).
- The Thoth client is now public in the repo instead of being an internal
  package.
- Custom-AsyncHttpClient's default User-Agent has an increased weight by
  default.
- Add option for replacing the default explanation text with a custom one.
- The contact email in the LibreJS header has been changed.
- Firefox for Android support has been fixed by embedding the challenge ID
  into the pass-challenge route. This also fixes some inconsistent issues with
  other mobile browsers.
- The default favicon pattern in data/common/keep-internet-working.yaml has
  been updated to permit requests for png/gif/jpg/svg files as well as ico.
- The --cookie-prefix flag has been fixed so that it is fully respected.
- The default patterns in data/common/keep-internet-working.yaml have been
  updated to appropriately escape the '.' character in the regular expression
  patterns.
- Add optional restrictions for JWT based on the value of a header
- The word "hack" has been removed from the translation strings for Anubis due
  to incidents involving people misunderstanding that word and sending
  particularly horrible things to the project lead over email.
- Bump AI-robots.txt to version 1.39
- Inject adversarial input to break AI coding assistants.
- Add better logging when using Subrequest Authentication.

Security-relevant changes

Add a server-side check for the meta-refresh challenge that makes sure clients
have waited for at least 95% of the time that they should.

Fix potential double-spend for challenges

Anubis operates by issuing a challenge and having the client present a
solution for that challenge. Challenges are identified by a unique UUID, which
is stored in the database.

The problem is that a challenge could potentially be used twice by a dedicated
attacker making a targeted attack against Anubis. Challenge records did not
have a "spent" or "used" field. In total, a dedicated attacker could solve a
challenge once and reuse that solution across multiple sessions in order to
mint additional tokens.

This was fixed by adding a "spent" field to challenges in the data store. When
a challenge is solved, that "spent" field gets set to true. If a future
attempt to solve this challenge is observed, it gets rejected.

With the advent of store based challenge issuance, this means that these
challenge IDs are only good for 30 minutes. Websites using the most recent
version of Anubis have limited exposure to this problem.

Websites using older versions of Anubis have a much more increased exposure to
this problem and are encouraged to keep this software updated as often and as
frequently as possible.

Breaking changes

- The "slow" frontend solver has been removed in order to reduce maintenance
  burden. Any existing uses of it will still work, but issue a warning upon
  startup asking administrators to upgrade to the "fast" frontend solver.
- The legacy JSON based policy file example has been removed and all
  documentation for how to write a policy file in JSON has been deleted. JSON
  based policy files will still work, but YAML is the superior option for
  Anubis configuration.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.9 2025/09/06 13:17:13 bsiegert Exp $
d5 1
@


1.9
log
@Revbump all Go packages after go125 security update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.8 2025/08/31 10:03:02 bsiegert Exp $
d3 1
a3 1
DISTNAME=	anubis-src-vendor-npm-1.21.3
a4 1
PKGREVISION=	2
a20 1
CONF_FILES+=		${EGDIR}/botPolicies.json ${PKG_SYSCONFDIR}/botPolicies.json
a35 1
	${INSTALL_DATA} ${WRKSRC}/data/botPolicies.json ${DESTDIR}${PREFIX}/share/examples/anubis
@


1.8
log
@Revbump all Go packages after moving to go125
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.7 2025/07/29 15:16:10 ryoon Exp $
d5 1
a5 1
PKGREVISION=	1
@


1.7
log
@www/anubis: Update to 1.21.3

Changelog:
1.21.3: Minfilia Warde - Echo 3

Fixes GHSA-jhjj-2g64-px7c

This could allow an attacker to craft an Anubis pass-challenge URL that forces
a redirect to nonstandard URLs, such as the javascript: scheme which executes
arbitrary JavaScript code in a browser context when the user clicks the "Try
again" button.

This has been fixed by disallowing any URLs without the scheme http or https.

Additionally, the "Try again" button has been fixed to completely ignore the
user-supplied redirect location. It now redirects to the home page (/).

Notes

An incomplete version of this fix was tagged at v1.21.2 and then the release
process was aborted upon final testing. Do not package or use v1.21.2.

What's Changed

  * fix(lib): add comprehensive XSS protection logic by @@Xe in #905
  * fix(web): make the try again button always go back to / by @@Xe in #907

1.21.1: Minfilia Warde - Echo 1

  * Expired records are now properly removed from bbolt databases (#848).
  * Fix hanging on service restart (#853)

Added

Anubis now supports the missingHeader to assert the absence of headers in
requests.

New locales

Anubis now supports these new languages:

  * Czech
  * Finnish
  * Norwegian Bokmål
  * Norwegian Nynorsk
  * Russian

Fixes

Fix "error: can't get challenge" when details about a challenge can't be found
in the server side state

v1.21.0 changed the core challenge flow to maintain information about
challenges on the server side instead of only doing them via stateless
idempotent generation functions and relying on details to not change. There was
a subtle bug introduced in this change: if a client has an unknown challenge ID
set in its test cookie, Anubis will clear that cookie and then throw an HTTP
500 error.

This has been fixed by making Anubis throw a new challenge page instead.

Fix event loop thrashing when solving a proof of work challenge

Previously the "fast" proof of work solver had a fragment of JavaScript that
attempted to only post an update about proof of work progress to the main
browser window every 1024 iterations. This fragment of JavaScript was subtly
incorrect in a way that passed review but actually made the workers send an
update back to the main thread every iteration. This caused a pileup of
unhandled async calls (similar to a socket accept() backlog pileup in Unix)
that caused stack space exhaustion.

This has been fixed in the following ways:

 1. The complicated boolean logic has been totally removed in favour of a
    worker-local iteration counter.
 2. The progress bar is updated by worker 0 instead of all workers.

Hopefully this should limit the event loop thrashing and let ia32 browsers (as
well as any environment with a smaller stack size than amd64 and aarch64 seem
to have) function normally when processing Anubis proof of work challenges.

Fix potential memory leak when discovering a solution

In some cases, the parallel solution finder in Anubis could cause all of the
worker promises to leak due to the fact the promises were being improperly
terminated. This was fixed by having Anubis debounce worker termination instead
of allowing it to potentially recurse infinitely.

What's Changed

  * docs(known-instances): update list of known instances by @@lotharsm in #847
  * fix(cmd/anubis): add signal handling to metrics server by @@EmRowlands in #
    856
  * test: add i18n smoke test by @@Xe in #858
  * test(ssh-ci): deflake SSH CI with exponential backoff by @@Xe in #859
  * Fix broken BBolt database cleanup process by @@thenickdude in #848
  * fix(localization): untranslated string in Filipino language by
    @@hankskyjames777 in #850
  * feat(localization): Add Czech language translation by @@xmorave2 in #849
  * feat(expressions): add missingHeader function to bot environment by @@Xe in
    #870
  * build(deps): bump the github-actions group with 2 updates by @@dependabot
    [bot] in #871
  * build(deps-dev): bump the npm group with 3 updates by @@dependabot[bot] in #
    872
  * build(deps): bump the gomod group with 6 updates by @@dependabot[bot] in #
    873
  * Revert "build(deps): bump the gomod group with 6 updates" by
    @@JasonLovesDoggo in #874
  * Remove duplicated string in Filipino language file by @@searinminecraft in #
    875
  * fix(web): amend future leak on proof of work solution by @@Xe in #879
  * fix(web/fast): remove event loop thrashing by @@Xe in #880
  * Update pt-BR.json by @@HQuest in #878
  * fix(lib): fix challenge issuance logic by @@Xe in #881
  * Add Finnish localization by @@ZerionSeven in #863
  * feat: Russian localization for Anubis by @@Xe in #882
  * feat(localization): Add in Bokmål and Nynorsk translations by @@turtlegarden
    in #855
  * chore: release v1.21.1 by @@Xe in #887


1.21.0: Minfilia Warde

    Please, be at ease. You are among friends here.

In this release, Anubis becomes internationalized, gains the ability to use
system load as input to issuing challenges, finally fixes the "invalid
response" after "success" bug, and more! Please read these notes before
upgrading as the changes are big enough that administrators should take action
to ensure that the upgrade goes smoothly.

This release is brought to you by FreeCAD, an open-source computer aided design
tool that lets you design things for the real world.

Big ticket changes

The biggest change is that the "invalid response" after "success" bug is now
finally fixed for good by totally rewriting how Anubis' challenge issuance flow
works. Instead of generating challenge strings from request metadata (under the
assumption that the values being compared against are stable), Anubis now
generates random data for each challenge. This data is stored in the active
storage backend for up to 30 minutes. This also fixes #746 and other similar
instances of this issue.

In order to reduce confusion, the "Success" interstitial that shows up when you
pass a proof of work challenge has been removed.

Storage

Anubis now is able to store things persistently in memory, on the disk, or in
Valkey (this includes other compatible software). By default Anubis uses the
in-memory backend. If you have an environment with mutable storage (even if it
is temporary), be sure to configure the bbolt storage backend.

Localization

Anubis now supports localized responses. Locales can be added in lib/
localization/locales/. This release includes support for the following
languages:

  * Brazilian Portugese
  * Chinese (Simplified)
  * Chinese (Traditional)
  * English
  * Estonian
  * Filipino
  * French
  * German
  * Icelandic
  * Italian
  * Japanese
  * Spanish
  * Turkish

If facts or local regulations demand, you can set Anubis default language with
the FORCED_LANGUAGE environment variable or the --forced-language command line
argument:

FORCED_LANGUAGE=de

Load average

Anubis can dynamically take action based on the system load average, allowing
you to write rules like this:

## System load based checks.
# If the system is under high load for the last minute, add weight.
- name: high-load-average
  action: WEIGH
  expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
  weight:
    adjust: 20

# If it is not for the last 15 minutes, remove weight.
- name: low-load-average
  action: WEIGH
  expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
  weight:
    adjust: -10

Something to keep in mind about system load average is that it is not aware of
the number of cores the system has. If you have a 16 core system that has 16
processes running but none of them is hogging the CPU, then you will get a load
average below 16. If you are in doubt, make your "high load" metric at least
two times the number of CPU cores and your "low load" metric at least half of
the number of CPU cores. For example:

     Kind Core count Load threshold
high load 4          8.0
 low load 4          2.0
high load 16         32.0
 low load 16         8

Also keep in mind that this does not account for other kinds of latency like I/
O latency. A system can have its web applications unresponsive due to high
latency from a MySQL server but still have that web application server report a
load near or at zero.

Other features and fixes

There are a bunch of other assorted features and fixes too:

  * Add COOKIE_SECURE option to set the cookie Secure flag
  * Sets cookie defaults to use SameSite: None
  * Determine the BIND_NETWORK/--bind-network value from the bind address (#677
    ).
  * Implement a development container manifest to make contributions easier.
  * Fix dynamic cookie domains functionality (#731)
  * Add option for custom cookie prefix (#732)
  * Make the Open Graph subsystem and DNSBL subsystem use storage backends
    instead of storing everything in memory by default.
  * Allow Common Crawl by default so scrapers have less incentive to scrape
  * The bbolt storage backend now runs its cleanup every hour instead of every
    five minutes.
  * Don't block Anubis starting up if Thoth health checks fail.
  * A race condition involving opening two challenge pages at once in different
    tabs causing one of them to fail has been fixed.
  * The "Try again" button on the error page has been fixed. Previously it
    meant "try the solution again" instead of "try the challenge again".
  * In certain cases, a user could be stuck with a test cookie that is invalid,
    locking them out of the service for up to half an hour. This has been fixed
    with better validation of this case and clearing the cookie.
  * Start exposing JA4H fingerprints for later use in CEL expressions.
  * Add /healthz route for use in platform-based health checks.

Potentially breaking changes

We try to introduce breaking changes as much as possible, but these are the
changes that may be relevant for you as an administrator:

Challenge format change

Previously Anubis did no accounting for challenges that it issued. This means
that if Anubis restarted during a client, the client would be able to proceed
once Anubis came back online.

During the upgrade to v1.21.0 and when v1.21.0 (or later) restarts with the
in-memory storage backend, you may see a higher rate of failed challenges than
normal. If this persists beyond a few minutes, open an issue.

If you are using the in-memory storage backend, please consider using a
different storage backend.

Systemd service changes

The following potentially breaking change applies to native installs with
systemd only:

Each instance of systemd service template now has a unique RuntimeDirectory, as
opposed to each instance of the service sharing a RuntimeDirectory. This change
was made to avoid the RuntimeDirectory getting nuked any time one of the Anubis
instances restarts.

If you configured Anubis' unix sockets to listen on /run/anubis/foo.sock for
instance anubis@@foo, you will need to configure Anubis to listen on /run/anubis
/foo/foo.sock and additionally configure your HTTP load balancer as
appropriate.

If you need the legacy behaviour, install this systemd unit dropin:

# /etc/systemd/system/anubis@@.service.d/50-runtimedir.conf
[Service]
RuntimeDirectory=anubis

Just keep in mind that this will cause problems when Anubis restarts.

What's Changed

  * feat: implement localization system by @@lolgzs in #716
  * fix: determine bind network from bind address by @@littlecxm in #714
  * Add Brazilian Portuguese translation by @@rffontenelle in #726
  * fix: Dynamic cookie domain not working by @@Earl0fPudding in #731
  * feat(cmd): Add custom cookie prefix by @@Earl0fPudding in #732
  * build(deps): bump the github-actions group with 2 updates by @@dependabot
    [bot] in #735
  * build(deps): bump the gomod group with 2 updates by @@dependabot[bot] in #
    736
  * feat: dev container support by @@Xe in #734
  * Fix translations in pt-BR.json by @@rffontenelle in #729
  * Set cookies to have the Secure flag default to true by @@victorvalenca in #
    739
  * fix(web/main): remove the success interstitial by @@Xe in #745
  * feat(localization): Add option for forcing a language by @@Earl0fPudding in
    #742
  * fix(run/anubis@@.service): unique runtimedir per instance by @@Xe in #750
  * feat(localization): Add German language translation by @@Earl0fPudding in
    ht...

1.21.0-pre3: Minfilia Warde

A small fix to amend broken RPM signatures.


1.21.0-pre2: Minfila Warde

Please report any issues with this prerelease so the full release can be the
best it can possibly be.

What's Changed

  * build(deps): bump the github-actions group with 2 updates by @@dependabot
    [bot] in #770
  * build(deps): bump github.com/shirou/gopsutil/v4 from 4.25.1 to 4.25.6 in
    the gomod group by @@dependabot[bot] in #771
  * minor typo fix: Update apache.mdx replace nginx with Apache in place by
    @@mihugo in #779
  * docs(known-instances): update list of known instances by @@lotharsm in #776
  * feat(localization): add Simplified Chinese by @@littlecxm in #774
  * docs(installation): Clarify information about private keys and multile
    instances by @@StandingPadAnimations in #788
  * fix(localization): HTML language header and forced-language by @@SlyEcho in
    #787
  * Update apache.mdx by @@jzb in #784
  * feat(localization): add Japanese language translation by @@dai in #772
  * feat(i18n): add Estonian locale by @@SlyEcho in #783
  * Create is.json by @@sveinki in #780
  * feat(localization): Add Italian language translation by @@giomba in #778
  * feat(localization): Add Filipino language by @@searinminecraft in #775
  * fix(internal/thoth): don't block Anubis starting if healthcheck fails by
    @@Xe in #794
  * feat(blog): incident report for TI-20250709-0001 by @@Xe in #795
  * chore: use nginx-micro to make the docs image 13 MB by @@Xe in #796
  * docs: update CHANGELOG for language changes by @@Xe in #793
  * docs(known-instances): update list of known instances by @@lotharsm in #801
  * correct gitea.botPolicies extension to be yaml, not json by @@evgeni in #800
  * docs(known-instances): add rpmfusion.org and wiki.freepascal.org to known
    instances by @@lotharsm in #807
  * chore(docs): fix typo in configuration/expressions by @@maximelouet in #811
  * fix(index.templ) centered-div class usage typo by @@ciencia in #812
  * chore(docs): add link to status page in the footer by @@Xe in #814
  * chore: release v1.21.0-pre2 by @@Xe in #816


1.21.0-pre1: Minfilia Warde

    Please, be at ease. You are among friends here.

In this release, Anubis becomes internationalized, gains the ability to use
system load as input to issuing challenges,

This release is brought to you by FreeCAD, an open-source computer aided design
tool that lets you design things for the real world.

Big ticket changes

The biggest change is that the "invalid response" after "success" bug is now
finally fixed for good by totally rewriting how Anubis' challenge issuance flow
works. Instead of generating challenge strings from request metadata (under the
assumption that the values being compared against are stable), Anubis now
generates random data for each challenge. This data is stored in the active
storage backend for up to 30 minutes. This also fixes #746 and other similar
instances of this issue.

In order to reduce confusion, the "Success" interstitial that shows up when you
pass a proof of work challenge has been removed.

Storage

Anubis now is able to store things persistently in memory, on the disk, or in
Valkey (this includes other compatible software). By default Anubis uses the
in-memory backend. If you have an environment with mutable storage (even if it
is temporary), be sure to configure the bbolt storage backend.

Localization

Anubis now supports localized responses. Locales can be added in lib/
localization/locales/. This release includes support for the following
languages:

  * Brazilian Portugese
  * Chinese (Traditional)
  * English
  * French
  * German
  * Spanish
  * Turkish

If facts or local regulations demand, you can set Anubis default language with
the FORCE_LANGUAGE environment variable:

FORCE_LANGUAGE=de

Load-based checks

Anubis can dynamically take action based on the system load average, allowing
you to write rules like this:

## System load based checks.
# If the system is under high load for the last minute, add weight.
- name: high-load-average
  action: WEIGH
  expression: load_1m >= 10.0 # make sure to end the load comparison in a .0
  weight:
    adjust: 20

# If it is not for the last 15 minutes, remove weight.
- name: low-load-average
  action: WEIGH
  expression: load_15m <= 4.0 # make sure to end the load comparison in a .0
  weight:
    adjust: -10

Something to keep in mind about system load average is that it is not aware of
the number of cores the system has. If you have a 16 core system that has 16
processes running but none of them is hogging the CPU, then you will get a load
average below 16. If you are in doubt, make your "high load" metric at least
two times the number of CPU cores and your "low load" metric at least half of
the number of CPU cores. For example:

     Kind Core count Load threshold
high load 4          8.0
 low load 4          2.0
high load 16         32.0
 low load 16         8

Also keep in mind that this does not account for other kinds of latency like I/
O latency. A system can have its web applications unresponsive due to high
latency from a MySQL server but still have that web application server report a
load near or at zero.

Other features and fixes

There are a bunch of other assorted features and fixes too:

  * Add COOKIE_SECURE option to set the cookie Secure flag
  * Sets cookie defaults to use SameSite: None
  * Determine the BIND_NETWORK/--bind-network value from the bind address (#677
    ).
  * Implement a development container manifest to make contributions easier.
  * Fix dynamic cookie domains functionality (#731)
  * Add option for custom cookie prefix (#732)
  * Make the Open Graph subsystem and DNSBL subsystem use storage backends
    instead of storing everything in memory by default.
  * Allow Common Crawl by default so scrapers have less incentive to scrape
  * The bbolt storage backend now runs its cleanup every hour instead of every
    five minutes.

Potentially breaking changes

The following potentially breaking change applies to native installs with
systemd only:

Each instance of systemd service template now has a unique RuntimeDirectory, as
opposed to each instance of the service sharing a RuntimeDirectory. This change
was made to avoid the RuntimeDirectory getting nuked any time one of the Anubis
instances restarts.

If you configured Anubis' unix sockets to listen on /run/anubis/foo.sock for
instance anubis@@foo, you will need to configure Anubis to listen on /run/anubis
/foo/foo.sock and additionally configure your HTTP load balancer as
appropriate.

If you need the legacy behaviour, install this systemd unit dropin:

# /etc/systemd/system/anubis@@.service.d/50-runtimedir.conf
[Service]
RuntimeDirectory=anubis

Just keep in mind that this will cause problems when Anubis restarts.

What's Changed

  * feat: implement localization system by @@lolgzs in #716
  * fix: determine bind network from bind address by @@littlecxm in #714
  * Add Brazilian Portuguese translation by @@rffontenelle in #726
  * fix: Dynamic cookie domain not working by @@Earl0fPudding in #731
  * feat(cmd): Add custom cookie prefix by @@Earl0fPudding in #732
  * build(deps): bump the github-actions group with 2 updates by @@dependabot in
    #735
  * build(deps): bump the gomod group with 2 updates by @@dependabot in #736
  * feat: dev container support by @@Xe in #734
  * Fix translations in pt-BR.json by @@rffontenelle in #729
  * Set cookies to have the Secure flag default to true by @@victorvalenca in #
    739
  * fix(web/main): remove the success interstitial by @@Xe in #745
  * feat(localization): Add option for forcing a language by @@Earl0fPudding in
    #742
  * fix(run/anubis@@.service): unique runtimedir per instance by @@Xe in #750
  * feat(localization): Add German language translation by @@Earl0fPudding in #
    741
  * docs: add BotStopper docs from the git repo by @@Xe in #752
  * chore(default-config): allowlist common crawl by @@Xe in #753
  * feat(localization): Add Turkish language translation by @@dcelasun in #751
  * docs(known-instances): add ebird.org by @@SGHFan in #755
  * feat(lib): use new challenge creation flow by @@Xe in #749
  * chore(devcontainer): move playwright to its own devcontainer service by @@Xe
    in #756
  * docs(known-instances): Add Duke University, coinhoards.org (and myself) to
    known instances by @@lotharsm in #757
  * fix: make ogtags and dnsbl use the Store instead of memory by @@Xe in #760
  * fix(lib/store/bbolt): use a multi-bucket flow instead of a single bucket
    flow by @@Xe in #761
  * fix(lib/store/bbolt): run cleanup every hour instead of every 5 minutes by
    @@Xe in #762
  * docs: remove proof of work branding by @@Xe in #763
  * feat(localization): Update German language translation by @@lotharsm in #764
  * docs(known-instances): update list of known instances by @@lotharsm in #767
  * feat(localization): Add Traditional Chinese language translation by
    @@xlionjuan in #759
  * feat(lib/policy/expressions): add system load average to bot expression
    inputs by @@Xe in #766


1.20.0: Thancred Waters

Anubis now has support for weighing the soul of incoming requests with custom
rules and thresholds. Anubis also can function without the use of client-side
JavaScript using the metarefresh challenge.

The big ticket items are as follows:

  * Implement a no-JS challenge method: metarefresh (#95)
  * Implement request "weight", allowing administrators to customize the
    behaviour of Anubis based on specific criteria
  * Implement GeoIP and ASN based checks via Thoth (#206)
  * Add custom weight thresholds via CEL (#688)
  * Move Open Graph configuration to the policy file
  * Enable support for Open Graph metadata to be returned by default instead of
    doing lookups against the target
  * Add robots2policy CLI utility to convert robots.txt files to Anubis
    challenge policies using CEL expressions (#409)
  * Refactor challenge presentation logic to use a challenge registry
  * Allow challenge implementations to register HTTP routes
  * Imprint/Impressum support (#362)
  * Fix "invalid response" after "Success!" in Chromium (#564)

A lot of performance improvements have been made:

  * Replace internal SHA256 hashing with xxhash for 4-6x performance
    improvement in policy evaluation and cache operations
  * Optimized the OGTags subsystem with reduced allocations and runtime per
    request by up to 66%
  * Replace cidranger with bart for IP range checking, improving IP matching
    performance by 3-20x with zero heap
    allocations

And some cleanups/refactors were added:

  * Fix OpenGraph passthrough (#717)
  * Remove the unused /test-error endpoint and update the testing endpoint /
    make-challenge to only be enabled in
    development
  * Add --xff-strip-private flag/envvar to toggle skipping X-Forwarded-For
    private addresses or not
  * Bump AI-robots.txt to version 1.37
  * Make progress bar styling more compatible (UXP, etc)
  * Add --strip-base-prefix flag/envvar to strip the base prefix from request
    paths when forwarding to target servers
  * Fix an off-by-one in the default threshold config
  * Add functionality for HS512 JWT algorithm
  * Add support for dynamic cookie domains with the --cookie-dynamic-domain/
    COOKIE_DYNAMIC_DOMAIN flag/envvar

Request weight is one of the biggest ticket features in Anubis. This enables
Anubis to be much closer to a Web Application Firewall and when combined with
custom thresholds allows administrators to have Anubis take advanced reactions.
For more information about request weight, see the request weight section of
the policy file documentation.

TL;DR when you have one or more WEIGHT rules like this:

bots:
  - name: gitea-session-token
    action: WEIGH
    expression:
      all:
        - '"Cookie" in headers'
        - headers["Cookie"].contains("i_love_gitea=")
    # Remove 5 weight points
    weight:
      adjust: -5

You can configure custom thresholds like this:

thresholds:
  - name: minimal-suspicion # This client is likely fine, its soul is lighter than a feather
    expression: weight < 0 # a feather weighs zero units
    action: ALLOW # Allow the traffic through

  # For clients that had some weight reduced through custom rules, give them a
  # lightweight challenge.
  - name: mild-suspicion
    expression:
      all:
        - weight >= 0
        - weight < 10
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/metarefresh
      algorithm: metarefresh
      difficulty: 1
      report_as: 1

  # For clients that are browser-like but have either gained points from custom
  # rules or report as a standard browser.
  - name: moderate-suspicion
    expression:
      all:
        - weight >= 10
        - weight < 20
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
      algorithm: fast
      difficulty: 2 # two leading zeros, very fast for most clients
      report_as: 2

  # For clients that are browser like and have gained many points from custom
  # rules
  - name: extreme-suspicion
    expression: weight >= 20
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
      algorithm: fast
      difficulty: 4
      report_as: 4

These thresholds apply when no other ALLOW, DENY, or CHALLENGE rule matches the
request. WEIGHT rules add and remove request weight as needed:

bots:
  - name: gitea-session-token
    action: WEIGH
    expression:
      all:
        - '"Cookie" in headers'
        - headers["Cookie"].contains("i_love_gitea=")
    # Remove 5 weight points
    weight:
      adjust: -5

  - name: bot-like-user-agent
    action: WEIGH
    expression: '"Bot" in userAgent'
    # Add 5 weight points
    weight:
      adjust: 5

Of note: the default "generic browser" rule assigns 10 weight points:

# Generic catchall rule
- name: generic-browser
  user_agent_regex: >-
    Mozilla|Opera
  action: WEIGH
  weight:
    adjust: 10

Adjust this as you see fit.

What's Changed

  * build(deps-dev): bump esbuild from 0.25.4 to 0.25.5 in the npm group by
    @@dependabot in #600
  * build(deps): bump docker/build-push-action from 6.17.0 to 6.18.0 in the
    github-actions group by @@dependabot in #602
  * build(deps): bump github.com/a-h/templ from 0.3.865 to 0.3.887 in the gomod
    group by @@dependabot in #601
  * docs(faq): anubis does not mine bitcoin by @@Xe in #609
  * feat: implement challenge registry by @@Xe in #607
  * docs(known-instances): add Alliance of Hessian Libraries by @@CryptoCopter
    in #611
  * docs(subrequest-auth): document required policy changes by @@foosinn in #613
  * docs: Adjust the name of the cookie to the current
    "techaro.lol-anubis-auth" by @@jieter in #615
  * fix(lib/challenge): allow challenges to register HTTP routes by @@Xe in #620
  * docs(known-instances): add wiki.dolphin-emu.org to known instances by
    @@lotharsm in #626
  * feat(lib/challenge): HTTP meta refresh challenge method by @@Xe in #623
  * style: Some minor fixes by @@JasonLovesDoggo in #548
  * Bump ai.robots.txt to v1.34 by @@Dryusdan in #632
  * build(deps): bump the gomod group with 2 updates by @@dependabot in #634
  * docs(admin/environments): Prefer IPv6 over IPv4 for apache2 listener
    directive by @@lotharsm in #628
  * build(deps): bump github/codeql-action from 3.28.18 to 3.28.19 in the
    github-actions group by @@dependabot in #635
  * Adds ability to toggle off stripping of private addrs from XFF by
    @@dchandekstark in #619
  * Make progress bar styling more compatible (UXP, etc) by @@Fierelier in #636
  * feat(lib): implement request weight by @@Xe in #621
  * Update known-instances.md to include SquirrelJME by @@XerTheSquirrel in #643
  * fix(anubis): nil check policy loading by @@JasonLovesDoggo in #645
  * test: introduce SSH based CI for non-native test hosts by @@Xe in #644
  * build(deps): bump github.com/cloudflare/circl from 1.6.0 to 1.6.1 by
    @@dependabot in #650
  * test(ssh-ci): re-enable GOARCH=ppc64le by @@Xe in #651
  * fix(gitattributes): update pattern for generated files by @@JasonLovesDoggo
    in #652
  * fix(ci): conditionally run SSH jobs for TecharoHQ/anubis by
    @@JasonLovesDoggo in #654
  * feat: add a strip-base-prefix option by @@JasonLovesDoggo in #655
  * refactor(ogtags): optimize URL construction and memory allocations by
    @@JasonLovesDoggo in #647
  * docs(known-instances): add bugs.scummvm.org and gitlab.postmarketos.org by
    @@lotharsm in #661
  * feat: add robots2policy CLI to convert robots.txt to Anubis CEL by
    @@JasonLovesDoggo in #657
  * Add ReactOS to known-instances.md by @@ColinFinck in #664
  * build(deps): bump the github-actions group with 3 updates by @@dependabot in
    #666
  * Add the blog section back by @@Xe in #670
  * feat: implement a client for Thoth, the IP reputation database for Anubis
    by @@Xe in #637
  * chore(sponsors): update canine.tools logo by @@hyper...


1.20.0-pre2: Thancred Waters

What's Changed

  * Makefile: Build robots2policy by @@heftig in #699
  * fix(default-config): off-by-one error in the default thresholds by @@Xe in #
    701
  * feat: implement imprint/impressum support by @@Xe in #706
  * fix(web/js): broken progress bar with slow algo by @@yut23 in #673
  * build(deps): bump the github-actions group with 3 updates by @@dependabot in
    #708
  * fix(lib): fix invalid response after success in Chrome by @@Xe in #711

1.20.0-pre1 Thancred Waters

The big ticket items are as follows:

  * Implement a no-JS challenge method: metarefresh (#95)
  * Implement request "weight", allowing administrators to customize the
    behaviour of Anubis based on specific criteria
  * Implement GeoIP and ASN based checks via Thoth (#206)
  * Add custom weight thresholds via CEL (#688)
  * Move Open Graph configuration to the policy file
  * Enable support for Open Graph metadata to be returned by default instead of
    doing lookups against the target
  * Add robots2policy CLI utility to convert robots.txt files to Anubis
    challenge policies using CEL expressions (#409)
  * Refactor challenge presentation logic to use a challenge registry
  * Allow challenge implementations to register HTTP routes

A lot of performance improvements have been made:

  * Replace internal SHA256 hashing with xxhash for 4-6x performance
    improvement in policy evaluation and cache operations
  * Optimized the OGTags subsystem with reduced allocations and runtime per
    request by up to 66%
  * Replace cidranger with bart for IP range checking, improving IP matching
    performance by 3-20x with zero heap
    allocations

And some cleanups/refactors were added:

  * Remove the unused /test-error endpoint and update the testing endpoint /
    make-challenge to only be enabled in
    development
  * Add --xff-strip-private flag/envvar to toggle skipping X-Forwarded-For
    private addresses or not
  * Bump AI-robots.txt to version 1.37
  * Make progress bar styling more compatible (UXP, etc)
  * Add --strip-base-prefix flag/envvar to strip the base prefix from request
    paths when forwarding to target servers

Request weight is one of the biggest ticket features in Anubis. This enables
Anubis to be much closer to a Web Application Firewall and when combined with
custom thresholds allows administrators to have Anubis take advanced reactions.
For more information about request weight, see the request weight section of
the policy file documentation.

TL;DR when you have one or more WEIGHT rules like this:

bots:
  - name: gitea-session-token
    action: WEIGH
    expression:
      all:
        - '"Cookie" in headers'
        - headers["Cookie"].contains("i_love_gitea=")
    # Remove 5 weight points
    weight:
      adjust: -5

You can configure custom thresholds like this:

thresholds:
  - name: minimal-suspicion # This client is likely fine, its soul is lighter than a feather
    expression: weight < 0 # a feather weighs zero units
    action: ALLOW # Allow the traffic through

  # For clients that had some weight reduced through custom rules, give them a
  # lightweight challenge.
  - name: mild-suspicion
    expression:
      all:
        - weight >= 0
        - weight < 10
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/metarefresh
      algorithm: metarefresh
      difficulty: 1
      report_as: 1

  # For clients that are browser-like but have either gained points from custom
  # rules or report as a standard browser.
  - name: moderate-suspicion
    expression:
      all:
        - weight >= 10
        - weight < 20
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
      algorithm: fast
      difficulty: 2 # two leading zeros, very fast for most clients
      report_as: 2

  # For clients that are browser like and have gained many points from custom
  # rules
  - name: extreme-suspicion
    expression: weight >= 20
    action: CHALLENGE
    challenge:
      # https://anubis.techaro.lol/docs/admin/configuration/challenges/proof-of-work
      algorithm: fast
      difficulty: 4
      report_as: 4

These thresholds apply when no other ALLOW, DENY, or CHALLENGE rule matches the
request. WEIGHT rules add and remove request weight as needed:

bots:
  - name: gitea-session-token
    action: WEIGH
    expression:
      all:
        - '"Cookie" in headers'
        - headers["Cookie"].contains("i_love_gitea=")
    # Remove 5 weight points
    weight:
      adjust: -5

  - name: bot-like-user-agent
    action: WEIGH
    expression: '"Bot" in userAgent'
    # Add 5 weight points
    weight:
      adjust: 5

Of note: the default "generic browser" rule assigns 10 weight points:

# Generic catchall rule
- name: generic-browser
  user_agent_regex: >-
    Mozilla|Opera
  action: WEIGH
  weight:
    adjust: 10

Adjust this as you see fit.

What's Changed

  * build(deps-dev): bump esbuild from 0.25.4 to 0.25.5 in the npm group by
    @@dependabot in #600
  * build(deps): bump docker/build-push-action from 6.17.0 to 6.18.0 in the
    github-actions group by @@dependabot in #602
  * build(deps): bump github.com/a-h/templ from 0.3.865 to 0.3.887 in the gomod
    group by @@dependabot in #601
  * docs(faq): anubis does not mine bitcoin by @@Xe in #609
  * feat: implement challenge registry by @@Xe in #607
  * docs(known-instances): add Alliance of Hessian Libraries by @@CryptoCopter
    in #611
  * docs(subrequest-auth): document required policy changes by @@foosinn in #613
  * docs: Adjust the name of the cookie to the current
    "techaro.lol-anubis-auth" by @@jieter in #615
  * fix(lib/challenge): allow challenges to register HTTP routes by @@Xe in #620
  * docs(known-instances): add wiki.dolphin-emu.org to known instances by
    @@lotharsm in #626
  * feat(lib/challenge): HTTP meta refresh challenge method by @@Xe in #623
  * style: Some minor fixes by @@JasonLovesDoggo in #548
  * Bump ai.robots.txt to v1.34 by @@Dryusdan in #632
  * build(deps): bump the gomod group with 2 updates by @@dependabot in #634
  * docs(admin/environments): Prefer IPv6 over IPv4 for apache2 listener
    directive by @@lotharsm in #628
  * build(deps): bump github/codeql-action from 3.28.18 to 3.28.19 in the
    github-actions group by @@dependabot in #635
  * Adds ability to toggle off stripping of private addrs from XFF by
    @@dchandekstark in #619
  * Make progress bar styling more compatible (UXP, etc) by @@Fierelier in #636
  * feat(lib): implement request weight by @@Xe in #621
  * Update known-instances.md to include SquirrelJME by @@XerTheSquirrel in #643
  * fix(anubis): nil check policy loading by @@JasonLovesDoggo in #645
  * test: introduce SSH based CI for non-native test hosts by @@Xe in #644
  * build(deps): bump github.com/cloudflare/circl from 1.6.0 to 1.6.1 by
    @@dependabot in #650
  * test(ssh-ci): re-enable GOARCH=ppc64le by @@Xe in #651
  * fix(gitattributes): update pattern for generated files by @@JasonLovesDoggo
    in #652
  * fix(ci): conditionally run SSH jobs for TecharoHQ/anubis by
    @@JasonLovesDoggo in #654
  * feat: add a strip-base-prefix option by @@JasonLovesDoggo in #655
  * refactor(ogtags): optimize URL construction and memory allocations by
    @@JasonLovesDoggo in #647
  * docs(known-instances): add bugs.scummvm.org and gitlab.postmarketos.org by
    @@lotharsm in #661
  * feat: add robots2policy CLI to convert robots.txt to Anubis CEL by
    @@JasonLovesDoggo in #657
  * Add ReactOS to known-instances.md by @@ColinFinck in #664
  * build(deps): bump the github-actions group with 3 updates by @@dependabot in
    #666
  * Add the blog section back by @@Xe in #670
  * feat: implement a client for Thoth, the IP reputation database for Anubis
    by @@Xe in #637
  * chore(sponsors): update canine.tools logo by @@hyperdefined in #672
  * perf: Replace internal SHA256 hashing with xxhash for 4-6x performance
    improvement by @@JasonLovesDoggo in #676
  * perf: replace cidranger with bart for significant performance improvements
    by @@JasonLovesDoggo in #675
  * docs(known-instances): add wiki.koha-community.org by @@prettysunflower in #
    683
  * chore: remove duplicate CHANGELOG entry by @@JasonLovesDoggo in #684
  * fix(geo): correct typo "counties" to "countries" by @@hydrargyrum in #678
  * docs(known-instances): add extensions.ty...


1.19.1: Jenomis cen Lexentale - Echo 1

Return data/bots/ai-robots-txt.yaml to avoid breaking configs #599

This is a smaller release, mostly focused on improving compatibility and fixes
a few major issues with cookies.

Users should upgrade to this release as soon as possible.

What's Changed

  * style: apply structpack & goimport by @@JasonLovesDoggo in #469
  * feat: add TARGET_INSECURE_SKIP_VERIFY setting to allow self-signed HTTPS
    backends by @@Xe in #426
  * Add check-spelling v0.0.24 by @@jsoref in #462
  * Overhaul anubis.freebsd by @@pswilde in #427
  * ci(check-spelling): allow release names in spelling allowlists by @@Xe in #
    483
  * test(playwright): Add Docker and Podman support by @@SlyEcho in #433
  * chore(go.mod): move yeet to be a go tool by @@Xe in #485
  * fix(jwt): update nonce value in challenge JWT cookie to be a string by
    @@JasonLovesDoggo in #486
  * feat(ci): use dynamic repository owner and name in Docker actions by
    @@JasonLovesDoggo in #487
  * fix(bots/phrik): add IPv6 address for phrik by @@Xe in #494
  * build(deps-dev): bump the npm group with 3 updates by @@dependabot in #496
  * fix(lib): use a new cookie per domain when COOKIE_DOMAIN is set by @@Xe in #
    490
  * docs(known-instances): add some entries to the list by @@Xe in #497
  * fix(lib): make ClearCookie respect the dynamic cookie name by @@Xe in #500
  * fix(systemd): add RuntimeDirectory by @@Xe in #510
  * docs: add HTMX workaround by @@Xe in #511
  * Bump AI-robots.txt rules to version 1.30 by @@Dryusdan in #509
  * feat: add TARGET_HOST to allow overriding the Host header when forwarding
    requests by @@OatmealDome in #507
  * feat(apps): add SearXNG instance tracker policy and Qualys Labs SSL testing
    rules by @@Xe in #512
  * feat(apps): Make SASL login work on bookstack with Anubis by @@Dryusdan in #
    502
  * feat(lib): ensure that clients store cookies by @@Xe in #501
  * chore(docs/deploy): move to new cluster by @@Xe in #519
  * Add reddit.nerdvpn.de to known instances by @@Lenni-builder in #518
  * fix(lib): properly clear out test cookie by @@Xe in #522
  * build(deps): bump the github-actions group with 4 updates by @@dependabot in
    #523
  * docs: REDIRECT_DOMAINS must include port numbers by @@gucci-on-fleek in #521
  * docs: correct the path for the default configuration file by @@gravityfargo
    in #535
  * Bump AI-robots.txt rules to version 1.31 by @@Dryusdan in #538
  * feat: add TARGET_SNI to allow overriding the TLS handshake hostname when
    forwarding requests by @@jprenken in #529
  * fix(lib): record challenges issused over embedded HTML by @@Xe in #543
  * docs(native-install): vague gesturing at distribution package managers by
    @@Xe in #544
  * fix(expression): add validation for empty expression list in CEL by
    @@JasonLovesDoggo in #545
  * docs(admin): add wordpress docs by @@Xe in #552
  * Create Anubis OpenRC init.d script by @@CyberTailor in #561
  * build(deps): bump astral-sh/setup-uv from 6.0.1 to 6.1.0 in the
    github-actions group by @@dependabot in #558
  * add Weblate to known-instances.md by @@jordigh in #571
  * feat(cli): Add --version flag by @@kdkasad in #572
  * build(deps): bump k8s.io/apimachinery from 0.33.0 to 0.33.1 in the gomod
    group by @@dependabot in #524
  * fix(internal): register mime type for .mjs files by @@Xe in #577
  * feat(expressions): add randInt function to allow making rules
    nondeterministic by @@Xe in #578
  * feat(data): add x-firefox-ai default challenge rule by @@Xe in #580
  * fix(internal/test): skip integration tests if SKIP_INTEGRATION is set by
    @@Xe in #586
  * feat(yeetfile): build GOARCH=ppc64le packages by @@Xe in #583
  * feat(lib): Add proxied requests counter metric by @@kdkasad in #570
  * fix(web): show Anubis version number on challenge pages by @@Xe in #587
  * fix(lib): only use the first five characters of Accept-Language header
    values by @@Xe in #588
  * style(bench): small cleanup by @@JasonLovesDoggo in #546
  * feat(lib): annotate cookies with what rule was passed by @@Xe in #576
  * Add Applebot definition by @@tabletcorry in #589
  * docs(known-instances): Add Gitea by @@jesentz in #591
  * Opt-in policies for OpenAI and MistralAI bots by @@tabletcorry in #590
  * docs(known-instances): add openwrt.org by @@Aloki in #594
  * docs(known-instances): add catgirl.click by @@Zohiu in #597
  * add my site to known-instances.md by @@minihoot in #595
  * Split up AI filtering files by @@tabletcorry in #592


1.19.0: Jenomis cen Lexentale
NOTE:

Prefer v1.19.1. This has a config bug that was fixed in v1.19.1.

This is a smaller release, mostly focused on improving compatibility and fixes
a few major issues with cookies.

Users should upgrade to this release as soon as possible.

What's Changed

  * style: apply structpack & goimport by @@JasonLovesDoggo in #469
  * feat: add TARGET_INSECURE_SKIP_VERIFY setting to allow self-signed HTTPS
    backends by @@Xe in #426
  * Add check-spelling v0.0.24 by @@jsoref in #462
  * Overhaul anubis.freebsd by @@pswilde in #427
  * ci(check-spelling): allow release names in spelling allowlists by @@Xe in #
    483
  * test(playwright): Add Docker and Podman support by @@SlyEcho in #433
  * chore(go.mod): move yeet to be a go tool by @@Xe in #485
  * fix(jwt): update nonce value in challenge JWT cookie to be a string by
    @@JasonLovesDoggo in #486
  * feat(ci): use dynamic repository owner and name in Docker actions by
    @@JasonLovesDoggo in #487
  * fix(bots/phrik): add IPv6 address for phrik by @@Xe in #494
  * build(deps-dev): bump the npm group with 3 updates by @@dependabot in #496
  * fix(lib): use a new cookie per domain when COOKIE_DOMAIN is set by @@Xe in #
    490
  * docs(known-instances): add some entries to the list by @@Xe in #497
  * fix(lib): make ClearCookie respect the dynamic cookie name by @@Xe in #500
  * fix(systemd): add RuntimeDirectory by @@Xe in #510
  * docs: add HTMX workaround by @@Xe in #511
  * Bump AI-robots.txt rules to version 1.30 by @@Dryusdan in #509
  * feat: add TARGET_HOST to allow overriding the Host header when forwarding
    requests by @@OatmealDome in #507
  * feat(apps): add SearXNG instance tracker policy and Qualys Labs SSL testing
    rules by @@Xe in #512
  * feat(apps): Make SASL login work on bookstack with Anubis by @@Dryusdan in #
    502
  * feat(lib): ensure that clients store cookies by @@Xe in #501
  * chore(docs/deploy): move to new cluster by @@Xe in #519
  * Add reddit.nerdvpn.de to known instances by @@Lenni-builder in #518
  * fix(lib): properly clear out test cookie by @@Xe in #522
  * build(deps): bump the github-actions group with 4 updates by @@dependabot in
    #523
  * docs: REDIRECT_DOMAINS must include port numbers by @@gucci-on-fleek in #521
  * docs: correct the path for the default configuration file by @@gravityfargo
    in #535
  * Bump AI-robots.txt rules to version 1.31 by @@Dryusdan in #538
  * feat: add TARGET_SNI to allow overriding the TLS handshake hostname when
    forwarding requests by @@jprenken in #529
  * fix(lib): record challenges issused over embedded HTML by @@Xe in #543
  * docs(native-install): vague gesturing at distribution package managers by
    @@Xe in #544
  * fix(expression): add validation for empty expression list in CEL by
    @@JasonLovesDoggo in #545
  * docs(admin): add wordpress docs by @@Xe in #552
  * Create Anubis OpenRC init.d script by @@CyberTailor in #561
  * build(deps): bump astral-sh/setup-uv from 6.0.1 to 6.1.0 in the
    github-actions group by @@dependabot in #558
  * add Weblate to known-instances.md by @@jordigh in #571
  * feat(cli): Add --version flag by @@kdkasad in #572
  * build(deps): bump k8s.io/apimachinery from 0.33.0 to 0.33.1 in the gomod
    group by @@dependabot in #524
  * fix(internal): register mime type for .mjs files by @@Xe in #577
  * feat(expressions): add randInt function to allow making rules
    nondeterministic by @@Xe in #578
  * feat(data): add x-firefox-ai default challenge rule by @@Xe in #580
  * fix(internal/test): skip integration tests if SKIP_INTEGRATION is set by
    @@Xe in #586
  * feat(yeetfile): build GOARCH=ppc64le packages by @@Xe in #583
  * feat(lib): Add proxied requests counter metric by @@kdkasad in #570
  * fix(web): show Anubis version number on challenge pages by @@Xe in #587
  * fix(lib): only use the first five characters of Accept-Language header
    values by @@Xe in #588
  * style(bench): small cleanup by @@JasonLovesDoggo in #546
  * feat(lib): annotate cookies with what rule was passed by @@Xe in #576
  * Add Applebot definition by @@tabletcorry in #589
  * docs(known-instances): Add Gitea by @@jesentz in #591
  * Opt-in policies for OpenAI and MistralAI bots by @@tabletcorry in #590
  * docs(known-instances): add openwrt.org by @@Aloki in #594
  * docs(known-instances): add catgirl.click by @@Zohiu in #597
  * add my site to known-instances.md by @@minihoot in #595
  * Split up AI filtering files by @@tabletcorry in #592


1.19.0-pre1: Jenomis cen Lexentale
What's Changed

  * style: apply structpack & goimport by @@JasonLovesDoggo in #469
  * feat: add TARGET_INSECURE_SKIP_VERIFY setting to allow self-signed HTTPS
    backends by @@Xe in #426
  * Add check-spelling v0.0.24 by @@jsoref in #462
  * Overhaul anubis.freebsd by @@pswilde in #427
  * ci(check-spelling): allow release names in spelling allowlists by @@Xe in #
    483
  * test(playwright): Add Docker and Podman support by @@SlyEcho in #433
  * chore(go.mod): move yeet to be a go tool by @@Xe in #485
  * fix(jwt): update nonce value in challenge JWT cookie to be a string by
    @@JasonLovesDoggo in #486
  * feat(ci): use dynamic repository owner and name in Docker actions by
    @@JasonLovesDoggo in #487
  * fix(bots/phrik): add IPv6 address for phrik by @@Xe in #494
  * build(deps-dev): bump the npm group with 3 updates by @@dependabot in #496
  * fix(lib): use a new cookie per domain when COOKIE_DOMAIN is set by @@Xe in #
    490
  * docs(known-instances): add some entries to the list by @@Xe in #497
  * fix(lib): make ClearCookie respect the dynamic cookie name by @@Xe in #500
  * fix(systemd): add RuntimeDirectory by @@Xe in #510
  * docs: add HTMX workaround by @@Xe in #511
  * Bump AI-robots.txt rules to version 1.30 by @@Dryusdan in #509
  * feat: add TARGET_HOST to allow overriding the Host header when forwarding
    requests by @@OatmealDome in #507
  * feat(apps): add SearXNG instance tracker policy and Qualys Labs SSL testing
    rules by @@Xe in #512
  * feat(apps): Make SASL login work on bookstack with Anubis by @@Dryusdan in #
    502
  * feat(lib): ensure that clients store cookies by @@Xe in #501
  * chore(docs/deploy): move to new cluster by @@Xe in #519
  * Add reddit.nerdvpn.de to known instances by @@Lenni-builder in #518
  * fix(lib): properly clear out test cookie by @@Xe in #522
  * build(deps): bump the github-actions group with 4 updates by @@dependabot in
    #523
  * docs: REDIRECT_DOMAINS must include port numbers by @@gucci-on-fleek in #521
  * docs: correct the path for the default configuration file by @@gravityfargo
    in #535
  * Bump AI-robots.txt rules to version 1.31 by @@Dryusdan in #538
  * feat: add TARGET_SNI to allow overriding the TLS handshake hostname when
    forwarding requests by @@jprenken in #529
  * fix(lib): record challenges issused over embedded HTML by @@Xe in #543
  * docs(native-install): vague gesturing at distribution package managers by
    @@Xe in #544
  * fix(expression): add validation for empty expression list in CEL by
    @@JasonLovesDoggo in #545
  * docs(admin): add wordpress docs by @@Xe in #552
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.6 2025/06/06 14:01:41 bsiegert Exp $
d5 1
@


1.6
log
@Revbump all Go packages after go124 update
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.5 2025/05/24 14:41:18 ryoon Exp $
d3 2
a4 2
DISTNAME=	anubis-1.18.0
PKGREVISION=	4
d7 1
a7 1
GITHUB_TAG=	v${PKGVERSION_NOREV}
a13 6
ASSETS=		anubis-1.18.0-assets.tar.bz2
DISTFILES=	${DEFAULT_DISTFILES} ${ASSETS}
SITES.${ASSETS}=	${MASTER_SITE_LOCAL}

EXTRACT_ONLY=	${DEFAULT_DISTFILES} ${ASSETS}

@


1.5
log
@www/anubis: Read default.env configuration file in rc script

* Remove command line option, define SERVE_ROBOTS_TXT=true
  in default.env instead.
* Bump PKGREVISION.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.4 2025/05/22 15:14:14 ryoon Exp $
d4 1
a4 1
PKGREVISION=	3
@


1.4
log
@www/anubis: Add rc script and use PKGVERSION_NOREV instead of devel in URI

* Bump PKGREVISION.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.3 2025/05/20 15:16:41 ryoon Exp $
d4 1
a4 1
PKGREVISION=	2
@


1.3
log
@www/anubis: Fix main.mjs 404 not found runtime error

* Provide pregenerated JavaScript and CSS assets.
* Bump PKGREVISION.
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.2 2025/05/13 17:33:14 bsiegert Exp $
d4 1
a4 1
PKGREVISION=	1
d21 2
a22 1
GO_BUILD_PATTERN=	./cmd/anubis
d30 11
@


1.2
log
@anubis: update to 1.18.0

v1.18.0: Varis zos Galvus

The big ticket feature in this release is CEL expression matching
support. This allows you to tailor your approach for the individual
services you are protecting.

These can be as simple as:

- name: allow-api-requests
  action: ALLOW
  expression:
    all:
      - '"Accept" in headers'
      - 'headers["Accept"] == "application/json"'
      - 'path.startsWith("/api/")'

Or as complicated as:

- name: allow-git-clients
  action: ALLOW
  expression:
    all:
      - >-
        (
          userAgent.startsWith("git/") ||
          userAgent.contains("libgit") ||
          userAgent.startsWith("go-git") ||
          userAgent.startsWith("JGit/") ||
          userAgent.startsWith("JGit-")
        )
      - '"Git-Protocol" in headers'
      - headers["Git-Protocol"] == "version=2"

The docs have more information.  This is a simple, lovable, and complete
implementation of this feature so that administrators can get hacking
ASAP.

Other changes:

-   Use CSS variables to deduplicate styles
-   Fixed native packages not containing the stdlib and botPolicies.yaml
-   Change import syntax to allow multi-level imports
-   Changed the startup logging to use JSON formatting as all the other
    logs do.
-   Added the ability to do expression matching with CEL
-   Add a warning for clients that don't store cookies
-   Disable Open Graph passthrough by default
-   Clarify the license of the mascot images
-   Started Suppressing 'Context canceled' errors from http in the logs

v1.17.0: Asahi sas Brutus

v.1.17.0 is a rather large release. This kind of giant feature release
will not happen again as this has caused significant problems with
testing in various configurations. Automated testing is being worked on
but I have nothing to report yet.

Big-ticket features include but are not limited to:

-   Configuration can be in YAML or JSON
-   Configuration snippets can be imported from the default library or
    anywhere on the filesystem
-   Default rules now flag "Opera" after seeing an attack in the wild
    that does that
-   Many documentation and build script fixes
-   AI-robots.txt rules are added to the default config to stop the
    worst offenders that care to identify themselves
-   Apache, Nginx, and Traefik have gotten documentation
-   Users can match by headers as well as user agents or paths
-   Internal refactoring to make Anubis faster and easier to maintain
-   "Secondary screening" has been removed to give a more consistent
    user experience
-   The Internet Archive is allowlisted by default
-   X-Forwarded-For header calculation should be a bit better
-   Subpath support (run anubis on /git)
-   Many implicit things have been documented
@
text
@d1 1
a1 1
# $NetBSD: Makefile,v 1.1 2025/05/09 19:16:40 bsiegert Exp $
d4 1
d14 6
@


1.1
log
@New package, www/anubis.

Anubis weighs the soul of your connection using a sha256 proof-of-work
challenge in order to protect upstream resources from scraper bots.

Installing and using this will likely result in your website not being
indexed by some search engines. This is considered a feature of Anubis,
not a bug.

This is a bit of a nuclear response, but AI scraper bots scraping so
aggressively have forced my hand. I hate that I have to do this, but
this is what we get for the modern Internet because bots don't conform
to standards like robots.txt, even when they claim to.

In most cases, you should not need this and can probably get by using
Cloudflare to protect a given origin. However, for circumstances where
you can't or won't use Cloudflare, Anubis is there for you.
@
text
@d1 1
a1 1
# $NetBSD$
d3 1
a3 1
DISTNAME=	anubis-1.16.0
@