# web-fetch-guard: an agent hook by Scalably

Canonical: https://scalably.io/hooks/web-fetch-guard
Source: https://github.com/scalably-io/agent-hooks/blob/v1.0.0/hooks/web-fetch-guard.sh
Runs on: PreToolUse on WebFetch and PostToolUse on WebFetch
Release: v1.0.0 at commit 53dcece
Integrity: sha256 of the script served at https://scalably.io/hooks/web-fetch-guard.sh is 48bbf9ddc1de6999b5858b5cb0810d503baa7c6a08358daa895c01d9bac9d64b. Verify: curl -s https://scalably.io/hooks/web-fetch-guard.sh | shasum -a 256
This is the machine-readable representation of the page at the canonical URL. Same facts, denser format.

## Summary

PreToolUse and PostToolUse hook (WebFetch): stop asking a site that has already refused.

## Install

```
/plugin marketplace add scalably-io/agent-skills
/plugin install agent-hooks@scalably-agent-skills
```

## What it does

WHAT THIS IS. When a site returns a refusal, an agent will often retry the
same domain over and over, burning the turn and stalling the work. This hook
remembers which domains refused and declines to ask them again.

WHAT THIS IS NOT. It does not alter requests to look more human, rotate
anything, or work around a refusal in any way. Its only action is to stop
asking. A site that says no is taken at its word, which is both the polite
behaviour and the one that gets the agent unstuck.

Scoring, so that one bad minute does not blacklist a site forever:
```text
  403 or a challenge page   a clear refusal, enough on its own
  429 or 503                could be transient, so it takes two
  a successful fetch        clears the transient score for that domain
```

Configuration:
```text
  WEB_FETCH_SCORES   path to the score file. Defaults to
                     .claude/web-fetch-scores.json in the working directory.
```

Exit: always 0. The decision travels in the JSON, and any internal problem
fails open rather than blocking the session.

## The whole script

```bash
#!/usr/bin/env bash
# PreToolUse and PostToolUse hook (WebFetch): stop asking a site that has
# already refused.
#
# WHAT THIS IS. When a site returns a refusal, an agent will often retry the
# same domain over and over, burning the turn and stalling the work. This hook
# remembers which domains refused and declines to ask them again.
#
# WHAT THIS IS NOT. It does not alter requests to look more human, rotate
# anything, or work around a refusal in any way. Its only action is to stop
# asking. A site that says no is taken at its word, which is both the polite
# behaviour and the one that gets the agent unstuck.
#
# Scoring, so that one bad minute does not blacklist a site forever:
#   403 or a challenge page   a clear refusal, enough on its own
#   429 or 503                could be transient, so it takes two
#   a successful fetch        clears the transient score for that domain
#
# Configuration:
#   WEB_FETCH_SCORES   path to the score file. Defaults to
#                      .claude/web-fetch-scores.json in the working directory.
#
# Exit: always 0. The decision travels in the JSON, and any internal problem
# fails open rather than blocking the session.

BLOCK_THRESHOLD=2

input=$(cat)
event=$(printf '%s' "$input" | jq -r '.hook_event_name // ""' 2>/dev/null) || exit 0
tool=$(printf '%s' "$input" | jq -r '.tool_name // ""' 2>/dev/null) || exit 0
[ "$tool" = "WebFetch" ] || exit 0

url=$(printf '%s' "$input" | jq -r '.tool_input.url // ""' 2>/dev/null) || exit 0
[ -n "$url" ] || exit 0

domain=$(printf '%s' "$url" | sed -E 's#^[a-zA-Z]+://##; s#/.*$##; s#:[0-9]+$##' | tr '[:upper:]' '[:lower:]')
[ -n "$domain" ] || exit 0

store="${WEB_FETCH_SCORES:-.claude/web-fetch-scores.json}"

read_score() {
  [ -f "$store" ] || { printf '0'; return; }
  jq -r --arg d "$domain" '(.[$d] // 0) | tostring' "$store" 2>/dev/null || printf '0'
}

write_score() {
  local value="$1" dir tmp
  dir=$(dirname "$store")
  mkdir -p "$dir" 2>/dev/null || return 0
  tmp=$(mktemp) || return 0
  if [ -f "$store" ] && jq -e . "$store" >/dev/null 2>&1; then
    jq --arg d "$domain" --argjson v "$value" '.[$d] = $v' "$store" > "$tmp" 2>/dev/null || printf '{"%s":%s}' "$domain" "$value" > "$tmp"
  else
    printf '{"%s":%s}' "$domain" "$value" > "$tmp"
  fi
  mv "$tmp" "$store" 2>/dev/null || rm -f "$tmp"
}

case "$event" in
  PreToolUse)
    score=$(read_score)
    case "$score" in ''|*[!0-9]*) score=0 ;; esac
    if [ "$score" -ge "$BLOCK_THRESHOLD" ]; then
      reason="This domain has already refused automated requests: ${domain}. Asking again will not change the answer, and repeating the request is not acceptable behaviour toward a site that declined. Find the information another way, or report that this source is unavailable and continue with the rest of the task."
      jq -nc --arg r "$reason" '{
        hookSpecificOutput: {
          hookEventName: "PreToolUse",
          permissionDecision: "deny",
          permissionDecisionReason: $r
        }
      }' 2>/dev/null
    fi
    exit 0
    ;;

  PostToolUse)
    body=$(printf '%s' "$input" | jq -r '(.tool_output // "") | tostring' 2>/dev/null) || exit 0
    lower=$(printf '%s' "$body" | tr '[:upper:]' '[:lower:]')
    score=$(read_score)
    case "$score" in ''|*[!0-9]*) score=0 ;; esac

    strong=0
    printf '%s' "$lower" | grep -qE 'http[/ ]?[0-9.]* ?403|(^|[^0-9])403([^0-9]|$).*(forbidden|denied)|forbidden' && strong=1
    printf '%s' "$lower" | grep -qE 'cloudflare|captcha|just a moment|checking your browser|ray id' && strong=1

    weak=0
    printf '%s' "$lower" | grep -qE '(^|[^0-9])(429|503)([^0-9]|$)|too many requests|service unavailable' && weak=1

    if [ "$strong" -eq 1 ]; then
      write_score "$BLOCK_THRESHOLD"
    elif [ "$weak" -eq 1 ]; then
      write_score "$((score + 1))"
    else
      # A real response clears a transient score. A domain that gave a clear
      # refusal keeps it, because that was not a blip.
      [ "$score" -lt "$BLOCK_THRESHOLD" ] && [ "$score" -gt 0 ] && write_score 0
    fi
    exit 0
    ;;
esac

exit 0
```