"""Web tools for Sentinel agents — fetch a URL's readable text.

Standard library only (no requirements to install). Runs in the leashed sandbox, so the
target host must be allowed by the operator's Leash egress policy; a blocked or failed
request returns a short error string rather than raising.
"""
from __future__ import annotations

import re
import urllib.error
import urllib.request

_TAG_RE = re.compile(r"<[^>]+>")
_WS_RE = re.compile(r"[ \t\r\f\v]+")
_BLANKS_RE = re.compile(r"\n\s*\n\s*")


def fetch_url(url: str, max_chars: int = 4000) -> str:
    """Fetch a web page and return its readable text (HTML tags stripped), truncated to
    `max_chars` characters. Use this to read an article, doc, or API page the user names.

    `url` must be an http(s) URL. Returns a short error string (never raises) if the URL is
    invalid, blocked by the egress policy, or the request fails."""
    if not isinstance(url, str) or not url.lower().startswith(("http://", "https://")):
        return "error: url must start with http:// or https://"
    req = urllib.request.Request(url, headers={"User-Agent": "sentinel-tool/1.0"})
    try:
        with urllib.request.urlopen(req, timeout=20) as resp:
            charset = resp.headers.get_content_charset() or "utf-8"
            raw = resp.read(2_000_000)  # cap the download at ~2 MB
    except urllib.error.HTTPError as exc:
        return f"error: HTTP {exc.code} fetching {url}"
    except (urllib.error.URLError, TimeoutError, ValueError) as exc:
        return f"error: could not fetch {url}: {exc}"
    body = raw.decode(charset, "replace")
    # Drop script/style blocks, then all tags, then collapse whitespace.
    body = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", body)
    text = _TAG_RE.sub(" ", body)
    text = _WS_RE.sub(" ", text)
    text = _BLANKS_RE.sub("\n\n", text).strip()
    if len(text) > max_chars:
        text = text[:max_chars].rstrip() + "\n…[truncated]"
    return text or "(no readable text found)"