diff --git a/src/agent/tools/web/cache.test.ts b/src/agent/tools/web/cache.test.ts new file mode 100644 index 00000000..85e461cf --- /dev/null +++ b/src/agent/tools/web/cache.test.ts @@ -0,0 +1,252 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + resolveTimeoutSeconds, + resolveCacheTtlMs, + normalizeCacheKey, + readCache, + writeCache, + withTimeout, + type CacheEntry, +} from "./cache.js"; + +describe("cache", () => { + describe("resolveTimeoutSeconds", () => { + it("should return the value if it is a valid number", () => { + expect(resolveTimeoutSeconds(30, 10)).toBe(30); + expect(resolveTimeoutSeconds(60, 10)).toBe(60); + }); + + it("should return fallback for non-number values", () => { + expect(resolveTimeoutSeconds("30", 10)).toBe(10); + expect(resolveTimeoutSeconds(null, 10)).toBe(10); + expect(resolveTimeoutSeconds(undefined, 10)).toBe(10); + expect(resolveTimeoutSeconds({}, 10)).toBe(10); + }); + + it("should return fallback for non-finite numbers", () => { + expect(resolveTimeoutSeconds(NaN, 10)).toBe(10); + expect(resolveTimeoutSeconds(Infinity, 10)).toBe(10); + expect(resolveTimeoutSeconds(-Infinity, 10)).toBe(10); + }); + + it("should enforce minimum of 1 second", () => { + expect(resolveTimeoutSeconds(0, 10)).toBe(1); + expect(resolveTimeoutSeconds(-5, 10)).toBe(1); + expect(resolveTimeoutSeconds(0.5, 10)).toBe(1); + }); + + it("should floor decimal values", () => { + expect(resolveTimeoutSeconds(5.9, 10)).toBe(5); + expect(resolveTimeoutSeconds(10.1, 5)).toBe(10); + }); + }); + + describe("resolveCacheTtlMs", () => { + it("should convert minutes to milliseconds", () => { + expect(resolveCacheTtlMs(1, 15)).toBe(60_000); + expect(resolveCacheTtlMs(15, 15)).toBe(900_000); + expect(resolveCacheTtlMs(60, 15)).toBe(3_600_000); + }); + + it("should return fallback for non-number values", () => { + expect(resolveCacheTtlMs("15", 15)).toBe(900_000); + expect(resolveCacheTtlMs(null, 10)).toBe(600_000); + expect(resolveCacheTtlMs(undefined, 5)).toBe(300_000); + }); + + it("should handle zero and negative values", () => { + expect(resolveCacheTtlMs(0, 15)).toBe(0); + expect(resolveCacheTtlMs(-5, 15)).toBe(0); + }); + + it("should handle fractional minutes", () => { + expect(resolveCacheTtlMs(0.5, 15)).toBe(30_000); + expect(resolveCacheTtlMs(1.5, 15)).toBe(90_000); + }); + }); + + describe("normalizeCacheKey", () => { + it("should trim whitespace", () => { + expect(normalizeCacheKey(" key ")).toBe("key"); + expect(normalizeCacheKey("\tkey\n")).toBe("key"); + }); + + it("should lowercase the key", () => { + expect(normalizeCacheKey("KEY")).toBe("key"); + expect(normalizeCacheKey("MyKey")).toBe("mykey"); + expect(normalizeCacheKey("HTTPS://EXAMPLE.COM")).toBe("https://example.com"); + }); + + it("should handle empty string", () => { + expect(normalizeCacheKey("")).toBe(""); + expect(normalizeCacheKey(" ")).toBe(""); + }); + }); + + describe("readCache", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should return null for missing key", () => { + const cache = new Map>(); + expect(readCache(cache, "missing")).toBeNull(); + }); + + it("should return cached value if not expired", () => { + const cache = new Map>(); + const now = Date.now(); + cache.set("key", { + value: "test-value", + expiresAt: now + 60_000, + insertedAt: now, + }); + + const result = readCache(cache, "key"); + expect(result).toEqual({ value: "test-value", cached: true }); + }); + + it("should return null and delete expired entries", () => { + const cache = new Map>(); + const now = Date.now(); + cache.set("key", { + value: "test-value", + expiresAt: now - 1000, // expired + insertedAt: now - 60_000, + }); + + const result = readCache(cache, "key"); + expect(result).toBeNull(); + expect(cache.has("key")).toBe(false); + }); + + it("should delete entry when exactly at expiration time", () => { + const cache = new Map>(); + const now = Date.now(); + cache.set("key", { + value: "test-value", + expiresAt: now, + insertedAt: now - 60_000, + }); + + vi.advanceTimersByTime(1); // Move past expiration + const result = readCache(cache, "key"); + expect(result).toBeNull(); + }); + }); + + describe("writeCache", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should write entry with correct expiration", () => { + const cache = new Map>(); + const now = Date.now(); + + writeCache(cache, "key", "value", 60_000); + + const entry = cache.get("key"); + expect(entry).toBeDefined(); + expect(entry?.value).toBe("value"); + expect(entry?.expiresAt).toBe(now + 60_000); + expect(entry?.insertedAt).toBe(now); + }); + + it("should not write if ttl is 0 or negative", () => { + const cache = new Map>(); + + writeCache(cache, "key1", "value1", 0); + writeCache(cache, "key2", "value2", -100); + + expect(cache.size).toBe(0); + }); + + it("should evict oldest entry when cache is full", () => { + const cache = new Map>(); + const now = Date.now(); + + // Fill up to max (100 entries) + for (let i = 0; i < 100; i++) { + cache.set(`key${i}`, { + value: `value${i}`, + expiresAt: now + 60_000, + insertedAt: now, + }); + } + + expect(cache.size).toBe(100); + + // Add one more - should evict first + writeCache(cache, "new-key", "new-value", 60_000); + + expect(cache.size).toBe(100); + expect(cache.has("key0")).toBe(false); + expect(cache.has("new-key")).toBe(true); + }); + + it("should overwrite existing entry", () => { + const cache = new Map>(); + + writeCache(cache, "key", "value1", 60_000); + writeCache(cache, "key", "value2", 60_000); + + expect(cache.size).toBe(1); + expect(cache.get("key")?.value).toBe("value2"); + }); + }); + + describe("withTimeout", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("should return aborted signal after timeout", async () => { + const signal = withTimeout(undefined, 1000); + + expect(signal.aborted).toBe(false); + + vi.advanceTimersByTime(1000); + + expect(signal.aborted).toBe(true); + }); + + it("should abort immediately if timeout is 0", () => { + const signal = withTimeout(undefined, 0); + expect(signal.aborted).toBe(false); + }); + + it("should abort when parent signal aborts", () => { + const parentController = new AbortController(); + const signal = withTimeout(parentController.signal, 60_000); + + expect(signal.aborted).toBe(false); + + parentController.abort(); + + expect(signal.aborted).toBe(true); + }); + + it("should clear timeout when parent signal aborts", () => { + const parentController = new AbortController(); + const clearTimeoutSpy = vi.spyOn(global, "clearTimeout"); + + withTimeout(parentController.signal, 60_000); + parentController.abort(); + + expect(clearTimeoutSpy).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/agent/tools/web/html-utils.test.ts b/src/agent/tools/web/html-utils.test.ts new file mode 100644 index 00000000..5477f241 --- /dev/null +++ b/src/agent/tools/web/html-utils.test.ts @@ -0,0 +1,230 @@ +import { describe, it, expect } from "vitest"; +import { + htmlToMarkdownSimple, + markdownToText, + truncateText, + convertWithTurndown, +} from "./html-utils.js"; + +describe("html-utils", () => { + describe("htmlToMarkdownSimple", () => { + it("should extract title from HTML", () => { + const html = "Test PageContent"; + const result = htmlToMarkdownSimple(html); + expect(result.title).toBe("Test Page"); + }); + + it("should handle missing title", () => { + const html = "Content"; + const result = htmlToMarkdownSimple(html); + expect(result.title).toBeUndefined(); + }); + + it("should remove script tags", () => { + const html = "

Before

After

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).not.toContain("alert"); + expect(result.text).toContain("Before"); + expect(result.text).toContain("After"); + }); + + it("should remove style tags", () => { + const html = "

Content

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).not.toContain("color"); + expect(result.text).toContain("Content"); + }); + + it("should remove noscript tags", () => { + const html = "

Content

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).not.toContain("JavaScript"); + }); + + it("should convert links to markdown format", () => { + const html = 'Example'; + const result = htmlToMarkdownSimple(html); + expect(result.text).toBe("[Example](https://example.com)"); + }); + + it("should handle links without text", () => { + const html = ''; + const result = htmlToMarkdownSimple(html); + expect(result.text).toBe("https://example.com"); + }); + + it("should convert headings to markdown", () => { + const html = "

Title

Subtitle

Section

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).toContain("# Title"); + expect(result.text).toContain("## Subtitle"); + expect(result.text).toContain("### Section"); + }); + + it("should convert list items", () => { + const html = "
  • Item 1
  • Item 2
"; + const result = htmlToMarkdownSimple(html); + expect(result.text).toContain("- Item 1"); + expect(result.text).toContain("- Item 2"); + }); + + it("should convert br and hr tags", () => { + const html = "

Line 1
Line 2


Line 3

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).toContain("Line 1"); + expect(result.text).toContain("Line 2"); + expect(result.text).toContain("Line 3"); + }); + + it("should decode HTML entities", () => { + const html = "

Hello & World <test>

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).toContain("Hello & World "); + }); + + it("should decode numeric entities", () => { + const html = "

<tag> <hex>

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).toContain(""); + expect(result.text).toContain(""); + }); + + it("should normalize whitespace", () => { + const html = "

Text with lots of spaces

"; + const result = htmlToMarkdownSimple(html); + expect(result.text).not.toContain(" "); + }); + + it("should handle empty HTML", () => { + const result = htmlToMarkdownSimple(""); + expect(result.text).toBe(""); + expect(result.title).toBeUndefined(); + }); + }); + + describe("markdownToText", () => { + it("should remove image syntax", () => { + const md = "Text ![alt](image.png) more text"; + const result = markdownToText(md); + expect(result).not.toContain("!["); + expect(result).toContain("Text"); + expect(result).toContain("more text"); + }); + + it("should extract link text and remove URLs", () => { + const md = "Click [here](https://example.com) for more"; + const result = markdownToText(md); + expect(result).toBe("Click here for more"); + }); + + it("should remove code blocks", () => { + const md = "Text\n```javascript\nconst x = 1;\n```\nMore text"; + const result = markdownToText(md); + expect(result).not.toContain("```"); + expect(result).toContain("const x = 1;"); + }); + + it("should remove inline code backticks", () => { + const md = "Use the `console.log` function"; + const result = markdownToText(md); + expect(result).toBe("Use the console.log function"); + }); + + it("should remove heading markers", () => { + const md = "# Title\n## Subtitle\n### Section"; + const result = markdownToText(md); + expect(result).not.toContain("#"); + expect(result).toContain("Title"); + expect(result).toContain("Subtitle"); + }); + + it("should remove list markers", () => { + const md = "- Item 1\n* Item 2\n+ Item 3\n1. Numbered"; + const result = markdownToText(md); + expect(result).not.toMatch(/^[-*+]\s/m); + expect(result).not.toMatch(/^\d+\.\s/m); + expect(result).toContain("Item 1"); + }); + + it("should handle empty string", () => { + expect(markdownToText("")).toBe(""); + }); + + it("should normalize whitespace", () => { + const md = "Text with spaces\n\n\nand lines"; + const result = markdownToText(md); + expect(result).not.toContain(" "); + expect(result).not.toContain("\n\n\n"); + }); + }); + + describe("truncateText", () => { + it("should not truncate text under max length", () => { + const result = truncateText("Hello", 10); + expect(result.text).toBe("Hello"); + expect(result.truncated).toBe(false); + }); + + it("should truncate text over max length", () => { + const result = truncateText("Hello World", 5); + expect(result.text).toBe("Hello"); + expect(result.truncated).toBe(true); + }); + + it("should handle exact length", () => { + const result = truncateText("Hello", 5); + expect(result.text).toBe("Hello"); + expect(result.truncated).toBe(false); + }); + + it("should handle empty string", () => { + const result = truncateText("", 10); + expect(result.text).toBe(""); + expect(result.truncated).toBe(false); + }); + + it("should handle zero max chars", () => { + const result = truncateText("Hello", 0); + expect(result.text).toBe(""); + expect(result.truncated).toBe(true); + }); + }); + + describe("convertWithTurndown", () => { + it("should convert HTML to markdown", () => { + const html = "Page

Hello

World

"; + const result = convertWithTurndown(html); + expect(result.title).toBe("Page"); + expect(result.text).toContain("# Hello"); + expect(result.text).toContain("World"); + }); + + it("should remove script and style tags", () => { + const html = "

Content

"; + const result = convertWithTurndown(html); + expect(result.text).not.toContain("alert"); + expect(result.text).not.toContain(".x{}"); + expect(result.text).toContain("Content"); + }); + + it("should convert links", () => { + const html = 'Link'; + const result = convertWithTurndown(html); + expect(result.text).toContain("[Link](https://example.com)"); + }); + + it("should convert lists with dash markers", () => { + const html = "
  • One
  • Two
"; + const result = convertWithTurndown(html); + expect(result.text).toContain("-"); + expect(result.text).toContain("One"); + expect(result.text).toContain("Two"); + }); + + it("should handle code blocks", () => { + const html = "
const x = 1;
"; + const result = convertWithTurndown(html); + expect(result.text).toContain("const x = 1;"); + }); + }); +}); diff --git a/src/agent/tools/web/ssrf.test.ts b/src/agent/tools/web/ssrf.test.ts new file mode 100644 index 00000000..cda9e22a --- /dev/null +++ b/src/agent/tools/web/ssrf.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect } from "vitest"; +import { isPrivateIpAddress, isBlockedHostname, SsrfBlockedError } from "./ssrf.js"; + +describe("ssrf", () => { + describe("isPrivateIpAddress", () => { + describe("IPv4 private ranges", () => { + it("should block 10.x.x.x range", () => { + expect(isPrivateIpAddress("10.0.0.1")).toBe(true); + expect(isPrivateIpAddress("10.255.255.255")).toBe(true); + expect(isPrivateIpAddress("10.50.100.200")).toBe(true); + }); + + it("should block 172.16.x.x - 172.31.x.x range", () => { + expect(isPrivateIpAddress("172.16.0.1")).toBe(true); + expect(isPrivateIpAddress("172.31.255.255")).toBe(true); + expect(isPrivateIpAddress("172.20.100.50")).toBe(true); + }); + + it("should not block 172.15.x.x or 172.32.x.x", () => { + expect(isPrivateIpAddress("172.15.0.1")).toBe(false); + expect(isPrivateIpAddress("172.32.0.1")).toBe(false); + }); + + it("should block 192.168.x.x range", () => { + expect(isPrivateIpAddress("192.168.0.1")).toBe(true); + expect(isPrivateIpAddress("192.168.255.255")).toBe(true); + expect(isPrivateIpAddress("192.168.1.100")).toBe(true); + }); + + it("should block 127.x.x.x loopback range", () => { + expect(isPrivateIpAddress("127.0.0.1")).toBe(true); + expect(isPrivateIpAddress("127.255.255.255")).toBe(true); + expect(isPrivateIpAddress("127.0.0.0")).toBe(true); + }); + + it("should block 169.254.x.x link-local range", () => { + expect(isPrivateIpAddress("169.254.0.1")).toBe(true); + expect(isPrivateIpAddress("169.254.255.255")).toBe(true); + }); + + it("should block 0.x.x.x range", () => { + expect(isPrivateIpAddress("0.0.0.0")).toBe(true); + expect(isPrivateIpAddress("0.0.0.1")).toBe(true); + }); + + it("should block 100.64.x.x - 100.127.x.x CGNAT range", () => { + expect(isPrivateIpAddress("100.64.0.1")).toBe(true); + expect(isPrivateIpAddress("100.127.255.255")).toBe(true); + expect(isPrivateIpAddress("100.100.50.25")).toBe(true); + }); + + it("should not block 100.63.x.x or 100.128.x.x", () => { + expect(isPrivateIpAddress("100.63.0.1")).toBe(false); + expect(isPrivateIpAddress("100.128.0.1")).toBe(false); + }); + + it("should allow public IPs", () => { + expect(isPrivateIpAddress("8.8.8.8")).toBe(false); + expect(isPrivateIpAddress("1.1.1.1")).toBe(false); + expect(isPrivateIpAddress("203.0.113.50")).toBe(false); + expect(isPrivateIpAddress("198.51.100.100")).toBe(false); + }); + }); + + describe("IPv6 addresses", () => { + it("should block loopback ::1", () => { + expect(isPrivateIpAddress("::1")).toBe(true); + expect(isPrivateIpAddress("::")).toBe(true); + }); + + it("should block fe80: link-local", () => { + expect(isPrivateIpAddress("fe80::1")).toBe(true); + expect(isPrivateIpAddress("fe80:0000:0000:0000:0000:0000:0000:0001")).toBe(true); + }); + + it("should block fc/fd unique local addresses", () => { + expect(isPrivateIpAddress("fc00::1")).toBe(true); + expect(isPrivateIpAddress("fd00::1")).toBe(true); + expect(isPrivateIpAddress("fdab:cdef:1234::1")).toBe(true); + }); + + it("should block fec0: site-local (deprecated)", () => { + expect(isPrivateIpAddress("fec0::1")).toBe(true); + }); + }); + + describe("IPv4-mapped IPv6 addresses", () => { + it("should block ::ffff:10.x.x.x", () => { + expect(isPrivateIpAddress("::ffff:10.0.0.1")).toBe(true); + expect(isPrivateIpAddress("::ffff:10.255.255.255")).toBe(true); + }); + + it("should block ::ffff:127.0.0.1", () => { + expect(isPrivateIpAddress("::ffff:127.0.0.1")).toBe(true); + }); + + it("should block ::ffff:192.168.x.x", () => { + expect(isPrivateIpAddress("::ffff:192.168.1.1")).toBe(true); + }); + + it("should allow ::ffff:public IPs", () => { + expect(isPrivateIpAddress("::ffff:8.8.8.8")).toBe(false); + expect(isPrivateIpAddress("::ffff:1.1.1.1")).toBe(false); + }); + }); + + describe("edge cases", () => { + it("should handle bracketed IPv6", () => { + expect(isPrivateIpAddress("[::1]")).toBe(true); + expect(isPrivateIpAddress("[fe80::1]")).toBe(true); + }); + + it("should handle whitespace", () => { + expect(isPrivateIpAddress(" 10.0.0.1 ")).toBe(true); + expect(isPrivateIpAddress("\t192.168.1.1\n")).toBe(true); + }); + + it("should handle case insensitivity", () => { + expect(isPrivateIpAddress("FE80::1")).toBe(true); + expect(isPrivateIpAddress("FC00::1")).toBe(true); + }); + + it("should return false for empty string", () => { + expect(isPrivateIpAddress("")).toBe(false); + expect(isPrivateIpAddress(" ")).toBe(false); + }); + + it("should return false for invalid IPs", () => { + expect(isPrivateIpAddress("not-an-ip")).toBe(false); + expect(isPrivateIpAddress("256.256.256.256")).toBe(false); + expect(isPrivateIpAddress("192.168.1")).toBe(false); + }); + }); + }); + + describe("isBlockedHostname", () => { + it("should block localhost", () => { + expect(isBlockedHostname("localhost")).toBe(true); + expect(isBlockedHostname("LOCALHOST")).toBe(true); + expect(isBlockedHostname("LocalHost")).toBe(true); + }); + + it("should block metadata.google.internal", () => { + expect(isBlockedHostname("metadata.google.internal")).toBe(true); + }); + + it("should block .localhost subdomains", () => { + expect(isBlockedHostname("foo.localhost")).toBe(true); + expect(isBlockedHostname("sub.domain.localhost")).toBe(true); + }); + + it("should block .local domains", () => { + expect(isBlockedHostname("myhost.local")).toBe(true); + expect(isBlockedHostname("printer.local")).toBe(true); + }); + + it("should block .internal domains", () => { + expect(isBlockedHostname("myservice.internal")).toBe(true); + expect(isBlockedHostname("app.internal")).toBe(true); + }); + + it("should handle trailing dots", () => { + expect(isBlockedHostname("localhost.")).toBe(true); + expect(isBlockedHostname("foo.local.")).toBe(true); + }); + + it("should allow public domains", () => { + expect(isBlockedHostname("google.com")).toBe(false); + expect(isBlockedHostname("github.com")).toBe(false); + expect(isBlockedHostname("example.org")).toBe(false); + }); + + it("should return false for empty hostname", () => { + expect(isBlockedHostname("")).toBe(false); + expect(isBlockedHostname(" ")).toBe(false); + }); + }); + + describe("SsrfBlockedError", () => { + it("should be an instance of Error", () => { + const error = new SsrfBlockedError("test message"); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe("test message"); + expect(error.name).toBe("SsrfBlockedError"); + }); + }); +});