diff --git a/src/tools/web.ts b/src/tools/web.ts index d542009..0fc3a3b 100644 --- a/src/tools/web.ts +++ b/src/tools/web.ts @@ -26,7 +26,7 @@ type StructuredFact = { id: number; claim: string; evidence_ids: number[]; sourc type SearchProvider = 'tavily' | 'google' | 'brave' | 'ddg' | 'ddg_html' | 'searxng' | 'ollama_cloud'; type SearchProviderAttempt = { provider: SearchProvider; - status: 'success' | 'failed' | 'skipped'; + status: 'success' | 'empty' | 'failed' | 'skipped'; reason?: string; duration_ms?: number; result_count?: number; @@ -813,7 +813,10 @@ export async function executeWebSearch(args: { query: string; max_results?: numb const cfg = getSearchConfig(); - const candidates: Array<'tavily' | 'searxng' | 'google' | 'brave' | 'ddg' | 'ollama_cloud'> = ['searxng', 'ollama_cloud', 'ddg', 'tavily', 'google', 'brave']; + // ddg sits last: its Instant Answer API only covers infobox-style queries and its + // HTML scrape fallback is routinely bot-walled (see searchDDGHtml) — it rarely adds + // value once searxng/ollama_cloud/tavily/google/brave have all had a shot. + const candidates: Array<'tavily' | 'searxng' | 'google' | 'brave' | 'ollama_cloud' | 'ddg'> = ['searxng', 'ollama_cloud', 'tavily', 'google', 'brave', 'ddg']; const providerOrder = [cfg.preferred, ...candidates.filter(p => p !== cfg.preferred)]; const diagnostics: SearchDiagnostics = { query: args.query, @@ -822,7 +825,24 @@ export async function executeWebSearch(args: { query: string; max_results?: numb attempted: [], }; + const runProvider = (provider: SearchProvider): Promise => { + switch (provider) { + case 'tavily': return searchTavily(args.query, limit, cfg.tavilyKey as string); + case 'searxng': return searchSearXNG(args.query, limit, cfg.searxngUrl as string); + case 'google': return searchGoogle(args.query, limit, cfg.googleKey as string, cfg.googleCx as string); + case 'brave': return searchBrave(args.query, limit, cfg.braveKey as string); + case 'ollama_cloud': return searchOllamaCloud(args.query, limit, cfg.ollamaApiKey as string); + case 'ddg': return searchDDG(args.query, limit); + default: throw new Error(`unhandled provider: ${provider}`); + } + }; + let lastErr = null; + // A provider that returned HTTP-OK but zero results is kept as a last-resort answer — + // better than a hard error if every provider ends up empty for this query. + let bestEmpty: ToolResult | null = null; + let bestEmptyProvider: SearchProvider | null = null; + for (const provider of providerOrder) { if (provider === 'tavily' && !cfg.tavilyKey) { diagnostics.attempted.push({ provider, status: 'skipped', reason: 'missing_tavily_api_key' }); @@ -847,89 +867,27 @@ export async function executeWebSearch(args: { query: string; max_results?: numb const started = Date.now(); try { - if (provider === 'tavily') { - const res = await searchTavily(args.query, limit, cfg.tavilyKey as string); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'tavily'; - res.data = { ...(res.data || {}), provider: 'tavily', search_diagnostics: diagnostics }; + const res = await runProvider(provider); + await augmentEventContract(args.query, res); + const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; + const hasResults = res.success && resultCount > 0; + + diagnostics.attempted.push({ + provider, + status: hasResults ? 'success' : (res.success ? 'empty' : 'failed'), + duration_ms: Date.now() - started, + result_count: resultCount, + ...(!res.success && { reason: res.error }), + }); + + if (hasResults) { + diagnostics.selected_provider = provider; + res.data = { ...(res.data || {}), provider, search_diagnostics: diagnostics }; return res; } - if (provider === 'searxng') { - const res = await searchSearXNG(args.query, limit, cfg.searxngUrl as string); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'searxng'; - res.data = { ...(res.data || {}), provider: 'searxng', search_diagnostics: diagnostics }; - return res; - } - if (provider === 'google') { - const res = await searchGoogle(args.query, limit, cfg.googleKey as string, cfg.googleCx as string); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'google'; - res.data = { ...(res.data || {}), provider: 'google', search_diagnostics: diagnostics }; - return res; - } - if (provider === 'brave') { - const res = await searchBrave(args.query, limit, cfg.braveKey as string); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'brave'; - res.data = { ...(res.data || {}), provider: 'brave', search_diagnostics: diagnostics }; - return res; - } - if (provider === 'ddg') { - const res = await searchDDG(args.query, limit); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'ddg'; - res.data = { ...(res.data || {}), provider: 'ddg', search_diagnostics: diagnostics }; - return res; - } - if (provider === 'ollama_cloud') { - const res = await searchOllamaCloud(args.query, limit, cfg.ollamaApiKey as string); - await augmentEventContract(args.query, res); - const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; - diagnostics.attempted.push({ - provider, - status: 'success', - duration_ms: Date.now() - started, - result_count: resultCount, - }); - diagnostics.selected_provider = 'ollama_cloud'; - res.data = { ...(res.data || {}), provider: 'ollama_cloud', search_diagnostics: diagnostics }; - return res; + if (res.success && !bestEmpty) { + bestEmpty = res; + bestEmptyProvider = provider; } } catch (err) { lastErr = err; @@ -942,20 +900,28 @@ export async function executeWebSearch(args: { query: string; max_results?: numb } } - // Final fallback if ddg path threw and wasn't already successful + // Final fallback: DDG HTML scrape (rarely succeeds — see comment on `candidates` above) const fallbackStarted = Date.now(); try { const res = await searchDDGHtml(args.query, limit); const resultCount = Array.isArray(res.data?.results) ? res.data.results.length : 0; + const hasResults = res.success && resultCount > 0; diagnostics.attempted.push({ provider: 'ddg_html', - status: 'success', + status: hasResults ? 'success' : (res.success ? 'empty' : 'failed'), duration_ms: Date.now() - fallbackStarted, result_count: resultCount, + ...(!res.success && { reason: res.error }), }); - diagnostics.selected_provider = 'ddg_html'; - res.data = { ...(res.data || {}), provider: 'ddg_html', search_diagnostics: diagnostics }; - return res; + if (hasResults) { + diagnostics.selected_provider = 'ddg_html'; + res.data = { ...(res.data || {}), provider: 'ddg_html', search_diagnostics: diagnostics }; + return res; + } + if (res.success && !bestEmpty) { + bestEmpty = res; + bestEmptyProvider = 'ddg_html'; + } } catch (err) { lastErr = err; diagnostics.attempted.push({ @@ -965,6 +931,13 @@ export async function executeWebSearch(args: { query: string; max_results?: numb duration_ms: Date.now() - fallbackStarted, }); } + + if (bestEmpty) { + diagnostics.selected_provider = bestEmptyProvider as SearchProvider; + bestEmpty.data = { ...(bestEmpty.data || {}), provider: bestEmptyProvider, search_diagnostics: diagnostics }; + return bestEmpty; + } + let errMsg = 'unknown error'; if (lastErr) { if (typeof lastErr === 'object' && 'message' in lastErr) errMsg = (lastErr as any).message;