diff --git a/components/tool/duckduckgo/README.md b/components/tool/duckduckgo/README.md new file mode 100644 index 0000000..c69ceae --- /dev/null +++ b/components/tool/duckduckgo/README.md @@ -0,0 +1,98 @@ +# DuckDuckGo Search Tool + +English | [简体中文](README_zh.md) + +A DuckDuckGo search tool implementation for [Eino](https://github.com/cloudwego/eino) that implements the `InvokableTool` interface. This enables seamless integration with Eino's ChatModel interaction system and `ToolsNode` for enhanced search capabilities. + +## Features + +- Implements `github.com/cloudwego/eino/components/tool.InvokableTool` +- Easy integration with Eino's tool system +- Configurable search parameters + +## Installation + +```bash +go get github.com/cloudwego/eino-ext/components/tool/duckduckgo +``` + +## Quick Start + +```go +package main + +import ( + "context" + "log" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo" + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" + "github.com/cloudwego/eino/components/tool" +) + +func main() { + // Create tool config + cfg := &duckduckgo.Config{ // All of these parameters are default values, for demonstration purposes only + ToolName: "duckduckgo_search", + ToolDesc: "search web for information by duckduckgo", + Region: ddgsearch.RegionWT, + Retries: 3, + Timeout: 10, + MaxResults: 10, + } + + // Create the search tool + searchTool, err := duckduckgo.NewTool(context.Background(), cfg) + if err != nil { + log.Fatal(err) + } + + // Use with Eino's ToolsNode + tools := []tool.BaseTool{searchTool} + // ... configure and use with ToolsNode +} +``` + +## Configuration + +The tool can be configured using the `Config` struct: + +```go +type Config struct { + ToolName string // Tool name for LLM interaction (default: "duckduckgo_search") + ToolDesc string // Tool description (default: "search web for information by duckduckgo") + Region ddgsearch.Region // Search region (default: "wt-wt" of no specified region) + Retries int // Number of retries (default: 3) + Timeout int // Max timeout in seconds (default: 10) + MaxResults int // Maximum results per search (default: 10) + Proxy string // Optional proxy URL +} +``` + +## Search + +### Request Schema +```go +type SearchRequest struct { + Query string `json:"query" jsonschema_description:"The query to search the web for"` + Page int `json:"page" jsonschema_description:"The page number to search for, default: 1"` +} +``` + +### Response Schema +```go +type SearchResponse struct { + Results []SearchResult `json:"results" jsonschema_description:"The results of the search"` +} + +type SearchResult struct { + Title string `json:"title" jsonschema_description:"The title of the search result"` + Description string `json:"description" jsonschema_description:"The description of the search result"` + Link string `json:"link" jsonschema_description:"The link of the search result"` +} +``` + +## For More Details + +- [DuckDuckGo Search Library Documentation](ddgsearch/README.md) +- [Eino Documentation](https://github.com/cloudwego/eino) diff --git a/components/tool/duckduckgo/README_zh.md b/components/tool/duckduckgo/README_zh.md new file mode 100644 index 0000000..808c8c6 --- /dev/null +++ b/components/tool/duckduckgo/README_zh.md @@ -0,0 +1,98 @@ +# DuckDuckGo 搜索工具 + +[English](README.md) | 简体中文 + +这是一个为 [Eino](https://github.com/cloudwego/eino) 实现的 DuckDuckGo 搜索工具。该工具实现了 `InvokableTool` 接口，可以与 Eino 的 ChatModel 交互系统和 `ToolsNode` 无缝集成。 + +## 特性 + +- 实现了 `github.com/cloudwego/eino/components/tool.InvokableTool` 接口 +- 易于与 Eino 工具系统集成 +- 可配置的搜索参数 + +## 安装 + +```bash +go get github.com/cloudwego/eino-ext/components/tool/duckduckgo +``` + +## 快速开始 + +```go +package main + +import ( + "context" + "log" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo" + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" + "github.com/cloudwego/eino/components/tool" +) + +func main() { + // 创建工具配置 + cfg := &duckduckgo.Config{ // 下面所有这些参数都是默认值，仅作用法展示 + ToolName: "duckduckgo_search", + ToolDesc: "search web for information by duckduckgo", + Region: ddgsearch.RegionWT, + Retries: 3, + Timeout: 10, + MaxResults: 10, + } + + // 创建搜索工具 + searchTool, err := duckduckgo.NewTool(context.Background(), cfg) + if err != nil { + log.Fatal(err) + } + + // 与 Eino 的 ToolsNode 一起使用 + tools := []tool.BaseTool{searchTool} + // ... 配置并使用 ToolsNode +} +``` + +## 配置 + +工具可以通过 `Config` 结构体进行配置： + +```go +type Config struct { + ToolName string // 用于 LLM 交互的工具名称（默认："duckduckgo_search"） + ToolDesc string // 工具描述（默认："search web for information by duckduckgo"） + Region ddgsearch.Region // 搜索地区（默认："wt-wt"） + Retries int // 重试次数（默认：3） + Timeout int // 最大超时时间（秒）（默认：10） + MaxResults int // 每次搜索的最大结果数（默认：10） + Proxy string // 可选的代理 URL +} +``` + +## Search + +### 请求 Schema +```go +type SearchRequest struct { + Query string `json:"query" jsonschema_description:"要搜索的查询内容"` + Page int `json:"page" jsonschema_description:"要搜索的页码，默认：1"` +} +``` + +### 响应 Schema +```go +type SearchResponse struct { + Results []SearchResult `json:"results" jsonschema_description:"搜索结果列��"` +} + +type SearchResult struct { + Title string `json:"title" jsonschema_description:"搜索结果的标题"` + Description string `json:"description" jsonschema_description:"搜索结果的描述"` + Link string `json:"link" jsonschema_description:"搜索结果的链接"` +} +``` + +## 更多详情 + +- [DuckDuckGo 搜索库文档](ddgsearch/README_zh.md) +- [Eino 文档](https://github.com/cloudwego/eino) \ No newline at end of file diff --git a/components/tool/duckduckgo/ddgsearch/README.md b/components/tool/duckduckgo/ddgsearch/README.md new file mode 100644 index 0000000..cddd72f --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/README.md @@ -0,0 +1,149 @@ +# DDGSearch + +English | [简体中文](README_zh.md) + +A native Go library for DuckDuckGo search functionality. This library provides a simple and efficient way to perform searches using DuckDuckGo's search engine. + +## Why DuckDuckGo? + +DuckDuckGo offers several advantages: +- **No Authentication Required**: Unlike other search engines, DuckDuckGo's API can be used without any API keys or authentication +- Privacy-focused search results +- No rate limiting for reasonable usage +- Support for multiple regions and languages +- Clean and relevant search results + +## Features + +- Clean and idiomatic Go implementation +- Comprehensive error handling +- Configurable search parameters +- In-memory caching with TTL +- Support for: + - Multiple regions (us-en, uk-en, de-de, etc.) + - Safe search levels (strict, moderate, off) + - Time-based filtering (day, week, month, year) + - Result pagination + - Custom HTTP headers + - Proxy configuration + +## Installation + +```bash +go get github.com/cloudwego/eino-ext/components/tool/duckduckgo +``` + +## Quick Start + +```go +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" +) + +func main() { + // Create a new client with configuration + cfg := &ddgsearch.Config{ + Timeout: 30 * time.Second, + MaxRetries: 3, + Cache: true, + } + client, err := ddgsearch.New(cfg) + if err != nil { + log.Fatal(err) + } + + // Configure search parameters + params := &ddgsearch.SearchParams{ + Query: "what is golang", + Region: ddgsearch.RegionUSEN, + SafeSearch: ddgsearch.SafeSearchModerate, + TimeRange: ddgsearch.TimeRangeMonth, + MaxResults: 10, + } + + // Perform search + response, err := client.Search(context.Background(), params) + if err != nil { + log.Fatal(err) + } + + // Print results + for i, result := range response.Results { + fmt.Printf("%d. %s\n URL: %s\n Description: %s\n\n", + i+1, result.Title, result.URL, result.Description) + } +} +``` + +## Advanced Usage + +### Configuration + +```go +// Create client with custom configuration +cfg := &ddgsearch.Config{ + Timeout: 20 * time.Second, + MaxRetries: 3, + Proxy: "http://proxy:8080", + Cache: true, + Headers: map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + }, +} +client, err := ddgsearch.New(cfg) +``` + +### Search Parameters + +```go +params := &ddgsearch.SearchParams{ + Query: "golang tutorial", // Search query + Region: ddgsearch.RegionUS, // Region for results (us-en, uk-en, etc.) + SafeSearch: ddgsearch.SafeSearchModerate, // Safe search level + TimeRange: ddgsearch.TimeRangeWeek, // Time filter + MaxResults: 10, // Maximum results to return +} +``` + +Available regions: +- RegionUS (United States) +- RegionUK (United Kingdom) +- RegionDE (Germany) +- RegionFR (France) +- RegionJP (Japan) +- RegionCN (China) +- RegionRU (Russia) + +Safe search levels: +- SafeSearchStrict +- SafeSearchModerate +- SafeSearchOff + +Time range options: +- TimeRangeDay +- TimeRangeWeek +- TimeRangeMonth +- TimeRangeYear + +### Proxy Support + +```go +// HTTP proxy +cfg := &ddgsearch.Config{ + Proxy: "http://proxy:8080", +} +client, err := ddgsearch.New(cfg) + +// SOCKS5 proxy +cfg := &ddgsearch.Config{ + Proxy: "socks5://proxy:1080", +} +client, err := ddgsearch.New(cfg) +``` diff --git a/components/tool/duckduckgo/ddgsearch/README_zh.md b/components/tool/duckduckgo/ddgsearch/README_zh.md new file mode 100644 index 0000000..92f0127 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/README_zh.md @@ -0,0 +1,149 @@ +# DDGSearch + +[English](README.md) | 简体中文 + +一个用于 DuckDuckGo 搜索功能的原生 Go 库。该库提供了一种简单高效的方式来使用 DuckDuckGo 搜索引擎进行搜索。 + +## 为什么选择 DuckDuckGo？ + +DuckDuckGo 提供了以下优势： +- **无需认证**：与其他搜索引擎不同，DuckDuckGo 的 API 无需任何 API 密钥或认证 +- 注重隐私的搜索结果 +- 合理使用范围内无速率限制 +- 支持多个地区和语言 +- 干净且相关的搜索结果 + +## 特性 + +- 简洁且地道的 Go 实现 +- 全面的错误处理 +- 可配置的搜索参数 +- 带 TTL 的内存缓存 +- 支持： + - 多个地区（us-en、uk-en、de-de 等） + - 安全搜索级别（严格、适中、关闭） + - 基于时间的过滤（天、周、月、年） + - 结果分页 + - 自定义 HTTP 头 + - 代理配置 + +## 安装 + +```bash +go get github.com/cloudwego/eino-ext/components/tool/duckduckgo +``` + +## 快速开始 + +```go +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" +) + +func main() { + // 创建带配置的新客户端 + cfg := &ddgsearch.Config{ + Timeout: 30 * time.Second, + MaxRetries: 3, + Cache: true, + } + client, err := ddgsearch.New(cfg) + if err != nil { + log.Fatal(err) + } + + // 配置搜索参数 + params := &ddgsearch.SearchParams{ + Query: "what is golang", + Region: ddgsearch.RegionUSEN, + SafeSearch: ddgsearch.SafeSearchModerate, + TimeRange: ddgsearch.TimeRangeMonth, + MaxResults: 10, + } + + // 执行搜索 + response, err := client.Search(context.Background(), params) + if err != nil { + log.Fatal(err) + } + + // 打印结果 + for i, result := range response.Results { + fmt.Printf("%d. %s\n URL: %s\n Description: %s\n\n", + i+1, result.Title, result.URL, result.Description) + } +} +``` + +## 高级用法 + +### 配置 + +```go +// 使用自定义配置创建客户端 +cfg := &ddgsearch.Config{ + Timeout: 20 * time.Second, + MaxRetries: 3, + Proxy: "http://proxy:8080", + Cache: true, + Headers: map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + }, +} +client, err := ddgsearch.New(cfg) +``` + +### 搜索参��� + +```go +params := &ddgsearch.SearchParams{ + Query: "golang tutorial", // 搜索查询 + Region: ddgsearch.RegionUS, // 结果地区（us-en、uk-en 等） + SafeSearch: ddgsearch.SafeSearchModerate, // 安全搜索级别 + TimeRange: ddgsearch.TimeRangeWeek, // 时间过滤器 + MaxResults: 10, // 返回的最大结果数 +} +``` + +可用地区： +- RegionUS（美国） +- RegionUK（英国） +- RegionDE（德国） +- RegionFR（法国） +- RegionJP（日本） +- RegionCN（中国） +- RegionRU（俄罗斯） + +安全搜索级别： +- SafeSearchStrict（严格） +- SafeSearchModerate（适中） +- SafeSearchOff（关闭） + +时间范围选项： +- TimeRangeDay（天） +- TimeRangeWeek（周） +- TimeRangeMonth（月） +- TimeRangeYear（年） + +### 代理支持 + +```go +// HTTP 代理 +cfg := &ddgsearch.Config{ + Proxy: "http://proxy:8080", +} +client, err := ddgsearch.New(cfg) + +// SOCKS5 代理 +cfg := &ddgsearch.Config{ + Proxy: "socks5://proxy:1080", +} +client, err := ddgsearch.New(cfg) +``` \ No newline at end of file diff --git a/components/tool/duckduckgo/ddgsearch/cache.go b/components/tool/duckduckgo/ddgsearch/cache.go new file mode 100644 index 0000000..560f631 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/cache.go @@ -0,0 +1,77 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "sync" + "time" +) + +// cache implements a simple in-memory cache with expiration +type cache struct { + mu sync.RWMutex + items map[string]*cacheItem + maxAge time.Duration +} + +type cacheItem struct { + value interface{} + expiration time.Time +} + +func newCache(maxAge time.Duration) *cache { + return &cache{ + items: make(map[string]*cacheItem), + maxAge: maxAge, + } +} + +func (c *cache) get(key string) (interface{}, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + item, exists := c.items[key] + if !exists { + return nil, false + } + + if time.Now().After(item.expiration) { + delete(c.items, key) + return nil, false + } + + return item.value, true +} + +func (c *cache) set(key string, value interface{}) { + c.mu.Lock() + defer c.mu.Unlock() + + c.items[key] = &cacheItem{ + value: value, + expiration: time.Now().Add(c.maxAge), + } +} + +func (c *cache) delete(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.items, key) +} + +func (c *cache) clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.items = make(map[string]*cacheItem) +} diff --git a/components/tool/duckduckgo/ddgsearch/client.go b/components/tool/duckduckgo/ddgsearch/client.go new file mode 100644 index 0000000..1b591c5 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/client.go @@ -0,0 +1,227 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// DDGS represents the DuckDuckGo search client. +// It handles all search-related operations including request configuration, +// caching, and result parsing. +// +// Use New() to create a new instance with proper configuration. +type DDGS struct { + client *http.Client + headers map[string]string + proxy string + timeout time.Duration + cache *cache + config *Config +} + +// Config configures the DDGS client behavior. +// All fields are optional and will use sensible defaults if not provided. +type Config struct { + // Headers specifies custom HTTP headers to be sent with each request. + // Common headers like "User-Agent" can be set here. + // Example: + // Headers: map[string]string{ + // "User-Agent": "MyApp/1.0", + // "Accept-Language": "en-US", + // } + Headers map[string]string + + // Proxy specifies the proxy server URL for all requests. + // Supports HTTP, HTTPS, and SOCKS5 proxies. + // Example values: + // - "http://proxy.example.com:8080" + // - "socks5://localhost:1080" + // - "tb" (special alias for Tor Browser) + Proxy string + + // Timeout specifies the maximum duration for a single request. + // Default is 30 seconds if not specified. + // Example: 5 * time.Second + Timeout time.Duration + + // Cache enables in-memory caching of search results. + // When enabled, identical search requests will return cached results + // for improved performance. Cache entries expire after 5 minutes. + Cache bool + + // MaxRetries specifies the maximum number of retry attempts for failed requests. + // Default is 3. + MaxRetries int +} + +// New creates a new DDGS client with the given configuration +func New(cfg *Config) (*DDGS, error) { + if cfg == nil { + cfg = &Config{ + Headers: make(map[string]string), + Timeout: 30 * time.Second, + MaxRetries: 3, + } + } + + if cfg.Timeout == 0 { + cfg.Timeout = 30 * time.Second + } + + if cfg.MaxRetries == 0 { + cfg.MaxRetries = 3 + } + + d := &DDGS{ + client: &http.Client{Timeout: cfg.Timeout}, + headers: cfg.Headers, + proxy: cfg.Proxy, + timeout: cfg.Timeout, + config: cfg, + } + + // Configure proxy if specified + if cfg.Proxy != "" { + proxyURL, err := url.Parse(cfg.Proxy) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %w", err) + } + + // Validate proxy scheme + switch proxyURL.Scheme { + case "http", "https", "socks5": + d.client.Transport = &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + } + default: + return nil, fmt.Errorf("unsupported proxy scheme: %s", proxyURL.Scheme) + } + } + + if cfg.Cache { + d.cache = newCache(5 * time.Minute) // 5 minutes cache expiration + } + + return d, nil +} + +// sendRequestWithRetry sends the request with retry +func (d *DDGS) sendRequestWithRetry(ctx context.Context, req *http.Request, params *SearchParams) (*SearchResponse, error) { + var resp *http.Response + var err error + var attempt int + + for attempt = 0; attempt <= d.config.MaxRetries; attempt++ { + // Check context cancellation + if err := ctx.Err(); err != nil { + return nil, err + } + + resp, err = d.client.Do(req) + if err != nil { + if attempt == d.config.MaxRetries { + return nil, fmt.Errorf("failed to send request after retries: %w", err) + } + time.Sleep(time.Second) // Simple fixed 1 second delay between retries + continue + } + + // Check for rate limit response + if resp.StatusCode == http.StatusTooManyRequests { + if attempt == d.config.MaxRetries { + return nil, ErrRateLimit + } + time.Sleep(time.Second) + continue + } + + break + } + + defer resp.Body.Close() + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + // Parse search response + response, err := parseSearchResponse(body) + if err != nil { + return nil, fmt.Errorf("failed to parse search results: %w", err) + } + + // Check for no results + if len(response.Results) == 0 { + return nil, ErrNoResults + } + + // Apply max results limit if specified + if params.MaxResults > 0 && len(response.Results) > params.MaxResults { + response.Results = response.Results[:params.MaxResults] + } + + return response, nil +} + +// getVQD retrieves the VQD token required for search requests +func (d *DDGS) getVQD(ctx context.Context, query string) (string, error) { + endpoint := "https://duckduckgo.com" + + // Create request with query parameter + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + // Add query parameter + q := req.URL.Query() + q.Set("q", query) + req.URL.RawQuery = q.Encode() + + // Set headers + for k, v := range d.headers { + req.Header.Set(k, v) + } + + // Set default User-Agent if not provided + if _, ok := req.Header["User-Agent"]; !ok { + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + } + + resp, err := d.client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response: %w", err) + } + + vqd := extractVQDToken(string(body)) + if vqd == "" { + return "", fmt.Errorf("failed to extract VQD token") + } + + return vqd, nil +} diff --git a/components/tool/duckduckgo/ddgsearch/client_test.go b/components/tool/duckduckgo/ddgsearch/client_test.go new file mode 100644 index 0000000..65a39ad --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/client_test.go @@ -0,0 +1,245 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestNew(t *testing.T) { + tests := []struct { + name string + cfg *Config + wantErr bool + }{ + { + name: "nil config", + cfg: nil, + wantErr: false, + }, + { + name: "valid config", + cfg: &Config{ + Headers: map[string]string{"User-Agent": "test"}, + Timeout: 5 * time.Second, + }, + wantErr: false, + }, + { + name: "invalid proxy", + cfg: &Config{ + Proxy: "invalid://proxy", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := New(tt.cfg) + if (err != nil) != tt.wantErr { + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && client == nil { + t.Error("New() returned nil client") + } + }) + } +} + +func TestDDGS_Search(t *testing.T) { + // Create a test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + // Return VQD token + w.Write([]byte(``)) + case "/d.js": + // Return search results + w.Write([]byte(`{ + "results": [ + {"t": "Test Result 1", "u": "http://example.com/1", "a": "Description 1"}, + {"t": "Test Result 2", "u": "http://example.com/2", "a": "Description 2"} + ], + "noResults": false + }`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + // Create client with test server URL + client, err := New(&Config{ + Headers: map[string]string{"User-Agent": "test"}, + Timeout: 5 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + + // Set test endpoints + searchURL = server.URL + "/d.js" + + tests := []struct { + name string + params *SearchParams + wantCount int + wantErr bool + wantErrMsg string + }{ + { + name: "valid search", + params: &SearchParams{ + Query: "test", + MaxResults: 2, + }, + wantCount: 2, + wantErr: false, + }, + { + name: "empty query", + params: &SearchParams{}, + wantCount: 0, + wantErr: true, + }, + { + name: "nil params", + params: nil, + wantCount: 0, + wantErr: true, + }, + { + name: "max results limit", + params: &SearchParams{ + Query: "test", + MaxResults: 1, + }, + wantCount: 1, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results, err := client.Search(context.Background(), tt.params) + if (err != nil) != tt.wantErr { + t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if results == nil { + t.Error("Search() returned nil results when error not expected") + return + } + if len(results.Results) != tt.wantCount { + t.Errorf("Search() got %d results, want %d", len(results.Results), tt.wantCount) + } + } + }) + } +} + +func TestDDGS_SearchCache(t *testing.T) { + requestCount := 0 + + // Create a test server that counts requests + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + switch r.URL.Path { + case "/": + // Return VQD token + w.Write([]byte(``)) + case "/d.js": + // Return search results + w.Write([]byte(`{ + "results": [ + {"t": "Test Result 1", "u": "http://example.com/1", "a": "Description 1"}, + {"t": "Test Result 2", "u": "http://example.com/2", "a": "Description 2"} + ], + "noResults": false + }`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + // Create client with cache enabled + client, err := New(&Config{ + Headers: map[string]string{"User-Agent": "test"}, + Timeout: 5 * time.Second, + Cache: true, + }) + if err != nil { + t.Fatal(err) + } + + // Set test endpoints + searchURL = server.URL + "/d.js" + + // First search request + params := &SearchParams{ + Query: "test query", + MaxResults: 2, + } + response, err := client.Search(context.Background(), params) + if err != nil { + t.Fatalf("First search failed: %v", err) + } + + results1 := response.Results + if len(results1) != 2 { + t.Errorf("Expected 2 results, got %d", len(results1)) + } + initialRequests := requestCount + + // Second search request with same parameters (should use cache) + response2, err := client.Search(context.Background(), params) + if err != nil { + t.Fatalf("Second search failed: %v", err) + } + results2 := response2.Results + if len(results2) != 2 { + t.Errorf("Expected 2 results, got %d", len(results2)) + } + + // Check if request count remained the same (indicating cache hit) + if requestCount != initialRequests { + t.Errorf("Cache not working: expected %d requests, got %d", initialRequests, requestCount) + } + + // Verify results are identical + for i := range results1 { + if results1[i].URL != results2[i].URL { + t.Errorf("Cache returned different results: expected %v, got %v", results1[i], results2[i]) + } + } + + // Different query (should not use cache) + params.Query = "different query" + _, err = client.Search(context.Background(), params) + if err != nil { + t.Fatalf("Third search failed: %v", err) + } + if requestCount <= initialRequests { + t.Error("Cache incorrectly used for different query") + } + +} diff --git a/components/tool/duckduckgo/ddgsearch/doc.go b/components/tool/duckduckgo/ddgsearch/doc.go new file mode 100644 index 0000000..b024902 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/doc.go @@ -0,0 +1,50 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ddgsearch provides a Go client for DuckDuckGo search API. +// +// Example usage: +// +// cfg := &ddgsearch.Config{ +// Headers: map[string]string{ +// "User-Agent": "MyApp/1.0", +// }, +// Timeout: 10 * time.Second, +// Cache: true, +// MaxRetries: 3, +// } +// +// client, err := ddgsearch.New(cfg) +// if err != nil { +// log.Fatal(err) +// } +// +// params := &ddgsearch.SearchParams{ +// Query: "golang programming", +// Region: ddgsearch.RegionUS, +// SafeSearch: ddgsearch.SafeSearchModerate, +// TimeRange: ddgsearch.TimeRangeYear, +// MaxResults: 10, +// } +// +// results, err := client.Search(context.Background(), params) +// if err != nil { +// log.Fatal(err) +// } +// +// for _, result := range results { +// fmt.Printf("Title: %s\nURL: %s\nDescription: %s\n\n", +// result.Title, result.URL, result.Description) +// } +package ddgsearch diff --git a/components/tool/duckduckgo/ddgsearch/errors.go b/components/tool/duckduckgo/ddgsearch/errors.go new file mode 100644 index 0000000..5abfae5 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/errors.go @@ -0,0 +1,129 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "errors" + "fmt" +) + +// SearchError represents an error that occurred during a search operation. +// It wraps the original error and provides additional context. +type SearchError struct { + Message string // Human readable error message + Err error // Original error +} + +func (e *SearchError) Error() string { + if e.Err != nil { + return fmt.Sprintf("%s: %v", e.Message, e.Err) + } + return e.Message +} + +func (e *SearchError) Unwrap() error { + return e.Err +} + +// NewSearchError creates a new SearchError with the given message and error. +func NewSearchError(message string, err error) error { + return &SearchError{Message: message, Err: err} +} + +// Common errors returned by the library +var ( + // ErrRateLimit is returned when DuckDuckGo rate limits the request. + // This typically happens when too many requests are made in a short period. + ErrRateLimit = &SearchError{Message: "rate limit exceeded"} + + // ErrTimeout is returned when a request to DuckDuckGo times out. + // This can happen due to network issues or server-side delays. + ErrTimeout = &SearchError{Message: "request timeout"} + + // ErrNoResults is returned when the search yields no results. + // This can happen with very specific queries or when DuckDuckGo has no matching content. + ErrNoResults = &SearchError{Message: "no results found"} + + // ErrInvalidResponse is returned when the response from DuckDuckGo cannot be parsed. + // This can happen if the API changes or returns an unexpected format. + ErrInvalidResponse = &SearchError{Message: "invalid response from DuckDuckGo"} + + // ErrInvalidRegion is returned when an unsupported region code is provided. + // Use one of the predefined Region constants. + ErrInvalidRegion = &SearchError{Message: "invalid region code"} + + // ErrInvalidSafeSearch is returned when an unsupported safe search level is provided. + // Use one of the predefined SafeSearch constants. + ErrInvalidSafeSearch = &SearchError{Message: "invalid safe search level"} + + // ErrInvalidTimeRange is returned when an unsupported time range is provided. + // Use one of the predefined TimeRange constants. + ErrInvalidTimeRange = &SearchError{Message: "invalid time range"} +) + +// IsRateLimitErr checks if the error is a rate limit error. +// +// Example: +// +// if IsRateLimitErr(err) { +// time.Sleep(time.Second * 5) +// // retry request +// } +func IsRateLimitErr(err error) bool { + var searchErr *SearchError + if err == nil { + return false + } + if ok := errors.As(err, &searchErr); ok { + return searchErr == ErrRateLimit + } + return false +} + +// IsTimeoutErr checks if the error is a timeout error. +// +// Example: +// +// if IsTimeoutErr(err) { +// // increase timeout and retry +// client.SetTimeout(30 * time.Second) +// } +func IsTimeoutErr(err error) bool { + var searchErr *SearchError + if err == nil { + return false + } + if ok := errors.As(err, &searchErr); ok { + return searchErr == ErrTimeout + } + return false +} + +// IsNoResultsErr checks if the error indicates no results were found. +// +// Example: +// +// if IsNoResultsErr(err) { +// fmt.Println("No results found, try different keywords") +// } +func IsNoResultsErr(err error) bool { + var searchErr *SearchError + if err == nil { + return false + } + if ok := errors.As(err, &searchErr); ok { + return searchErr == ErrNoResults + } + return false +} diff --git a/components/tool/duckduckgo/ddgsearch/search_news.go b/components/tool/duckduckgo/ddgsearch/search_news.go new file mode 100644 index 0000000..7d838a6 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/search_news.go @@ -0,0 +1,189 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// News performs a DuckDuckGo news search with the given parameters. +func (d *DDGS) News(ctx context.Context, params *NewsParams) (*NewsResponse, error) { + if params.Query == "" { + return nil, fmt.Errorf("query is required") + } + + // Get vqd token + vqd, err := d.getVQD(ctx, params.Query) + if err != nil { + return nil, fmt.Errorf("failed to get vqd: %w", err) + } + + // Prepare safe search parameter + safeSearchMap := map[SafeSearch]string{ + SafeSearchStrict: "1", + SafeSearchModerate: "-1", + SafeSearchOff: "-2", + } + + // Build query parameters + queryParams := url.Values{ + "l": {string(params.Region)}, + "o": {"json"}, + "noamp": {"1"}, + "q": {params.Query}, + "vqd": {vqd}, + "p": {safeSearchMap[params.SafeSearch]}, + "t": {"n"}, // Ensure we're requesting news + } + + if params.TimeRange != "" { + queryParams.Set("df", string(params.TimeRange)) + } + + maxResults := params.MaxResults + if maxResults <= 0 || maxResults > 120 { + maxResults = 30 // Default to first page + } + + var allResults []NewsResult + seenURLs := make(map[string]bool) + + // Fetch results in batches of 30 + for offset := 0; offset < maxResults; offset += 30 { + queryParams.Set("s", fmt.Sprintf("%d", offset)) + + // Create request + req, err := http.NewRequestWithContext(ctx, http.MethodGet, newsURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Set query parameters + req.URL.RawQuery = queryParams.Encode() + + // Set headers + for k, v := range d.headers { + req.Header.Set(k, v) + } + + // Ensure we have a User-Agent header + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + } + + // Set additional required headers + req.Header.Set("Accept", "application/json") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Referer", "https://duckduckgo.com/") + req.Header.Set("Authority", "duckduckgo.com") + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Site", "same-origin") + + // Send request with retry + var resp *http.Response + var lastErr error + for retries := 0; retries < 3; retries++ { + if retries > 0 { + time.Sleep(time.Second * time.Duration(retries)) + } + + resp, lastErr = d.client.Do(req) + if lastErr == nil && resp.StatusCode == http.StatusOK { + break + } + if resp != nil { + resp.Body.Close() + } + } + if lastErr != nil { + return nil, fmt.Errorf("failed to send request after retries: %w", lastErr) + } + if resp == nil { + return nil, fmt.Errorf("no response received after retries") + } + defer resp.Body.Close() + + // Check response status + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body)) + } + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + // Debug the response + if len(body) == 0 { + return nil, fmt.Errorf("empty response body") + } + + // Try to parse the response + var raw rawNewsResponse + if err := json.Unmarshal(body, &raw); err != nil { + // Try to get more information about the response + bodyStr := truncateString(string(body), 200) + return nil, fmt.Errorf("failed to parse news response (status: %d, body: %s): %w", + resp.StatusCode, bodyStr, err) + } + + // Process results + for _, r := range raw.Results { + if !seenURLs[r.URL] { + seenURLs[r.URL] = true + + // Convert Unix timestamp to ISO8601 + date := time.Unix(r.Date, 0).UTC().Format(time.RFC3339) + + result := NewsResult{ + Date: date, + Title: r.Title, + Body: r.Excerpt, + URL: normalizeURL(r.URL), + Image: normalizeURL(r.Image), + Source: r.Source, + } + allResults = append(allResults, result) + } + } + + // If we got less than 30 results, there are no more to fetch + if len(raw.Results) < 30 { + break + } + } + + // If we got no results at all, return an error with more context + if len(allResults) == 0 { + return nil, fmt.Errorf("no news results found for query: %s", params.Query) + } + + // Trim results to max requested + if len(allResults) > maxResults { + allResults = allResults[:maxResults] + } + + return &NewsResponse{ + Results: allResults, + }, nil +} diff --git a/components/tool/duckduckgo/ddgsearch/search_news_test.go b/components/tool/duckduckgo/ddgsearch/search_news_test.go new file mode 100644 index 0000000..1bc094e --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/search_news_test.go @@ -0,0 +1,200 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "context" + "fmt" + "testing" + "time" +) + +func TestDDGS_News(t *testing.T) { + // Create a client with custom configuration for testing + cfg := &Config{ + Headers: map[string]string{ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + }, + Timeout: 30 * time.Second, + MaxRetries: 3, + } + + client, err := New(cfg) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + tests := []struct { + name string + params *NewsParams + want func(*NewsResponse) error + wantErr bool + }{ + { + name: "basic_search", + params: &NewsParams{ + Query: "technology news", + Region: RegionUS, + SafeSearch: SafeSearchModerate, + MaxResults: 5, + }, + want: func(resp *NewsResponse) error { + if len(resp.Results) == 0 { + return fmt.Errorf("expected results, got none") + } + for i, r := range resp.Results { + if r.Title == "" { + return fmt.Errorf("result %d has empty title", i) + } + if r.URL == "" { + return fmt.Errorf("result %d has empty URL", i) + } + if r.Source == "" { + return fmt.Errorf("result %d has empty source", i) + } + if r.Date == "" { + return fmt.Errorf("result %d has empty date", i) + } + } + return nil + }, + wantErr: false, + }, + { + name: "empty_query", + params: &NewsParams{ + Query: "", + }, + want: nil, + wantErr: true, + }, + { + name: "search_with_max_results", + params: &NewsParams{ + Query: "technology", + Region: RegionUS, + SafeSearch: SafeSearchModerate, + MaxResults: 2, + }, + want: func(resp *NewsResponse) error { + if len(resp.Results) > 2 { + return fmt.Errorf("expected at most 2 results, got %d", len(resp.Results)) + } + // Print sample results for debugging + t.Logf("Sample results for %q:", "search with max results") + for i, r := range resp.Results { + t.Logf(" %d. %s (%s) - %s", i+1, r.Title, r.Source, r.Date) + } + return nil + }, + wantErr: false, + }, + { + name: "search_with_time_range", + params: &NewsParams{ + Query: "artificial intelligence", + Region: RegionUS, + SafeSearch: SafeSearchModerate, + TimeRange: TimeRangeDay, + MaxResults: 3, + }, + want: func(resp *NewsResponse) error { + if len(resp.Results) == 0 { + return fmt.Errorf("expected results, got none") + } + // Verify date is within the last day + now := time.Now() + for i, r := range resp.Results { + date, err := time.Parse(time.RFC3339, r.Date) + if err != nil { + return fmt.Errorf("result %d has invalid date format: %s", i, r.Date) + } + if now.Sub(date) > 24*time.Hour { + return fmt.Errorf("result %d date %s is older than 24 hours", i, r.Date) + } + } + return nil + }, + wantErr: false, + }, + { + name: "search_with_region", + params: &NewsParams{ + Query: "news", + Region: RegionJP, + SafeSearch: SafeSearchModerate, + MaxResults: 3, + }, + want: func(resp *NewsResponse) error { + if len(resp.Results) == 0 { + return fmt.Errorf("expected results, got none") + } + return nil + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resp *NewsResponse + var err error + + // Retry logic for flaky tests + for retries := 0; retries < 3; retries++ { + if retries > 0 { + t.Logf("Retry %d for %s", retries, tt.name) + time.Sleep(time.Second * time.Duration(retries)) + } + + resp, err = client.News(ctx, tt.params) + + if (err != nil) == tt.wantErr { + break // Test passed + } + + if err != nil { + t.Logf("Retry %d failed: %v", retries+1, err) + continue + } + + if tt.want != nil { + if err := tt.want(resp); err == nil { + break // Test passed + } else { + t.Logf("Retry %d failed: %v", retries+1, err) + } + } + } + + if (err != nil) != tt.wantErr { + t.Errorf("DDGS.News() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr && tt.want != nil { + if err := tt.want(resp); err != nil { + t.Errorf("DDGS.News() validation failed: %v", err) + } + } + }) + + // Add delay between tests to avoid rate limiting + time.Sleep(time.Second) + } +} diff --git a/components/tool/duckduckgo/ddgsearch/search_text.go b/components/tool/duckduckgo/ddgsearch/search_text.go new file mode 100644 index 0000000..877a4b1 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/search_text.go @@ -0,0 +1,187 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" +) + +// Search performs a search with the given parameters +func (d *DDGS) Search(ctx context.Context, params *SearchParams) (*SearchResponse, error) { + if params == nil { + return nil, fmt.Errorf("search params cannot be nil") + } + + if params.Query == "" { + return nil, fmt.Errorf("search query cannot be empty") + } + + // Generate cache key if caching is enabled + if d.cache != nil { + params.cacheKey = params.getCacheKey() + + // Try to get from cache + if cached, ok := d.cache.get(params.cacheKey); ok { + if response, ok := cached.(*SearchResponse); ok { + return response, nil + } + } + } + + // Get VQD token + vqd, err := d.getVQD(ctx, params.Query) + if err != nil { + return nil, fmt.Errorf("failed to get vqd token: %w", err) + } + + // Build search URL using SearchParams method + searchURL := params.buildSearchURL(vqd) + + // Create request + req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Send request with retry + response, err := d.sendRequestWithRetry(ctx, req, params) + if err != nil { + return nil, err + } + + // Cache the response if caching is enabled + if d.cache != nil && params.cacheKey != "" { + d.cache.set(params.cacheKey, response) + } + + // max results + if params.MaxResults > 0 && len(response.Results) > params.MaxResults { + response.Results = response.Results[:params.MaxResults] + } + + return response, nil +} + +// validate checks if the search parameters are valid +func (p *SearchParams) validate() error { + if p.Query == "" { + return NewSearchError("search query cannot be empty", nil) + } + if p.Page < 0 { + return NewSearchError("page number cannot be negative", nil) + } + if p.MaxResults < 0 { + return NewSearchError("max results cannot be negative", nil) + } + return nil +} + +// buildSearchURL constructs the search URL with all necessary parameters +func (p *SearchParams) buildSearchURL(vqd string) string { + // Use test endpoint if available + endpoint := searchURL + + // Initialize URL parameters + params := url.Values{} + + // Main search parameters + params.Set("q", p.Query) // The search query text + params.Set("vqd", vqd) // Verification query ID, required by DuckDuckGo + + // Regional and language settings + params.Set("kl", string(p.Region)) // Knowledge location - affects result relevance by region + params.Set("l", string(p.Region)) + + // Search behavior flags + params.Set("ss", "1") // Show snippets in results + params.Set("sp", "1") // Show preference cookies + params.Set("sc", "1") // Show category headers + params.Set("o", "json") // Output format (JSON) + + // Optional parameters + if p.SafeSearch != "" { + params.Set("p", string(p.SafeSearch)) // Safe search level (strict/moderate/off) + } + if p.TimeRange != "" { + params.Set("df", string(p.TimeRange)) // Date filter for results (d/w/m/y) + } + if p.Page > 1 { + // Skip page results + // Example: page 2 of max 10 results starts at result 10, page 3 at result 20, etc. + pageSize := p.MaxResults + if pageSize == 0 { + pageSize = 10 // default page size + } + params.Set("s", strconv.Itoa((p.Page-1)*pageSize)) + } + + // Construct final URL + return endpoint + "?" + params.Encode() +} + +// getCacheKey generates a unique cache key for the search parameters +func (p *SearchParams) getCacheKey() string { + // Use url.Values to consistently encode parameters + v := url.Values{} + v.Set("q", p.Query) + v.Set("r", string(p.Region)) + v.Set("s", string(p.SafeSearch)) + v.Set("t", string(p.TimeRange)) + v.Set("p", strconv.Itoa(p.Page)) + + return v.Encode() +} + +// parseSearchResponse parses the search response from DuckDuckGo +func parseSearchResponse(body []byte) (*SearchResponse, error) { + var response struct { + Results []struct { + Title string `json:"t"` + URL string `json:"u"` + Description string `json:"a"` + } `json:"results"` + NoResults bool `json:"noResults"` + } + + err := json.Unmarshal(body, &response) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + if response.NoResults { + return &SearchResponse{}, nil + } + + results := make([]SearchResult, 0, len(response.Results)) + for _, r := range response.Results { + if r.Description == "" && r.URL == "" && r.Title == "" { + continue + } + + results = append(results, SearchResult{ + Title: r.Title, + URL: r.URL, + Description: r.Description, + }) + } + + return &SearchResponse{ + Results: results, + }, nil +} diff --git a/components/tool/duckduckgo/ddgsearch/types.go b/components/tool/duckduckgo/ddgsearch/types.go new file mode 100644 index 0000000..510ce29 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/types.go @@ -0,0 +1,226 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +// Common constants +var ( + baseURL = "https://duckduckgo.com" + searchURL = "https://links.duckduckgo.com/d.js" + newsURL = "https://duckduckgo.com/news.js" +) + +// Region represents a geographical region for search results. +// Different regions may return different search results based on local relevance. +// others can be found at: https://duckduckgo.com/duckduckgo-help-pages/settings/params/ +type Region string + +// Available regions for DuckDuckGo search +const ( + // RegionWT represents World region (No specific region, default) + RegionWT Region = "wt-wt" + // RegionUS represents United States region + RegionUS Region = "us-en" + // RegionUK represents United Kingdom region + RegionUK Region = "uk-en" + // RegionDE represents Germany region + RegionDE Region = "de-de" + // RegionFR represents France region + RegionFR Region = "fr-fr" + // RegionJP represents Japan region + RegionJP Region = "jp-jp" + // RegionCN represents China region + RegionCN Region = "cn-zh" + // RegionRU represents Russia region + RegionRU Region = "ru-ru" +) + +// SafeSearch represents the safe search level for filtering explicit content. +type SafeSearch string + +const ( + // SafeSearchStrict enables strict filtering of explicit content + SafeSearchStrict SafeSearch = "strict" + // SafeSearchModerate enables moderate filtering of explicit content + SafeSearchModerate SafeSearch = "moderate" + // SafeSearchOff disables filtering of explicit content + SafeSearchOff SafeSearch = "off" +) + +// TimeRange represents the time range for search results. +type TimeRange string + +const ( + // TimeRangeDay limits results to the past day + TimeRangeDay TimeRange = "d" + // TimeRangeWeek limits results to the past week + TimeRangeWeek TimeRange = "w" + // TimeRangeMonth limits results to the past month + TimeRangeMonth TimeRange = "m" + // TimeRangeYear limits results to the past year + TimeRangeYear TimeRange = "y" + // TimeRangeAll includes results from all time periods + TimeRangeAll TimeRange = "" +) + +// NewsParams configures the news search behavior. +type NewsParams struct { + // Query is the search term or phrase + Query string `json:"query"` + + // Region specifies the geographical region for results + // Use one of the Region constants (e.g., RegionUS, RegionUK) + Region Region `json:"region"` + + // SafeSearch controls filtering of explicit content + // Use one of the SafeSearch constants + SafeSearch SafeSearch `json:"safe_search"` + + // TimeRange limits results to a specific time period + // Use one of the TimeRange constants + TimeRange TimeRange `json:"time_range"` + + // MaxResults limits the number of results returned. + // Set to 0 for no limit. Note that: + // 1. DuckDuckGo API typically returns 10 results per page + // 2. This parameter only truncates results when the API returns more results than MaxResults + // 3. To get more results, use NextPage() to paginate through results + MaxResults int `json:"max_results"` +} + +// SearchParams configures the search behavior. +// Example usage: +// +// params := &SearchParams{ +// Query: "golang tutorials", // Required +// Region: RegionCN, // Optional, defaults to RegionCN +// SafeSearch: SafeSearchModerate, // Optional, defaults to Moderate +// TimeRange: TimeRangeMonth, // Optional, defaults to All +// Page: 1, // Optional, defaults to 1 +// MaxResults: 10, // Optional, defaults to all results +// } +// +// see more at: https://duckduckgo.com/duckduckgo-help-pages/settings/params/ +type SearchParams struct { + // Query is the search term or phrase + Query string `json:"query"` + + // Region specifies the geographical region for results + // Use one of the Region constants (e.g., RegionUS, RegionUK) + Region Region `json:"region"` + + // SafeSearch controls filtering of explicit content + // Use one of the SafeSearch constants + SafeSearch SafeSearch `json:"safe_search"` + + // TimeRange limits results to a specific time period + // Use one of the TimeRange constants + TimeRange TimeRange `json:"time_range"` + + // Page specifies which page of results to return + // Starts from 1 + Page int `json:"page"` + + // MaxResults limits the number of results returned. + // Set to 0 for no limit. Note that: + // 1. DuckDuckGo API typically returns 10 results per page + // 2. This parameter only truncates results when the API returns more results than MaxResults + // 3. To get more results, use NextPage() to paginate through results + MaxResults int `json:"max_results"` + + // cacheKey is used internally for caching search results + cacheKey string `json:"-"` +} + +// NextPage returns a new SearchParams with the page number incremented. +// This is useful for paginating through search results. +// +// Example usage: +// +// params := &SearchParams{ +// Query: "golang", +// MaxResults: 10, +// } +// +// // Get first page results +// results1, err := client.Search(ctx, params) +// if err != nil { +// return err +// } +// +// // Get next page results +// nextParams := params.NextPage() +// results2, err := client.Search(ctx, nextParams) +func (p *SearchParams) NextPage() *SearchParams { + return &SearchParams{ + Query: p.Query, + Region: p.Region, + SafeSearch: p.SafeSearch, + TimeRange: p.TimeRange, + MaxResults: p.MaxResults, + Page: p.Page + 1, + } +} + +// SearchResult represents a single search result. +// Contains the title, URL, and description of the result. +type SearchResult struct { + // Title is the title of the search result + Title string `json:"t"` + + // URL is the web address of the result + URL string `json:"u"` + + // Description is a brief summary of the result content + Description string `json:"a"` +} + +// SearchResponse represents the complete response from a search request. +type SearchResponse struct { + // Results contains the list of search results + Results []SearchResult `json:"results"` + + // NoResults indicates whether the search returned any results + NoResults bool `json:"noResults"` +} + +// NewsResult represents a single news search result from DuckDuckGo. +type NewsResult struct { + Date string `json:"date"` // ISO8601 formatted date + Title string `json:"title"` // News article title + Body string `json:"body"` // News article excerpt/summary + URL string `json:"url"` // Article URL + Image string `json:"image"` // URL of the article's image + Source string `json:"source"` // News source name +} + +// NewsResponse represents the complete response from a news search request. +type NewsResponse struct { + Results []NewsResult `json:"results"` // List of news results +} + +// rawNewsResponse represents the raw response from DuckDuckGo news API. +type rawNewsResponse struct { + Results []struct { + Date int64 `json:"date"` // Unix timestamp + Title string `json:"title"` // Article title + Excerpt string `json:"excerpt"` // Article excerpt + URL string `json:"url"` // Article URL + Image string `json:"image"` // Image URL + Source string `json:"source"` // Source name + } `json:"results"` + Query string `json:"query"` // The search query + QueryEncoded string `json:"queryEncoded"` // URL encoded query + ResponseType string `json:"response_type"` // Type of response + QueryCategory string `json:"query_category"` // Category of query +} diff --git a/components/tool/duckduckgo/ddgsearch/utils.go b/components/tool/duckduckgo/ddgsearch/utils.go new file mode 100644 index 0000000..3c50d4e --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/utils.go @@ -0,0 +1,96 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "fmt" + "net/url" + "regexp" + "strings" +) + +// extractVQD extracts the VQD token from the DuckDuckGo response +func extractVQD(body []byte, query string) (string, error) { + content := string(body) + + // Try to find vqd in JavaScript code + re := regexp.MustCompile(`vqd=["']([^"']+)["']`) + matches := re.FindStringSubmatch(content) + if len(matches) > 1 { + return matches[1], nil + } + + // Try to find vqd in meta tags + re = regexp.MustCompile(`]+content=["']([^"']+)["'][^>]+name=["']vqd["']`) + matches = re.FindStringSubmatch(content) + if len(matches) > 1 { + return matches[1], nil + } + + // Try to find vqd in any context + re = regexp.MustCompile(`vqd=([^&"']+)`) + matches = re.FindStringSubmatch(content) + if len(matches) > 1 { + return matches[1], nil + } + + return "", fmt.Errorf("could not extract vqd for keywords: %s", query) +} + +// extractVQDToken extracts the VQD token from the HTML response +func extractVQDToken(html string) string { + re := regexp.MustCompile(`vqd=["']([^"']+)["']`) + matches := re.FindStringSubmatch(html) + if len(matches) > 1 { + return matches[1] + } + return "" +} + +// normalizeURL removes AMP and tracking parameters from URLs. +func normalizeURL(u string) string { + if u == "" { + return "" + } + + // Remove AMP + u = strings.ReplaceAll(u, "/amp/", "/") + u = strings.ReplaceAll(u, "?amp=1", "") + u = strings.ReplaceAll(u, "&=1", "") + + // Parse URL + parsed, err := url.Parse(u) + if err != nil { + return u + } + + // Remove tracking parameters + q := parsed.Query() + for k := range q { + if strings.Contains(strings.ToLower(k), "utm_") { + q.Del(k) + } + } + parsed.RawQuery = q.Encode() + + return parsed.String() +} + +// truncateString truncates a string to a maximum length. +func truncateString(s string, maxLength int) string { + if len(s) <= maxLength { + return s + } + return s[:maxLength] + "..." +} diff --git a/components/tool/duckduckgo/ddgsearch/utils_test.go b/components/tool/duckduckgo/ddgsearch/utils_test.go new file mode 100644 index 0000000..24d19d2 --- /dev/null +++ b/components/tool/duckduckgo/ddgsearch/utils_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ddgsearch + +import ( + "testing" +) + +func TestExtractVQD(t *testing.T) { + tests := []struct { + name string + html []byte + keywords string + want string + wantErr bool + }{ + { + name: "valid vqd double quotes", + html: []byte(``), + keywords: "test", + want: "123456", + wantErr: false, + }, + { + name: "valid vqd single quotes", + html: []byte(``), + keywords: "test", + want: "789012", + wantErr: false, + }, + { + name: "valid vqd with ampersand", + html: []byte(``), + keywords: "test", + want: "345678", + wantErr: false, + }, + { + name: "no vqd", + html: []byte(``), + keywords: "test", + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractVQD(tt.html, tt.keywords) + if (err != nil) != tt.wantErr { + t.Errorf("extractVQD() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("extractVQD() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/components/tool/duckduckgo/example/main.go b/components/tool/duckduckgo/example/main.go new file mode 100644 index 0000000..6813f1f --- /dev/null +++ b/components/tool/duckduckgo/example/main.go @@ -0,0 +1,78 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo" + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" +) + +func main() { + ctx := context.Background() + + // Create configuration + config := &duckduckgo.Config{ + MaxResults: 3, // Limit to return 3 results + Region: ddgsearch.RegionCN, + DDGConfig: &ddgsearch.Config{ + Timeout: 10, + Cache: true, + MaxRetries: 5, + }, + } + + // Create search client + tool, err := duckduckgo.NewTool(ctx, config) + if err != nil { + log.Fatal("Failed to create tool:", err) + } + + // Create search request + searchReq := &duckduckgo.SearchRequest{ + Query: "Golang programming development", + Page: 1, + } + + jsonReq, err := json.Marshal(searchReq) + if err != nil { + log.Fatal("Failed to marshal search request:", err) + } + + // Execute search + resp, err := tool.InvokableRun(ctx, string(jsonReq)) + if err != nil { + log.Fatal("Search failed:", err) + } + + var searchResp duckduckgo.SearchResponse + if err := json.Unmarshal([]byte(resp), &searchResp); err != nil { + log.Fatal("Failed to unmarshal search response:", err) + } + + // Print results + fmt.Println("Search Results:") + fmt.Println("==============") + for i, result := range searchResp.Results { + fmt.Printf("\n%d. Title: %s\n", i+1, result.Title) + fmt.Printf(" Link: %s\n", result.Link) + fmt.Printf(" Description: %s\n", result.Description) + } + fmt.Println("") + fmt.Println("==============") +} diff --git a/components/tool/duckduckgo/go.mod b/components/tool/duckduckgo/go.mod new file mode 100644 index 0000000..520850b --- /dev/null +++ b/components/tool/duckduckgo/go.mod @@ -0,0 +1,48 @@ +module github.com/cloudwego/eino-ext/components/tool/duckduckgo + +go 1.21 + +require ( + github.com/bytedance/mockey v1.2.13 + github.com/cloudwego/eino v0.3.0 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/bytedance/sonic v1.12.2 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/getkin/kin-openapi v0.118.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/invopop/yaml v0.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/perimeterx/marshmallow v1.1.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/smartystreets/goconvey v1.8.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/sys v0.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/components/tool/duckduckgo/go.sum b/components/tool/duckduckgo/go.sum new file mode 100644 index 0000000..d9ee49b --- /dev/null +++ b/components/tool/duckduckgo/go.sum @@ -0,0 +1,158 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/mockey v1.2.13 h1:jokWZAm/pUEbD939Rhznz615MKUCZNuvCFQlJ2+ntoo= +github.com/bytedance/mockey v1.2.13/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= +github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/eino v0.3.0 h1:Xp/zqvyyskRn0obOUvH0Rj/INZwq68z9vvTjXOsNhLw= +github.com/cloudwego/eino v0.3.0/go.mod h1:+kmJimGEcKuSI6OKhet7kBedkm1WUZS3H1QRazxgWUo= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM= +github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= +github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= +github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/components/tool/duckduckgo/search.go b/components/tool/duckduckgo/search.go new file mode 100644 index 0000000..1bc21da --- /dev/null +++ b/components/tool/duckduckgo/search.go @@ -0,0 +1,164 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package duckduckgo + +import ( + "context" + "fmt" + "time" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" + + "github.com/cloudwego/eino/components/tool" + "github.com/cloudwego/eino/components/tool/utils" +) + +type Config struct { + ToolName string `json:"tool_name"` // default: duckduckgo_search + ToolDesc string `json:"tool_desc"` // default: "search web for information by duckduckgo" + + Region ddgsearch.Region `json:"region"` // default: "wt-wt" + MaxResults int `json:"max_results"` // default: 10 + SafeSearch ddgsearch.SafeSearch `json:"safe_search"` // default: ddgsearch.SafeSearchModerate + TimeRange ddgsearch.TimeRange `json:"time_range"` // default: ddgsearch.TimeRangeAll + + DDGConfig *ddgsearch.Config `json:"ddg_config"` +} + +func NewTool(ctx context.Context, config *Config) (tool.InvokableTool, error) { + ddgs, err := newDDGS(ctx, config) + if err != nil { + return nil, fmt.Errorf("failed to create ddg search tool: %w", err) + } + + searchTool, err := utils.InferTool(config.ToolName, config.ToolDesc, ddgs.Search) + if err != nil { + return nil, fmt.Errorf("failed to infer tool: %w", err) + } + + return searchTool, nil +} + +// validate validates the configuration and sets default values if not provided. +func (conf *Config) validate() error { + if conf == nil { + return fmt.Errorf("config is nil") + } + + if conf.ToolName == "" { + conf.ToolName = "duckduckgo_search" + } + + if conf.ToolDesc == "" { + conf.ToolDesc = "search web for information by duckduckgo" + } + + if conf.Region == "" { + conf.Region = ddgsearch.RegionWT + } + + if conf.MaxResults <= 0 { + conf.MaxResults = 10 + } + + if conf.SafeSearch == "" { + conf.SafeSearch = ddgsearch.SafeSearchOff + } + + if conf.TimeRange == "" { + conf.TimeRange = ddgsearch.TimeRangeAll + } + + if conf.DDGConfig == nil { + conf.DDGConfig = &ddgsearch.Config{} + } + + if conf.DDGConfig.Timeout == 0 { + conf.DDGConfig.Timeout = 30 * time.Second + } + + if conf.DDGConfig.MaxRetries == 0 { + conf.DDGConfig.MaxRetries = 3 + } + + return nil +} + +func newDDGS(_ context.Context, config *Config) (*ddgs, error) { + if config == nil { + config = &Config{} + } + + if err := config.validate(); err != nil { + return nil, err + } + + ddg, err := ddgsearch.New(config.DDGConfig) + if err != nil { + return nil, err + } + + return &ddgs{ + config: config, + ddg: ddg, + }, nil +} + +type ddgs struct { + config *Config + ddg *ddgsearch.DDGS +} + +type SearchRequest struct { + Query string `json:"query" jsonschema_description:"The query to search the web for"` + Page int `json:"page" jsonschema_description:"The page number to search for, default: 1"` +} + +type SearchResult struct { + Title string `json:"title" jsonschema_description:"The title of the search result"` + Description string `json:"description" jsonschema_description:"The description of the search result"` + Link string `json:"link" jsonschema_description:"The link of the search result"` +} + +type SearchResponse struct { + Results []*SearchResult `json:"results" jsonschema_description:"The results of the search"` +} + +func (d *ddgs) Search(ctx context.Context, request *SearchRequest) (*SearchResponse, error) { + results, err := d.ddg.Search(ctx, &ddgsearch.SearchParams{ + Query: request.Query, + Region: ddgsearch.Region(d.config.Region), + MaxResults: d.config.MaxResults, + Page: request.Page, + SafeSearch: d.config.SafeSearch, + TimeRange: d.config.TimeRange, + }) + if err != nil { + return nil, err + } + + searchResponse := &SearchResponse{ + Results: make([]*SearchResult, len(results.Results)), + } + + for i, result := range results.Results { + searchResponse.Results[i] = &SearchResult{ + Title: result.Title, + Description: result.Description, + Link: result.URL, + } + } + + return searchResponse, nil +} diff --git a/components/tool/duckduckgo/search_test.go b/components/tool/duckduckgo/search_test.go new file mode 100644 index 0000000..9ac91c0 --- /dev/null +++ b/components/tool/duckduckgo/search_test.go @@ -0,0 +1,266 @@ +// Copyright 2024 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package duckduckgo + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/bytedance/mockey" + "github.com/stretchr/testify/assert" + + "github.com/cloudwego/eino-ext/components/tool/duckduckgo/ddgsearch" +) + +func MockDDGS() *mockey.Mocker { + return mockey.Mock((*ddgsearch.DDGS).Search).To(func(ctx context.Context, request *ddgsearch.SearchParams) (*ddgsearch.SearchResponse, error) { + if request == nil { + return nil, fmt.Errorf("request is nil") + } + if request.Query == "" { + return nil, fmt.Errorf("query is empty") + } + fmt.Println("mocked ddgs.Search", request) + return &ddgsearch.SearchResponse{Results: []ddgsearch.SearchResult{ + { + Title: "test title", + Description: "test description", + URL: "test link", + }, + { + Title: "test title 2", + Description: "test description 2", + URL: "test link 2", + }, + }}, nil + }).Build() +} + +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + config *Config + wantErr bool + }{ + { + name: "nil config", + config: nil, + wantErr: true, + }, + { + name: "empty config", + config: &Config{}, + wantErr: false, + }, + { + name: "valid config", + config: &Config{ + ToolName: "custom_ddg", + ToolDesc: "custom description", + Region: "us-en", + MaxResults: 20, + DDGConfig: &ddgsearch.Config{ + Timeout: 20, + Proxy: "http://proxy.example.com", + Cache: true, + MaxRetries: 5, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.validate() + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + if tt.config.ToolName == "" { + assert.Equal(t, "duckduckgo_search", tt.config.ToolName) + } + if tt.config.ToolDesc == "" { + assert.Equal(t, "search web for information by duckduckgo", tt.config.ToolDesc) + } + if tt.config.Region == "" { + assert.Equal(t, "zh-CN", tt.config.Region) + } + if tt.config.MaxResults <= 0 { + assert.Equal(t, 10, tt.config.MaxResults) + } + }) + } +} + +func TestNewDDGS(t *testing.T) { + tests := []struct { + name string + config *Config + wantErr bool + }{ + { + name: "nil config", + config: nil, + wantErr: false, + }, + { + name: "valid config", + config: &Config{ + DDGConfig: &ddgsearch.Config{ + Timeout: 20, + Proxy: "http://proxy.example.com", + Cache: true, + MaxRetries: 5, + }, + }, + wantErr: false, + }, + { + name: "invalid timeout", + config: &Config{ + DDGConfig: &ddgsearch.Config{ + Timeout: -1, + }, + }, + wantErr: false, // won't error because Validate() fixes it + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ddgs, err := newDDGS(ctx, tt.config) + if tt.wantErr { + assert.Error(t, err) + return + } + mocker := MockDDGS() + assert.NoError(t, err) + assert.NotNil(t, ddgs) + assert.NotNil(t, ddgs.config) + assert.NotNil(t, ddgs.ddg) + mocker.UnPatch() + }) + } +} + +func TestDDGS_Search(t *testing.T) { + ctx := context.Background() + config := &Config{ + Region: "zh-CN", + MaxResults: 5, + } + + ddgs, err := newDDGS(ctx, config) + assert.NoError(t, err) + mocker := MockDDGS() + defer mocker.UnPatch() + + tests := []struct { + name string + request *SearchRequest + wantErr bool + }{ + { + name: "basic search", + request: &SearchRequest{ + Query: "golang testing", + Page: 1, + }, + wantErr: false, + }, + { + name: "empty query", + request: &SearchRequest{ + Query: "", + Page: 1, + }, + wantErr: true, + }, + { + name: "page number handling", + request: &SearchRequest{ + Query: "test query", + Page: 0, + }, + wantErr: false, + }, + { + name: "search with max results", + request: &SearchRequest{ + Query: "popular technology news", + Page: 1, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := ddgs.Search(ctx, tt.request) + + if err != nil { + if err.Error() == "no results found" { + // This is an acceptable case for any search + t.Logf("no results found for query: %s", tt.request.Query) + return + } + if tt.wantErr { + assert.Error(t, err) + return + } + t.Errorf("unexpected error: %v", err) + return + } + + if tt.wantErr { + t.Error("expected error but got none") + return + } + + assert.NotNil(t, resp) + assert.NotNil(t, resp.Results) + if len(resp.Results) > 0 { + result := resp.Results[0] + assert.NotEmpty(t, result.Title) + assert.NotEmpty(t, result.Link) + } + + for _, result := range resp.Results { + fmt.Printf("title: %s, description: %s, link: %s\n", result.Title, result.Description, result.Link) + } + }) + } +} + +func TestNewTool(t *testing.T) { + ctx := context.Background() + config := &Config{ + DDGConfig: &ddgsearch.Config{ + Timeout: 10, + MaxRetries: 5, + }, + } + tool, err := NewTool(ctx, config) + assert.NoError(t, err) + assert.NotNil(t, tool) +}