opt.: http req

lollipopkit · Jul 25, 2024 · f613301 · f613301
1 parent 83cf2dd
commit f613301
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 20 deletions.
diff --git a/lib/core/util/tool_func/func/http.dart b/lib/core/util/tool_func/func/http.dart
@@ -9,7 +9,7 @@ final class TfHttpReq extends ToolFunc {
           description: '''
 Send an HTTP request. It can be used for searching, downloading, etc.
 
-If for searching: use Bing.
+If user wants to search, use `bing.com` as the search engine and set `forSearch` to true.
 
 Both request/response body is String. If json, encode it into String.
 If blob, encode it into base64 String.''',
@@ -60,18 +60,13 @@ If blob, encode it into base64 String.''',
 
   @override
   Future<_Ret> run(_CallResp call, _Map args, OnToolLog log) async {
-    final method = args['method'] ?? 'GET';
+    final method = args['method'] as String? ?? 'GET';
     final url = args['url'] as String;
-    final headers = args['headers'];
-    final body = args['body'];
+    final headers = (args['headers'] as Map?)?.cast<String, dynamic>();
+    final body = args['body'] as String?;
     final forSearch = args['forSearch'] as bool? ?? false;
     final truncateSize = args['truncateSize'] as int?;
-    int? followRedirects;
-    try {
-      followRedirects = args['followRedirects'] as int?;
-    } catch (e, s) {
-      Loggers.app.warning('Failed to parse followRedirects', e, s);
-    }
+    final followRedirects = args['followRedirects'] as int?;
 
     log('Http $method -> $url');
     final resp = await myDio.request(
@@ -100,7 +95,7 @@ If blob, encode it into base64 String.''',
       'application/x-www-form-urlencoded',
     ];
 
-    final contentType = resp.headers['content-type']?.firstOrNull;
+    final contentType = resp.headers['content-type']?.join(';');
 
     String tryConvertStr(raw) {
       try {
@@ -123,16 +118,42 @@ If blob, encode it into base64 String.''',
     };
 
     if (forSearch) {
-      respBody = await compute(_filterHtml, respBody);
+      final urlMap = await compute(_filterHtmlUrls, respBody);
+      if (urlMap.isNotEmpty) {
+        respBody = '';
+
+        var count = 0;
+        for (final entry in urlMap.entries) {
+          if (count++ > 5) break;
+          final url = entry.value;
+          log('Http $method -> $url');
+          final resp = await myDio.request(
+            entry.value,
+            options: Options(
+              method: 'GET',
+              maxRedirects: followRedirects,
+              validateStatus: (_) => true,
+              responseType: ResponseType.plain,
+            ),
+          );
+
+          final data = resp.data;
+          if (data is! String) continue;
+          final html = await compute(_filterHtmlBody, data);
+          if (html != null) {
+            respBody += html;
+          }
+        }
+      }
     }
 
     if (truncateSize != null && respBody.length > truncateSize) {
       respBody = respBody.substring(0, truncateSize);
     }
 
-    await Future.delayed(Durations.medium1);
+    await Future.delayed(Durations.short3);
     log('Http $method -> ${l10n.success}');
-    await Future.delayed(Durations.medium1);
+    await Future.delayed(Durations.short3);
 
     return [ChatContent.text(respBody)];
   }

diff --git a/lib/core/util/tool_func/type.dart b/lib/core/util/tool_func/type.dart
@@ -26,7 +26,7 @@ Future<Map<String, dynamic>> _parseMap(dynamic value) async {
 }
 
 /// Only return the content insides body tag as a <title: url> map.
-String _filterHtml(String html) {
+Map<String, String> _filterHtmlUrls(String html) {
   // Remove the first line of <!DOCTYPE html>
   if (html.startsWith('<!')) {
     html = html.substring(html.indexOf('>') + 1);
@@ -36,19 +36,43 @@ String _filterHtml(String html) {
   final map = <String, String>{};
   // Find all <a> tag with href.
   for (final a in aInBody) {
-    final href = a.attributes['href'];
+    var href = a.attributes['href'];
     if (href == null) continue;
     // If there is no complete url in href, ignore it.
     // Usually, the uncomplete url is a relative path for the search engine.
     if (!httpReg.hasMatch(href)) continue;
     final title = a.text;
+    // `/url?q=` is the query string for google search result.
+    if (href.startsWith('/url?q=')) {
+      final uri = Uri.parse(href);
+      href = uri.queryParameters['q'] ?? href;
+    }
     map[title] = href;
   }
-  final sb = StringBuffer();
-  for (final entry in map.entries) {
-    sb.writeln('${entry.key}: ${entry.value}');
+  return map;
+}
+
+/// Return all text content insides body tag.
+String? _filterHtmlBody(String raw) {
+  final doc = html_parser.parse(raw);
+  final body = doc.querySelector('body');
+  final text = body?.text;
+  if (text == null || text.isEmpty) return null;
+
+  final lines = text.split('\n');
+  final rmIdxs = <int>[];
+  for (var i = 0; i < lines.length; i++) {
+    final line = lines[i];
+    if (line.trim().isEmpty) {
+      rmIdxs.add(i);
+    }
   }
-  return sb.toString();
+
+  for (var i = rmIdxs.length - 1; i >= 0; i--) {
+    lines.removeAt(rmIdxs[i]);
+  }
+
+  return lines.join('\n');
 }
 
 final httpReg = RegExp(r'https?://');