From a5c6f1321b2fb02081d5e2355ee00f4c27bbedb3 Mon Sep 17 00:00:00 2001 From: Shish Date: Thu, 14 Dec 2023 16:33:21 +0000 Subject: [PATCH] refactor search a little and add much better testing --- core/extension.php | 6 +- core/imageboard/image.php | 387 +---------------------------- core/imageboard/search.php | 369 +++++++++++++++++++++++++++- core/tests/SearchTest.php | 490 +++++++++++++++++++++++++++++++++++++ ext/artists/main.php | 2 +- ext/bulk_actions/main.php | 2 +- ext/danbooru_api/main.php | 4 +- ext/favorites/main.php | 2 +- ext/home/main.php | 2 +- ext/image/main.php | 2 +- ext/index/main.php | 10 +- ext/index/test.php | 185 -------------- ext/numeric_score/main.php | 4 +- ext/ouroboros_api/main.php | 2 +- ext/pools/main.php | 2 +- ext/rating/main.php | 2 +- ext/regen_thumb/main.php | 2 +- ext/rss_images/main.php | 2 +- ext/rule34/main.php | 2 +- ext/shimmie_api/main.php | 4 +- ext/sitemap/main.php | 6 +- ext/tag_edit/main.php | 6 +- tests/bootstrap.php | 10 +- 23 files changed, 902 insertions(+), 601 deletions(-) create mode 100644 core/tests/SearchTest.php diff --git a/core/extension.php b/core/extension.php index cf246e8cb..296934139 100644 --- a/core/extension.php +++ b/core/extension.php @@ -110,6 +110,10 @@ protected function set_version(string $name, int $ver) } } +class ExtensionNotFound extends SCoreException +{ +} + enum ExtensionVisibility { case DEFAULT; @@ -235,7 +239,7 @@ public static function get_for_extension_class(string $base): ExtensionInfo return self::$all_info_by_class[$normal]; } else { $infos = print_r(array_keys(self::$all_info_by_class), true); - throw new SCoreException("$normal not found in {$infos}"); + throw new ExtensionNotFound("$normal not found in {$infos}"); } } diff --git a/core/imageboard/image.php b/core/imageboard/image.php index fff41e985..12bbaae60 100644 --- a/core/imageboard/image.php +++ b/core/imageboard/image.php @@ -126,7 +126,7 @@ public static function by_id_or_hash(string $id): ?Image public static function by_random(array $tags = [], int $limit_range = 0): ?Image { - $max = Image::count_images($tags); + $max = Search::count_images($tags); if ($max < 1) { return null; } // From Issue #22 - opened by HungryFeline on May 30, 2011. @@ -134,7 +134,7 @@ public static function by_random(array $tags = [], int $limit_range = 0): ?Image $max = $limit_range; } $rand = mt_rand(0, $max - 1); - $set = Image::find_images($rand, 1, $tags); + $set = Search::find_images($rand, 1, $tags); if (count($set) > 0) { return $set[0]; } else { @@ -142,138 +142,6 @@ public static function by_random(array $tags = [], int $limit_range = 0): ?Image } } - private static function find_images_internal(int $start = 0, ?int $limit = null, array $tags = []): iterable - { - global $database, $user; - - if ($start < 0) { - $start = 0; - } - if ($limit !== null && $limit < 1) { - $limit = 1; - } - - if (SPEED_HAX) { - if (!$user->can(Permissions::BIG_SEARCH) and count($tags) > 3) { - throw new PermissionDeniedException("Anonymous users may only search for up to 3 tags at a time"); - } - } - - [$tag_conditions, $img_conditions, $order] = self::terms_to_conditions($tags); - $querylet = self::build_search_querylet($tag_conditions, $img_conditions, $order, $limit, $start); - return $database->get_all_iterable($querylet->sql, $querylet->variables); - } - - /** - * Search for an array of images - * - * @param string[] $tags - * @return Image[] - */ - #[Query(name: "posts", type: "[Post!]!", args: ["tags" => "[string!]"])] - public static function find_images(int $offset = 0, ?int $limit = null, array $tags = []): array - { - $result = self::find_images_internal($offset, $limit, $tags); - - $images = []; - foreach ($result as $row) { - $images[] = new Image($row); - } - return $images; - } - - /** - * Search for an array of images, returning a iterable object of Image - */ - public static function find_images_iterable(int $start = 0, ?int $limit = null, array $tags = []): \Generator - { - $result = self::find_images_internal($start, $limit, $tags); - foreach ($result as $row) { - yield new Image($row); - } - } - - /* - * Image-related utility functions - */ - - public static function count_total_images(): int - { - global $cache, $database; - $total = $cache->get("image-count"); - if (is_null($total)) { - $total = (int)$database->get_one("SELECT COUNT(*) FROM images"); - $cache->set("image-count", $total, 600); - } - return $total; - } - - public static function count_tag(string $tag): int - { - global $database; - return (int)$database->get_one( - "SELECT count FROM tags WHERE LOWER(tag) = LOWER(:tag)", - ["tag" => $tag] - ); - } - - /** - * Count the number of image results for a given search - * - * @param string[] $tags - */ - public static function count_images(array $tags = []): int - { - global $cache, $database; - $tag_count = count($tags); - - if (SPEED_HAX && $tag_count === 0) { - // total number of images in the DB - $total = self::count_total_images(); - } elseif (SPEED_HAX && $tag_count === 1 && !preg_match("/[:=><\*\?]/", $tags[0])) { - if (!str_starts_with($tags[0], "-")) { - // one tag - we can look that up directly - $total = self::count_tag($tags[0]); - } else { - // one negative tag - subtract from the total - $total = self::count_total_images() - self::count_tag(substr($tags[0], 1)); - } - } else { - // complex query - // implode(tags) can be too long for memcache... - $cache_key = "image-count:" . md5(Tag::implode($tags)); - $total = $cache->get($cache_key); - if (is_null($total)) { - if (Extension::is_enabled(RatingsInfo::KEY)) { - $tags[] = "rating:*"; - } - [$tag_conditions, $img_conditions, $order] = self::terms_to_conditions($tags); - $querylet = self::build_search_querylet($tag_conditions, $img_conditions, $order); - $total = (int)$database->get_one("SELECT COUNT(*) AS cnt FROM ($querylet->sql) AS tbl", $querylet->variables); - if (SPEED_HAX && $total > 5000) { - // when we have a ton of images, the count - // won't change dramatically very often - $cache->set($cache_key, $total, 3600); - } - } - } - if (is_null($total)) { - return 0; - } - return $total; - } - - /** - * Count the number of pages for a given search - * - * @param string[] $tags - */ - public static function count_pages(array $tags = []): int - { - global $config; - return (int)ceil(Image::count_images($tags) / $config->get_int(IndexConfig::IMAGES)); - } - /* * Accessors & mutators */ @@ -306,16 +174,13 @@ public function get_next(array $tags = [], bool $next = true): ?Image ORDER BY images.id '.$dir.' LIMIT 1 '); + return ($row ? new Image($row) : null); } else { $tags[] = 'id'. $gtlt . $this->id; $tags[] = 'order:id_'. strtolower($dir); - [$tag_conditions, $img_conditions, $order] = self::terms_to_conditions($tags); - $querylet = self::build_search_querylet($tag_conditions, $img_conditions, $order); - $querylet->append_sql(' LIMIT 1'); - $row = $database->get_row($querylet->sql, $querylet->variables); + $images = Search::find_images(0, 1, $tags); + return (count($images) > 0) ? $images[0] : null; } - - return ($row ? new Image($row) : null); } /** @@ -740,246 +605,4 @@ public function parse_link_template(string $tmpl, int $n = 0): string $tmpl = $plte->link; return load_balance_url($tmpl, $this->hash, $n); } - - private static function tag_or_wildcard_to_ids(string $tag): array - { - global $database; - $sq = "SELECT id FROM tags WHERE LOWER(tag) LIKE LOWER(:tag)"; - if ($database->get_driver_id() === DatabaseDriverID::SQLITE) { - $sq .= "ESCAPE '\\'"; - } - return $database->get_col($sq, ["tag" => Tag::sqlify($tag)]); - } - - /** - * Turn a human input string into a an abstract search query - * - * @param string[] $terms - * @return array{0: TagCondition[], 1: ImgCondition[], 2: string} - */ - private static function terms_to_conditions(array $terms): array - { - global $config; - - $tag_conditions = []; - $img_conditions = []; - $order = null; - - /* - * Turn a bunch of strings into a bunch of TagCondition - * and ImgCondition objects - */ - $stpen = 0; // search term parse event number - foreach (array_merge([null], $terms) as $term) { - $stpe = send_event(new SearchTermParseEvent($stpen++, $term, $terms)); - $order ??= $stpe->order; - $img_conditions = array_merge($img_conditions, $stpe->img_conditions); - $tag_conditions = array_merge($tag_conditions, $stpe->tag_conditions); - } - - $order = ($order ?: "images.".$config->get_string(IndexConfig::ORDER)); - - return [$tag_conditions, $img_conditions, $order]; - } - - /** - * Turn an abstract search query into an SQL Querylet - * - * Must follow the format - * - * SELECT images.* - * FROM (...) AS images - * WHERE (...) - * - * ie, return a set of images.* columns, and end with a WHERE - * - * @param TagCondition[] $tag_conditions - * @param ImgCondition[] $img_conditions - */ - private static function build_search_querylet( - array $tag_conditions, - array $img_conditions, - string $order, - ?int $limit = null, - ?int $offset = null - ): Querylet { - // no tags, do a simple search - if (count($tag_conditions) === 0) { - $query = new Querylet("SELECT images.* FROM images WHERE 1=1"); - } - - // one tag sorted by ID - we can fetch this from the image_tags table, - // and do the offset / limit there, which is 10x faster than fetching - // all the image_tags and doing the offset / limit on the result. - // - // NOTE: this is currently impossible to test, because the test suite - // loads all extensions, some of whom add generic img_conditions onto - // the search, which prevents this optimisation from being used. - elseif ( - count($tag_conditions) === 1 - && $tag_conditions[0]->positive - // We can only do this if img_conditions is empty, because - // we're going to apply the offset / limit to the image_tags - // subquery, and applying extra conditions to the top-level - // query might reduce the total results below the target limit - && empty($img_conditions) - // We can only do this if we're sorting by ID, because - // we're going to be using the image_tags table, which - // only has image_id and tag_id, not any other columns - && ($order == "id DESC" || $order == "images.id DESC") - // This is only an optimisation if we are applying limit - // and offset - && !is_null($limit) - && !is_null($offset) - ) { - $tc = $tag_conditions[0]; - // IN (SELECT id FROM tags) is 100x slower than doing a separate - // query and then a second query for IN(first_query_results)?? - $tag_array = self::tag_or_wildcard_to_ids($tc->tag); - if (count($tag_array) == 0) { - // if wildcard expanded to nothing, take a shortcut - if ($tc->positive) { - $query = new Querylet("SELECT images.* FROM images WHERE 1=0"); - } else { - $query = new Querylet("SELECT images.* FROM images WHERE 1=1"); - } - } else { - $set = implode(', ', $tag_array); - $query = new Querylet(" - SELECT images.* - FROM images INNER JOIN ( - SELECT DISTINCT it.image_id - FROM image_tags it - WHERE it.tag_id IN ($set) - ORDER BY it.image_id DESC - LIMIT :limit OFFSET :offset - ) a on a.image_id = images.id - WHERE 1=1 - ", ["limit" => $limit, "offset" => $offset]); - // don't offset at the image level because - // we already offset at the image_tags level - $limit = null; - $offset = null; - } - } - - // more than one tag, or more than zero other conditions, or a non-default sort order - else { - $positive_tag_id_array = []; - $positive_wildcard_id_array = []; - $negative_tag_id_array = []; - $all_nonexistent_negatives = true; - - foreach ($tag_conditions as $tq) { - $tag_ids = self::tag_or_wildcard_to_ids($tq->tag); - $tag_count = count($tag_ids); - - if ($tq->positive) { - $all_nonexistent_negatives = false; - if ($tag_count == 0) { - # one of the positive tags had zero results, therefor there - # can be no results; "where 1=0" should shortcut things - return new Querylet("SELECT images.* FROM images WHERE 1=0"); - } elseif ($tag_count == 1) { - // All wildcard terms that qualify for a single tag can be treated the same as non-wildcards - $positive_tag_id_array[] = $tag_ids[0]; - } else { - // Terms that resolve to multiple tags act as an OR within themselves - // and as an AND in relation to all other terms, - $positive_wildcard_id_array[] = $tag_ids; - } - } else { - if ($tag_count > 0) { - $all_nonexistent_negatives = false; - // Unlike positive criteria, negative criteria are all handled in an OR fashion, - // so we can just compile them all into a single sub-query. - $negative_tag_id_array = array_merge($negative_tag_id_array, $tag_ids); - } - } - } - - assert($positive_tag_id_array || $positive_wildcard_id_array || $negative_tag_id_array || $all_nonexistent_negatives, @$_GET['q']); - - if ($all_nonexistent_negatives) { - $query = new Querylet("SELECT images.* FROM images WHERE 1=1"); - } elseif (!empty($positive_tag_id_array) || !empty($positive_wildcard_id_array)) { - $inner_joins = []; - if (!empty($positive_tag_id_array)) { - foreach ($positive_tag_id_array as $tag) { - $inner_joins[] = "= $tag"; - } - } - if (!empty($positive_wildcard_id_array)) { - foreach ($positive_wildcard_id_array as $tags) { - $positive_tag_id_list = join(', ', $tags); - $inner_joins[] = "IN ($positive_tag_id_list)"; - } - } - - $first = array_shift($inner_joins); - $sub_query = "SELECT DISTINCT it.image_id FROM image_tags it "; - $i = 0; - foreach ($inner_joins as $inner_join) { - $i++; - $sub_query .= " INNER JOIN image_tags it$i ON it$i.image_id = it.image_id AND it$i.tag_id $inner_join "; - } - if (!empty($negative_tag_id_array)) { - $negative_tag_id_list = join(', ', $negative_tag_id_array); - $sub_query .= " LEFT JOIN image_tags negative ON negative.image_id = it.image_id AND negative.tag_id IN ($negative_tag_id_list) "; - } - $sub_query .= "WHERE it.tag_id $first "; - if (!empty($negative_tag_id_array)) { - $sub_query .= " AND negative.image_id IS NULL"; - } - $sub_query .= " GROUP BY it.image_id "; - - $query = new Querylet(" - SELECT images.* - FROM images - INNER JOIN ($sub_query) a on a.image_id = images.id - "); - } elseif (!empty($negative_tag_id_array)) { - $negative_tag_id_list = join(', ', $negative_tag_id_array); - $query = new Querylet(" - SELECT images.* - FROM images - LEFT JOIN image_tags negative ON negative.image_id = images.id AND negative.tag_id in ($negative_tag_id_list) - WHERE negative.image_id IS NULL - "); - } else { - throw new SCoreException("No criteria specified"); - } - } - - /* - * Merge all the image metadata searches into one generic querylet - * and append to the base querylet with "AND blah" - */ - if (!empty($img_conditions)) { - $n = 0; - $img_sql = ""; - $img_vars = []; - foreach ($img_conditions as $iq) { - if ($n++ > 0) { - $img_sql .= " AND"; - } - if (!$iq->positive) { - $img_sql .= " NOT"; - } - $img_sql .= " (" . $iq->qlet->sql . ")"; - $img_vars = array_merge($img_vars, $iq->qlet->variables); - } - $query->append_sql(" AND "); - $query->append(new Querylet($img_sql, $img_vars)); - } - - $query->append(new Querylet(" ORDER BY ".$order)); - - if (!is_null($limit)) { - $query->append(new Querylet(" LIMIT :limit ", ["limit" => $limit])); - $query->append(new Querylet(" OFFSET :offset ", ["offset" => $offset])); - } - - return $query; - } } diff --git a/core/imageboard/search.php b/core/imageboard/search.php index 624f52237..d6af5815e 100644 --- a/core/imageboard/search.php +++ b/core/imageboard/search.php @@ -4,6 +4,8 @@ namespace Shimmie2; +use GQLA\Query; + class Querylet { public function __construct( @@ -33,7 +35,7 @@ class TagCondition { public function __construct( public string $tag, - public bool $positive, + public bool $positive = true, ) { } } @@ -42,7 +44,370 @@ class ImgCondition { public function __construct( public Querylet $qlet, - public bool $positive, + public bool $positive = true, ) { } } + +class Search +{ + public static array $_search_path = []; + + private static function find_images_internal(int $start = 0, ?int $limit = null, array $tags = []): iterable + { + global $database, $user; + + if ($start < 0) { + $start = 0; + } + if ($limit !== null && $limit < 1) { + $limit = 1; + } + + if (SPEED_HAX) { + if (!$user->can(Permissions::BIG_SEARCH) and count($tags) > 3) { + throw new PermissionDeniedException("Anonymous users may only search for up to 3 tags at a time"); + } + } + + [$tag_conditions, $img_conditions, $order] = self::terms_to_conditions($tags); + $querylet = self::build_search_querylet($tag_conditions, $img_conditions, $order, $limit, $start); + return $database->get_all_iterable($querylet->sql, $querylet->variables); + } + + /** + * Search for an array of images + * + * @param string[] $tags + * @return Image[] + */ + #[Query(name: "posts", type: "[Post!]!", args: ["tags" => "[string!]"])] + public static function find_images(int $offset = 0, ?int $limit = null, array $tags = []): array + { + $result = self::find_images_internal($offset, $limit, $tags); + + $images = []; + foreach ($result as $row) { + $images[] = new Image($row); + } + return $images; + } + + /** + * Search for an array of images, returning a iterable object of Image + */ + public static function find_images_iterable(int $start = 0, ?int $limit = null, array $tags = []): \Generator + { + $result = self::find_images_internal($start, $limit, $tags); + foreach ($result as $row) { + yield new Image($row); + } + } + + /* + * Image-related utility functions + */ + + public static function count_tag(string $tag): int + { + global $database; + return (int)$database->get_one( + "SELECT count FROM tags WHERE LOWER(tag) = LOWER(:tag)", + ["tag" => $tag] + ); + } + + private static function count_total_images(): int + { + global $cache, $database; + $total = $cache->get("image-count"); + if (is_null($total)) { + $total = (int)$database->get_one("SELECT COUNT(*) FROM images"); + $cache->set("image-count", $total, 600); + } + return $total; + } + + /** + * Count the number of image results for a given search + * + * @param string[] $tags + */ + public static function count_images(array $tags = []): int + { + global $cache, $database; + $tag_count = count($tags); + + // SPEED_HAX ignores the fact that extensions can add img_conditions + // even when there are no tags being searched for + if (SPEED_HAX && $tag_count === 0) { + // total number of images in the DB + $total = self::count_total_images(); + } elseif (SPEED_HAX && $tag_count === 1 && !preg_match("/[:=><\*\?]/", $tags[0])) { + if (!str_starts_with($tags[0], "-")) { + // one positive tag - we can look that up directly + $total = self::count_tag($tags[0]); + } else { + // one negative tag - subtract from the total + $total = self::count_total_images() - self::count_tag(substr($tags[0], 1)); + } + } else { + // complex query + // implode(tags) can be too long for memcache, so use the hash of tags as the key + $cache_key = "image-count:" . md5(Tag::implode($tags)); + $total = $cache->get($cache_key); + if (is_null($total)) { + if (Extension::is_enabled(RatingsInfo::KEY)) { + $tags[] = "rating:*"; + } + [$tag_conditions, $img_conditions, $order] = self::terms_to_conditions($tags); + $querylet = self::build_search_querylet($tag_conditions, $img_conditions, $order); + $total = (int)$database->get_one("SELECT COUNT(*) AS cnt FROM ($querylet->sql) AS tbl", $querylet->variables); + if (SPEED_HAX && $total > 5000) { + // when we have a ton of images, the count + // won't change dramatically very often + $cache->set($cache_key, $total, 3600); + } + } + } + if (is_null($total)) { + return 0; + } + return $total; + } + + + private static function tag_or_wildcard_to_ids(string $tag): array + { + global $database; + $sq = "SELECT id FROM tags WHERE LOWER(tag) LIKE LOWER(:tag)"; + if ($database->get_driver_id() === DatabaseDriverID::SQLITE) { + $sq .= "ESCAPE '\\'"; + } + return $database->get_col($sq, ["tag" => Tag::sqlify($tag)]); + } + + /** + * Turn a human input string into a an abstract search query + * + * @param string[] $terms + * @return array{0: TagCondition[], 1: ImgCondition[], 2: string} + */ + private static function terms_to_conditions(array $terms): array + { + global $config; + + $tag_conditions = []; + $img_conditions = []; + $order = null; + + /* + * Turn a bunch of strings into a bunch of TagCondition + * and ImgCondition objects + */ + $stpen = 0; // search term parse event number + foreach (array_merge([null], $terms) as $term) { + $stpe = send_event(new SearchTermParseEvent($stpen++, $term, $terms)); + $order ??= $stpe->order; + $img_conditions = array_merge($img_conditions, $stpe->img_conditions); + $tag_conditions = array_merge($tag_conditions, $stpe->tag_conditions); + } + + $order = ($order ?: "images.".$config->get_string(IndexConfig::ORDER)); + + return [$tag_conditions, $img_conditions, $order]; + } + + /** + * Turn an abstract search query into an SQL Querylet + * + * @param TagCondition[] $tag_conditions + * @param ImgCondition[] $img_conditions + */ + private static function build_search_querylet( + array $tag_conditions, + array $img_conditions, + string $order, + ?int $limit = null, + ?int $offset = null + ): Querylet { + // no tags, do a simple search + if (count($tag_conditions) === 0) { + static::$_search_path[] = "no_tags"; + $query = new Querylet("SELECT images.* FROM images WHERE 1=1"); + } + + // one tag sorted by ID - we can fetch this from the image_tags table, + // and do the offset / limit there, which is 10x faster than fetching + // all the image_tags and doing the offset / limit on the result. + elseif ( + count($tag_conditions) === 1 + && $tag_conditions[0]->positive + // We can only do this if img_conditions is empty, because + // we're going to apply the offset / limit to the image_tags + // subquery, and applying extra conditions to the top-level + // query might reduce the total results below the target limit + && empty($img_conditions) + // We can only do this if we're sorting by ID, because + // we're going to be using the image_tags table, which + // only has image_id and tag_id, not any other columns + && ($order == "id DESC" || $order == "images.id DESC") + // This is only an optimisation if we are applying limit + // and offset + && !is_null($limit) + && !is_null($offset) + ) { + static::$_search_path[] = "fast"; + $tc = $tag_conditions[0]; + // IN (SELECT id FROM tags) is 100x slower than doing a separate + // query and then a second query for IN(first_query_results)?? + $tag_array = self::tag_or_wildcard_to_ids($tc->tag); + if (count($tag_array) == 0) { + // if wildcard expanded to nothing, take a shortcut + static::$_search_path[] = "invalid_tag"; + $query = new Querylet("SELECT images.* FROM images WHERE 1=0"); + } else { + $set = implode(', ', $tag_array); + $query = new Querylet(" + SELECT images.* + FROM images INNER JOIN ( + SELECT DISTINCT it.image_id + FROM image_tags it + WHERE it.tag_id IN ($set) + ORDER BY it.image_id DESC + LIMIT :limit OFFSET :offset + ) a on a.image_id = images.id + WHERE 1=1 + ", ["limit" => $limit, "offset" => $offset]); + // don't offset at the image level because + // we already offset at the image_tags level + $limit = null; + $offset = null; + } + } + + // more than one tag, or more than zero other conditions, or a non-default sort order + else { + static::$_search_path[] = "general"; + $positive_tag_id_array = []; + $positive_wildcard_id_array = []; + $negative_tag_id_array = []; + $all_nonexistent_negatives = true; + + foreach ($tag_conditions as $tq) { + $tag_ids = self::tag_or_wildcard_to_ids($tq->tag); + $tag_count = count($tag_ids); + + if ($tq->positive) { + $all_nonexistent_negatives = false; + if ($tag_count == 0) { + # one of the positive tags had zero results, therefor there + # can be no results; "where 1=0" should shortcut things + static::$_search_path[] = "invalid_tag"; + return new Querylet("SELECT images.* FROM images WHERE 1=0"); + } elseif ($tag_count == 1) { + // All wildcard terms that qualify for a single tag can be treated the same as non-wildcards + $positive_tag_id_array[] = $tag_ids[0]; + } else { + // Terms that resolve to multiple tags act as an OR within themselves + // and as an AND in relation to all other terms, + $positive_wildcard_id_array[] = $tag_ids; + } + } else { + if ($tag_count > 0) { + $all_nonexistent_negatives = false; + // Unlike positive criteria, negative criteria are all handled in an OR fashion, + // so we can just compile them all into a single sub-query. + $negative_tag_id_array = array_merge($negative_tag_id_array, $tag_ids); + } + } + } + + assert($positive_tag_id_array || $positive_wildcard_id_array || $negative_tag_id_array || $all_nonexistent_negatives, @$_GET['q']); + + if ($all_nonexistent_negatives) { + static::$_search_path[] = "all_nonexistent_negatives"; + $query = new Querylet("SELECT images.* FROM images WHERE 1=1"); + } elseif (!empty($positive_tag_id_array) || !empty($positive_wildcard_id_array)) { + static::$_search_path[] = "some_positives"; + $inner_joins = []; + if (!empty($positive_tag_id_array)) { + foreach ($positive_tag_id_array as $tag) { + $inner_joins[] = "= $tag"; + } + } + if (!empty($positive_wildcard_id_array)) { + foreach ($positive_wildcard_id_array as $tags) { + $positive_tag_id_list = join(', ', $tags); + $inner_joins[] = "IN ($positive_tag_id_list)"; + } + } + + $first = array_shift($inner_joins); + $sub_query = "SELECT DISTINCT it.image_id FROM image_tags it "; + $i = 0; + foreach ($inner_joins as $inner_join) { + $i++; + $sub_query .= " INNER JOIN image_tags it$i ON it$i.image_id = it.image_id AND it$i.tag_id $inner_join "; + } + if (!empty($negative_tag_id_array)) { + $negative_tag_id_list = join(', ', $negative_tag_id_array); + $sub_query .= " LEFT JOIN image_tags negative ON negative.image_id = it.image_id AND negative.tag_id IN ($negative_tag_id_list) "; + } + $sub_query .= "WHERE it.tag_id $first "; + if (!empty($negative_tag_id_array)) { + $sub_query .= " AND negative.image_id IS NULL"; + } + $sub_query .= " GROUP BY it.image_id "; + + $query = new Querylet(" + SELECT images.* + FROM images + INNER JOIN ($sub_query) a on a.image_id = images.id + "); + } elseif (!empty($negative_tag_id_array)) { + static::$_search_path[] = "only_negative_tags"; + $negative_tag_id_list = join(', ', $negative_tag_id_array); + $query = new Querylet(" + SELECT images.* + FROM images + LEFT JOIN image_tags negative ON negative.image_id = images.id AND negative.tag_id in ($negative_tag_id_list) + WHERE negative.image_id IS NULL + "); + } else { + throw new SCoreException("No criteria specified"); + } + } + + /* + * Merge all the image metadata searches into one generic querylet + * and append to the base querylet with "AND blah" + */ + if (!empty($img_conditions)) { + $n = 0; + $img_sql = ""; + $img_vars = []; + foreach ($img_conditions as $iq) { + if ($n++ > 0) { + $img_sql .= " AND"; + } + if (!$iq->positive) { + $img_sql .= " NOT"; + } + $img_sql .= " (" . $iq->qlet->sql . ")"; + $img_vars = array_merge($img_vars, $iq->qlet->variables); + } + $query->append_sql(" AND "); + $query->append(new Querylet($img_sql, $img_vars)); + } + + $query->append(new Querylet(" ORDER BY ".$order)); + + if (!is_null($limit)) { + $query->append(new Querylet(" LIMIT :limit ", ["limit" => $limit])); + $query->append(new Querylet(" OFFSET :offset ", ["offset" => $offset])); + } + + return $query; + } +} diff --git a/core/tests/SearchTest.php b/core/tests/SearchTest.php new file mode 100644 index 000000000..303072f7b --- /dev/null +++ b/core/tests/SearchTest.php @@ -0,0 +1,490 @@ +log_in_as_user(); + $image_id_1 = $this->post_image("tests/pbx_screenshot.jpg", "question? colon:thing exclamation!"); + $image_id_2 = $this->post_image("tests/bedroom_workshop.jpg", "question. colon_thing exclamation%"); + + $this->assert_search_results(["question?"], [$image_id_1]); + $this->assert_search_results(["question."], [$image_id_2]); + $this->assert_search_results(["colon:thing"], [$image_id_1]); + $this->assert_search_results(["colon_thing"], [$image_id_2]); + $this->assert_search_results(["exclamation!"], [$image_id_1]); + $this->assert_search_results(["exclamation%"], [$image_id_2]); + } + + // base case + public function testUpload(): array + { + $this->log_in_as_user(); + $image_id_1 = $this->post_image("tests/pbx_screenshot.jpg", "thing computer screenshot pbx phone"); + $image_id_2 = $this->post_image("tests/bedroom_workshop.jpg", "thing computer computing bedroom workshop"); + $this->log_out(); + + # make sure both uploads were ok + $this->assertTrue($image_id_1 > 0); + $this->assertTrue($image_id_2 > 0); + + return [$image_id_1, $image_id_2]; + } + + + /******************************************************** + * Test turning a string into an abstract query + */ + private function assert_TTC(string $tags, array $expected_tag_conditions, array $expected_img_conditions, string $expected_order) + { + $class = new \ReflectionClass('\Shimmie2\Search'); + $terms_to_conditions = $class->getMethod("terms_to_conditions"); + $terms_to_conditions->setAccessible(true); // Use this if you are running PHP older than 8.1.0 + + $obj = new Search(); + [$tag_conditions, $img_conditions, $order] = $terms_to_conditions->invokeArgs($obj, [Tag::explode($tags, false)]); + + static::assertThat( + [ + "tags" => $expected_tag_conditions, + "imgs" => $expected_img_conditions, + "order" => $expected_order, + ], + new IsEqual([ + "tags" => $tag_conditions, + "imgs" => $img_conditions, + "order" => $order, + ]) + ); + } + + public function testTTC_Empty() + { + $this->assert_TTC( + "", + [ + ], + [ + new ImgCondition(new Querylet("trash != :true", ["true" => true])), + new ImgCondition(new Querylet("private != :true OR owner_id = :private_owner_id", [ + "private_owner_id" => 1, + "true" => true])), + new ImgCondition(new Querylet("rating IN ('?', 's', 'q', 'e')", [])), + ], + "images.id DESC" + ); + } + + public function testTTC_Hash() + { + $this->assert_TTC( + "hash=1234567890", + [ + ], + [ + new ImgCondition(new Querylet("trash != :true", ["true" => true])), + new ImgCondition(new Querylet("private != :true OR owner_id = :private_owner_id", [ + "private_owner_id" => 1, + "true" => true])), + new ImgCondition(new Querylet("rating IN ('?', 's', 'q', 'e')", [])), + new ImgCondition(new Querylet("images.hash = :hash", ["hash" => "1234567890"])), + ], + "images.id DESC" + ); + } + + public function testTTC_Ratio() + { + $this->assert_TTC( + "ratio=42:12345", + [ + ], + [ + new ImgCondition(new Querylet("trash != :true", ["true" => true])), + new ImgCondition(new Querylet("private != :true OR owner_id = :private_owner_id", [ + "private_owner_id" => 1, + "true" => true])), + new ImgCondition(new Querylet("rating IN ('?', 's', 'q', 'e')", [])), + new ImgCondition(new Querylet("width / :width1 = height / :height1", ['width1' => 42, + 'height1' => 12345])), + ], + "images.id DESC" + ); + } + + public function testTTC_Order() + { + $this->assert_TTC( + "order=score", + [ + ], + [ + new ImgCondition(new Querylet("trash != :true", ["true" => true])), + new ImgCondition(new Querylet("private != :true OR owner_id = :private_owner_id", [ + "private_owner_id" => 1, + "true" => true])), + new ImgCondition(new Querylet("rating IN ('?', 's', 'q', 'e')", [])), + ], + "images.numeric_score DESC" + ); + } + + /******************************************************** + * Test turning an abstract query into SQL + fetching the results + */ + private function assert_BSQ( + array $tcs = [], + array $ics = [], + string $order = "id DESC", + int $limit = 9999, + int $start = 0, + array $res = [], + array $path = null, + ) { + global $database; + + $tcs = array_map( + fn ($tag) => ($tag[0] == "-") ? + new TagCondition(substr($tag, 1), false) : + new TagCondition($tag), + $tcs + ); + + $ics = array_map( + fn ($ic) => send_event(new SearchTermParseEvent(0, $ic, []))->img_conditions, + $ics + ); + $ics = array_merge(...$ics); + + Search::$_search_path = []; + + $class = new \ReflectionClass('\Shimmie2\Search'); + $build_search_querylet = $class->getMethod("build_search_querylet"); + $build_search_querylet->setAccessible(true); // Use this if you are running PHP older than 8.1.0 + + $obj = new Search(); + $querylet = $build_search_querylet->invokeArgs($obj, [$tcs, $ics, $order, $limit, $start]); + + $results = $database->get_all($querylet->sql, $querylet->variables); + + static::assertThat( + [ + "res" => array_map(fn ($row) => $row['id'], $results), + "path" => Search::$_search_path, + ], + new IsEqual([ + "res" => $res, + "path" => $path ?? Search::$_search_path, + ]) + ); + } + + /* * * * * * * * * * * + * No-tag search * + * * * * * * * * * * */ + #[Depends('testUpload')] + public function testBSQ_NoTags($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: [], + res: [$image_ids[1], $image_ids[0]], + path: ["no_tags"], + ); + } + + /* * * * * * * * * * * + * Fast-path search * + * * * * * * * * * * */ + #[Depends('testUpload')] + public function testBSQ_FastPath_NoResults($image_ids) + { + $this->testUpload(); + $this->assert_BSQ( + tcs: ["maumaumau"], + res: [], + path: ["fast", "invalid_tag"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_FastPath_OneResult($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["pbx"], + res: [$image_ids[0]], + path: ["fast"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_FastPath_ManyResults($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["computer"], + res: [$image_ids[1], $image_ids[0]], + path: ["fast"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_FastPath_WildNoResults($image_ids) + { + $this->testUpload(); + $this->assert_BSQ( + tcs: ["asdfasdf*"], + res: [], + path: ["fast", "invalid_tag"], + ); + } + + /** + * Only the first image matches both the wildcard and the tag. + * This checks for a bug where searching for "a* b" would return + * an image tagged "a1 a2" because the number of matched tags + * was equal to the number of searched tags. + * + * https://github.com/shish/shimmie2/issues/547 + */ + #[Depends('testUpload')] + public function testBSQ_FastPath_WildOneResult($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["screen*"], + res: [$image_ids[0]], + path: ["fast"], + ); + } + + /** + * Test that the fast path doesn't return duplicate results + * when a wildcard matches one image multiple times. + */ + #[Depends('testUpload')] + public function testBSQ_FastPath_WildManyResults($image_ids) + { + $image_ids = $this->testUpload(); + // two images match comp* - one matches it once, one matches it twice + $this->assert_BSQ( + tcs: ["comp*"], + res: [$image_ids[1], $image_ids[0]], + path: ["fast"], + ); + } + + /* * * * * * * * * * * + * General search * + * * * * * * * * * * */ + #[Depends('testUpload')] + public function testBSQ_GeneralPath_NoResults($image_ids) + { + $this->testUpload(); + # multiple tags, one of which doesn't exist + # (test the "one tag doesn't exist = no hits" path) + $this->assert_BSQ( + tcs: ["computer", "not_a_tag"], + res: [], + path: ["general", "invalid_tag"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_OneResult($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["computer", "screenshot"], + res: [$image_ids[0]], + path: ["general", "some_positives"], + ); + } + + /** + * Only the first image matches both the wildcard and the tag. + * This checks for a bug where searching for "a* b" would return + * an image tagged "a1 a2" because the number of matched tags + * was equal to the number of searched tags. + * + * https://github.com/shish/shimmie2/issues/547 + */ + #[Depends('testUpload')] + public function testBSQ_GeneralPath_WildOneResult($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["comp*", "screenshot"], + res: [$image_ids[0]], + path: ["general", "some_positives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_ManyResults($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["computer", "thing"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "some_positives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_WildManyResults($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["comp*", "-asdf"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "some_positives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_SubtractValidFromResults($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["computer", "-pbx"], + res: [$image_ids[1]], + path: ["general", "some_positives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_SubtractNotValidFromResults($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + tcs: ["computer", "-not_a_tag"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "some_positives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_SubtractValidFromDefault($image_ids) + { + $image_ids = $this->testUpload(); + // negative tag alone, should remove the image with that tag + $this->assert_BSQ( + tcs: ["-pbx"], + res: [$image_ids[1]], + path: ["general", "only_negative_tags"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_SubtractNotValidFromDefault($image_ids) + { + $image_ids = $this->testUpload(); + // negative that doesn't exist, should return all results + $this->assert_BSQ( + tcs: ["-not_a_tag"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "all_nonexistent_negatives"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_GeneralPath_SubtractMultipleNotValidFromDefault($image_ids) + { + $image_ids = $this->testUpload(); + // multiple negative tags that don't exist, should return all results + $this->assert_BSQ( + tcs: ["-not_a_tag", "-also_not_a_tag"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "all_nonexistent_negatives"], + ); + } + + /* * * * * * * * * * * + * Meta Search * + * * * * * * * * * * */ + #[Depends('testUpload')] + public function testBSQ_ImgCond_NoResults($image_ids) + { + $this->testUpload(); + $this->assert_BSQ( + ics: ["hash=1234567890"], + res: [], + path: ["no_tags"], + ); + $this->assert_BSQ( + ics: ["ratio=42:12345"], + res: [], + path: ["no_tags"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_ImgCond_OneResult($image_ids) + { + $image_ids = $this->testUpload(); + $this->assert_BSQ( + ics: ["hash=feb01bab5698a11dd87416724c7a89e3"], + res: [$image_ids[0]], + path: ["no_tags"], + ); + $this->assert_BSQ( + ics: ["id={$image_ids[1]}"], + res: [$image_ids[1]], + path: ["no_tags"], + ); + $this->assert_BSQ( + ics: ["filename=screenshot"], + res: [$image_ids[0]], + path: ["no_tags"], + ); + } + + #[Depends('testUpload')] + public function testBSQ_ImgCond_ManyResults($image_ids) + { + $image_ids = $this->testUpload(); + + $this->assert_BSQ( + ics: ["size=640x480"], + res: [$image_ids[1], $image_ids[0]], + path: ["no_tags"], + ); + $this->assert_BSQ( + ics: ["tags=5"], + res: [$image_ids[1], $image_ids[0]], + path: ["no_tags"], + ); + $this->assert_BSQ( + ics: ["ext=jpg"], + res: [$image_ids[1], $image_ids[0]], + path: ["no_tags"], + ); + } + + /* * * * * * * * * * * + * Mixed * + * * * * * * * * * * */ + #[Depends('testUpload')] + public function testBSQ_TagCondWithImgCond($image_ids) + { + $image_ids = $this->testUpload(); + // multiple tags, many results + $this->assert_BSQ( + tcs: ["computer"], + ics: ["size=640x480"], + res: [$image_ids[1], $image_ids[0]], + path: ["general", "some_positives"], + ); + } +} diff --git a/ext/artists/main.php b/ext/artists/main.php index 51be763be..05eba47ca 100644 --- a/ext/artists/main.php +++ b/ext/artists/main.php @@ -209,7 +209,7 @@ public function onPageRequest(PageRequestEvent $event) $userIsLogged = !$user->is_anonymous(); $userIsAdmin = $user->can(Permissions::ARTISTS_ADMIN); - $images = Image::find_images(limit: 4, tags: Tag::explode($artist['name'])); + $images = Search::find_images(limit: 4, tags: Tag::explode($artist['name'])); $this->theme->show_artist($artist, $aliases, $members, $urls, $images, $userIsLogged, $userIsAdmin); /* diff --git a/ext/bulk_actions/main.php b/ext/bulk_actions/main.php index ee235eed8..e296893dc 100644 --- a/ext/bulk_actions/main.php +++ b/ext/bulk_actions/main.php @@ -214,7 +214,7 @@ private function yield_items(array $data): \Generator private function yield_search_results(string $query): \Generator { $tags = Tag::explode($query); - return Image::find_images_iterable(0, null, $tags); + return Search::find_images_iterable(0, null, $tags); } private function sort_blocks($a, $b) diff --git a/ext/danbooru_api/main.php b/ext/danbooru_api/main.php index 569de860c..1104f04f3 100644 --- a/ext/danbooru_api/main.php +++ b/ext/danbooru_api/main.php @@ -199,8 +199,8 @@ private function api_find_posts(): HTMLElement $tags = array_filter($tags, static function ($element) { return $element !== "*"; }); - $count = Image::count_images($tags); - $results = Image::find_images(max($start, 0), min($limit, 100), $tags); + $count = Search::count_images($tags); + $results = Search::find_images(max($start, 0), min($limit, 100), $tags); } // Now we have the array $results filled with Image objects diff --git a/ext/favorites/main.php b/ext/favorites/main.php index af2db5bc2..75e3584a5 100644 --- a/ext/favorites/main.php +++ b/ext/favorites/main.php @@ -72,7 +72,7 @@ public function onPageRequest(PageRequestEvent $event) public function onUserPageBuilding(UserPageBuildingEvent $event) { - $i_favorites_count = Image::count_images(["favorited_by={$event->display_user->name}"]); + $i_favorites_count = Search::count_images(["favorited_by={$event->display_user->name}"]); $i_days_old = ((time() - strtotime($event->display_user->join_date)) / 86400) + 1; $h_favorites_rate = sprintf("%.1f", ($i_favorites_count / $i_days_old)); $favorites_link = search_link(["favorited_by={$event->display_user->name}"]); diff --git a/ext/home/main.php b/ext/home/main.php index aa214ddff..081eb0509 100644 --- a/ext/home/main.php +++ b/ext/home/main.php @@ -55,7 +55,7 @@ private function get_body(): string $num_comma = ""; $counter_text = ""; if ($counter_dir != 'none') { - $total = Image::count_images(); + $total = Search::count_images(); $num_comma = number_format($total); if ($counter_dir != 'text-only') { diff --git a/ext/image/main.php b/ext/image/main.php index 76cac8615..16c846e75 100644 --- a/ext/image/main.php +++ b/ext/image/main.php @@ -263,7 +263,7 @@ public function onImageReplace(ImageReplaceEvent $event) public function onUserPageBuilding(UserPageBuildingEvent $event) { $u_name = url_escape($event->display_user->name); - $i_image_count = Image::count_images(["user={$event->display_user->name}"]); + $i_image_count = Search::count_images(["user={$event->display_user->name}"]); $i_days_old = ((time() - strtotime($event->display_user->join_date)) / 86400) + 1; $h_image_rate = sprintf("%.1f", ($i_image_count / $i_days_old)); $images_link = search_link(["user=$u_name"]); diff --git a/ext/index/main.php b/ext/index/main.php index d8c57900a..ea776f227 100644 --- a/ext/index/main.php +++ b/ext/index/main.php @@ -22,7 +22,7 @@ public function onInitExt(InitExtEvent $event) public function onPageRequest(PageRequestEvent $event) { - global $cache, $page, $user; + global $cache, $config, $page, $user; if ($event->page_matches("post/list")) { if (isset($_GET['search'])) { $page->set_mode(PageMode::REDIRECT); @@ -67,7 +67,7 @@ public function onPageRequest(PageRequestEvent $event) return; } - $total_pages = Image::count_pages($search_terms); + $total_pages = (int)ceil(Search::count_images($search_terms) / $config->get_int(IndexConfig::IMAGES)); $images = []; if (SPEED_HAX && $total_pages > $fast_page_limit && !$user->can("big_search")) { @@ -79,14 +79,14 @@ public function onPageRequest(PageRequestEvent $event) // extra caching for the first few post/list pages $images = $cache->get("post-list:$page_number"); if (is_null($images)) { - $images = Image::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); + $images = Search::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); $cache->set("post-list:$page_number", $images, 60); } } } if (!$images) { - $images = Image::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); + $images = Search::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); } } catch (PermissionDeniedException $pde) { $this->theme->display_error(403, "Permission denied", $pde->error); @@ -156,7 +156,7 @@ public function onCommand(CommandEvent $event) } if ($event->cmd == "search") { $query = count($event->args) > 0 ? Tag::explode($event->args[0]) : []; - $items = Image::find_images(limit: 1000, tags: $query); + $items = Search::find_images(limit: 1000, tags: $query); foreach ($items as $item) { print("{$item->hash}\n"); } diff --git a/ext/index/test.php b/ext/index/test.php index 127b306ac..ab6d16f9b 100644 --- a/ext/index/test.php +++ b/ext/index/test.php @@ -49,191 +49,6 @@ public function testIndexPage() $this->assert_response(200); } - public function testWeirdTags() - { - $this->log_in_as_user(); - $image_id_1 = $this->post_image("tests/pbx_screenshot.jpg", "question? colon:thing exclamation!"); - $image_id_2 = $this->post_image("tests/bedroom_workshop.jpg", "question. colon_thing exclamation%"); - - $this->assert_search_results(["question?"], [$image_id_1]); - $this->assert_search_results(["question."], [$image_id_2]); - $this->assert_search_results(["colon:thing"], [$image_id_1]); - $this->assert_search_results(["colon_thing"], [$image_id_2]); - $this->assert_search_results(["exclamation!"], [$image_id_1]); - $this->assert_search_results(["exclamation%"], [$image_id_2]); - } - - // base case - public function testUpload(): array - { - $this->log_in_as_user(); - $image_id_1 = $this->post_image("tests/pbx_screenshot.jpg", "thing computer screenshot pbx phone"); - $image_id_2 = $this->post_image("tests/bedroom_workshop.jpg", "thing computer computing bedroom workshop"); - $this->log_out(); - - # make sure both uploads were ok - $this->assertTrue($image_id_1 > 0); - $this->assertTrue($image_id_2 > 0); - - return [$image_id_1, $image_id_2]; - } - - /* * * * * * * * * * * - * Tag Search * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testTagSearchNoResults($image_ids) - { - $this->testUpload(); - $this->assert_search_results(["maumaumau"], []); - } - - #[Depends('testUpload')] - public function testTagSearchOneResult($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["pbx"], [$image_ids[0]]); - } - - #[Depends('testUpload')] - public function testTagSearchManyResults($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["computer"], [$image_ids[1], $image_ids[0]]); - } - - /* * * * * * * * * * * - * Multi-Tag Search * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testMultiTagSearchNoResults($image_ids) - { - $this->testUpload(); - # multiple tags, one of which doesn't exist - # (test the "one tag doesn't exist = no hits" path) - $this->assert_search_results(["computer", "asdfasdfwaffle"], []); - } - - #[Depends('testUpload')] - public function testMultiTagSearchOneResult($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["computer", "screenshot"], [$image_ids[0]]); - } - - #[Depends('testUpload')] - public function testMultiTagSearchManyResults($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["computer", "thing"], [$image_ids[1], $image_ids[0]]); - } - - /* * * * * * * * * * * - * Meta Search * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testMetaSearchNoResults($image_ids) - { - $this->testUpload(); - $this->assert_search_results(["hash=1234567890"], []); - $this->assert_search_results(["ratio=42:12345"], []); - } - - #[Depends('testUpload')] - public function testMetaSearchOneResult($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["hash=feb01bab5698a11dd87416724c7a89e3"], [$image_ids[0]]); - $this->assert_search_results(["md5=feb01bab5698a11dd87416724c7a89e3"], [$image_ids[0]]); - $this->assert_search_results(["id={$image_ids[1]}"], [$image_ids[1]]); - $this->assert_search_results(["filename=screenshot"], [$image_ids[0]]); - } - - #[Depends('testUpload')] - public function testMetaSearchManyResults($image_ids) - { - $image_ids = $this->testUpload(); - $this->assert_search_results(["size=640x480"], [$image_ids[1], $image_ids[0]]); - $this->assert_search_results(["tags=5"], [$image_ids[1], $image_ids[0]]); - $this->assert_search_results(["ext=jpg"], [$image_ids[1], $image_ids[0]]); - } - - /* * * * * * * * * * * - * Wildcards * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testWildSearchNoResults($image_ids) - { - $this->testUpload(); - $this->assert_search_results(["asdfasdf*"], []); - } - - #[Depends('testUpload')] - public function testWildSearchOneResult($image_ids) - { - $image_ids = $this->testUpload(); - // Only the first image matches both the wildcard and the tag. - // This checks for https://github.com/shish/shimmie2/issues/547 - $this->assert_search_results(["comp*", "screenshot"], [$image_ids[0]]); - } - - #[Depends('testUpload')] - public function testWildSearchManyResultsSimple($image_ids) - { - $image_ids = $this->testUpload(); - // two images match comp* - one matches it once, one matches it twice - $this->assert_search_results(["comp*"], [$image_ids[1], $image_ids[0]]); - } - - #[Depends('testUpload')] - public function testWildSearchManyResultsComplex($image_ids) - { - $image_ids = $this->testUpload(); - // same thing, but with the complex branch - $this->assert_search_results(["comp*", "-asdf"], [$image_ids[1], $image_ids[0]]); - } - - /* * * * * * * * * * * - * Mixed * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testMixedSearchTagMeta($image_ids) - { - $image_ids = $this->testUpload(); - // multiple tags, many results - $this->assert_search_results(["computer", "size=640x480"], [$image_ids[1], $image_ids[0]]); - } - - /* * * * * * * * * * * - * Negative * - * * * * * * * * * * */ - #[Depends('testUpload')] - public function testSubtractFromSearch($image_ids) - { - $image_ids = $this->testUpload(); - - // negative tag, should have one result - $this->assert_search_results(["computer", "-pbx"], [$image_ids[1]]); - - // removing something that doesn't exist should have no effect - $this->assert_search_results(["computer", "-not_a_tag"], [$image_ids[1], $image_ids[0]]); - } - - #[Depends('testUpload')] - public function testSubtractFromDefault($image_ids) - { - $image_ids = $this->testUpload(); - - // negative tag alone, should work - $this->assert_search_results(["-pbx"], [$image_ids[1]]); - - // negative that doesn't exist - $this->assert_search_results(["-not_a_tag"], [$image_ids[1], $image_ids[0]]); - - // multiple negative tags that don't exist - $this->assert_search_results(["-not_a_tag", "-also_not_a_tag"], [$image_ids[1], $image_ids[0]]); - } - // This isn't really an index thing, we just want to test this from // SOMEWHERE because the default theme doesn't use them. public function test_nav() diff --git a/ext/numeric_score/main.php b/ext/numeric_score/main.php index a42a25f69..61b0b0c30 100644 --- a/ext/numeric_score/main.php +++ b/ext/numeric_score/main.php @@ -119,9 +119,9 @@ public function onUserPageBuilding(UserPageBuildingEvent $event) $this->theme->get_nuller($event->display_user); } - $n_up = Image::count_images(["upvoted_by={$event->display_user->name}"]); + $n_up = Search::count_images(["upvoted_by={$event->display_user->name}"]); $link_up = search_link(["upvoted_by={$event->display_user->name}"]); - $n_down = Image::count_images(["downvoted_by={$event->display_user->name}"]); + $n_down = Search::count_images(["downvoted_by={$event->display_user->name}"]); $link_down = search_link(["downvoted_by={$event->display_user->name}]"]); $event->add_stats("$n_up Upvotes / $n_down Downvotes"); } diff --git a/ext/ouroboros_api/main.php b/ext/ouroboros_api/main.php index 29b78993b..70e9758e3 100644 --- a/ext/ouroboros_api/main.php +++ b/ext/ouroboros_api/main.php @@ -420,7 +420,7 @@ protected function postShow(int $id = null) protected function postIndex(int $limit, int $page, array $tags) { $start = ($page - 1) * $limit; - $results = Image::find_images(max($start, 0), min($limit, 100), $tags); + $results = Search::find_images(max($start, 0), min($limit, 100), $tags); $posts = []; foreach ($results as $img) { if (!is_object($img)) { diff --git a/ext/pools/main.php b/ext/pools/main.php index 226949564..e9f38cc6c 100644 --- a/ext/pools/main.php +++ b/ext/pools/main.php @@ -372,7 +372,7 @@ public function onPageRequest(PageRequestEvent $event) break; case "import": if ($this->have_permission($user, $pool)) { - $images = Image::find_images( + $images = Search::find_images( limit: $config->get_int(PoolsConfig::MAX_IMPORT_RESULTS, 1000), tags: Tag::explode($_POST["pool_tag"]) ); diff --git a/ext/rating/main.php b/ext/rating/main.php index b2db0f5f4..9ad774152 100644 --- a/ext/rating/main.php +++ b/ext/rating/main.php @@ -376,7 +376,7 @@ public function onPageRequest(PageRequestEvent $event) } else { $n = 0; while (true) { - $images = Image::find_images($n, 100, Tag::explode($_POST["query"])); + $images = Search::find_images($n, 100, Tag::explode($_POST["query"])); if (count($images) == 0) { break; } diff --git a/ext/regen_thumb/main.php b/ext/regen_thumb/main.php index 227d23cdb..b5352cd14 100644 --- a/ext/regen_thumb/main.php +++ b/ext/regen_thumb/main.php @@ -30,7 +30,7 @@ public function onPageRequest(PageRequestEvent $event) } if ($event->page_matches("regen_thumb/mass") && $user->can(Permissions::DELETE_IMAGE) && isset($_POST['tags'])) { $tags = Tag::explode(strtolower($_POST['tags']), false); - $images = Image::find_images(limit: 10000, tags: $tags); + $images = Search::find_images(limit: 10000, tags: $tags); foreach ($images as $image) { $this->regenerate_thumbnail($image); diff --git a/ext/rss_images/main.php b/ext/rss_images/main.php index f1e015835..2bc29c185 100644 --- a/ext/rss_images/main.php +++ b/ext/rss_images/main.php @@ -31,7 +31,7 @@ public function onPageRequest(PageRequestEvent $event) return; } try { - $images = Image::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); + $images = Search::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); $this->do_rss($images, $search_terms, $page_number); } catch (SearchTermParseException $stpe) { $this->theme->display_error(400, "Search parse error", $stpe->error); diff --git a/ext/rule34/main.php b/ext/rule34/main.php index 2ec39607c..ee3da462b 100644 --- a/ext/rule34/main.php +++ b/ext/rule34/main.php @@ -84,7 +84,7 @@ public function onCommand(CommandEvent $event) { global $cache; if ($event->cmd == "wipe-thumb-cache") { - foreach (Image::find_images_iterable(0, null, Tag::explode($event->args[0])) as $image) { + foreach (Search::find_images_iterable(0, null, Tag::explode($event->args[0])) as $image) { print($image->id . "\n"); $cache->delete("thumb-block:{$image->id}"); } diff --git a/ext/shimmie_api/main.php b/ext/shimmie_api/main.php index cf590df2e..9fe638eed 100644 --- a/ext/shimmie_api/main.php +++ b/ext/shimmie_api/main.php @@ -70,7 +70,7 @@ public function onPageRequest(PageRequestEvent $event) $search_terms = $event->get_search_terms(); $page_number = $event->get_page_number(); $page_size = $event->get_page_size(); - $images = Image::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); + $images = Search::find_images(($page_number - 1) * $page_size, $page_size, $search_terms); $safe_images = []; foreach ($images as $image) { $image->get_tag_array(); @@ -131,7 +131,7 @@ private function api_get_user(string $type, string $query): array for ($i = 0; $i < 4; $i++) { unset($all[$i]); } - $all['uploadcount'] = Image::count_images(["user_id=" . $all['id']]); + $all['uploadcount'] = Search::count_images(["user_id=" . $all['id']]); $all['commentcount'] = $database->get_one( "SELECT COUNT(*) AS count FROM comments WHERE owner_id=:owner_id", ["owner_id" => $all['id']] diff --git a/ext/sitemap/main.php b/ext/sitemap/main.php index c3fefe18a..0d08142b4 100644 --- a/ext/sitemap/main.php +++ b/ext/sitemap/main.php @@ -42,7 +42,7 @@ public function onSetupBuilding(SetupBuildingEvent $event) private function handle_smaller_sitemap() { /* --- Add latest images to sitemap with higher priority --- */ - $latestimages = Image::find_images(limit: 50); + $latestimages = Search::find_images(limit: 50); if (empty($latestimages)) { return; } @@ -85,7 +85,7 @@ private function handle_full_sitemap() $this->add_sitemap_queue($popular_tags, "monthly", "0.9" /* not sure how to deal with date here */); /* --- Add latest images to sitemap with higher priority --- */ - $latestimages = Image::find_images(limit: 50); + $latestimages = Search::find_images(limit: 50); $latestimages_urllist = []; $latest_image = null; foreach ($latestimages as $arrayid => $image) { @@ -107,7 +107,7 @@ private function handle_full_sitemap() $this->add_sitemap_queue($other_tags, "monthly", "0.7" /* not sure how to deal with date here */); /* --- Add all other images to sitemap with lower priority --- */ - $otherimages = Image::find_images(offset: 51, limit: 10000000); + $otherimages = Search::find_images(offset: 51, limit: 10000000); $image = null; foreach ($otherimages as $arrayid => $image) { // create url from image id's diff --git a/ext/tag_edit/main.php b/ext/tag_edit/main.php index 47b0f13cc..3e49ee27b 100644 --- a/ext/tag_edit/main.php +++ b/ext/tag_edit/main.php @@ -300,7 +300,7 @@ private function mass_tag_edit(string $search, string $replace, bool $commit) log_info("tag_edit", "Mass editing tags: '$search' -> '$replace'"); if (count($search_set) == 1 && count($replace_set) == 1) { - $images = Image::find_images(limit: 10, tags: $replace_set); + $images = Search::find_images(limit: 10, tags: $replace_set); if (count($images) == 0) { log_info("tag_edit", "No images found with target tag, doing in-place rename"); $database->execute( @@ -329,7 +329,7 @@ private function mass_tag_edit(string $search, string $replace, bool $commit) $search_forward[] = "id<$last_id"; } - $images = Image::find_images(limit: 100, tags: $search_forward); + $images = Search::find_images(limit: 100, tags: $search_forward); if (count($images) == 0) { break; } @@ -364,7 +364,7 @@ private function mass_source_edit(string $tags, string $source) $search_forward[] = "id<$last_id"; } - $images = Image::find_images(limit: 100, tags: $search_forward); + $images = Search::find_images(limit: 100, tags: $search_forward); if (count($images) == 0) { break; } diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 715838322..808539b4f 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -67,8 +67,12 @@ public function setUp(): void $_tracer->begin($this->name()); $_tracer->begin("setUp"); $class = str_replace("Test", "", get_class($this)); - if (!ExtensionInfo::get_for_extension_class($class)->is_supported()) { - $this->markTestSkipped("$class not supported with this database"); + try { + if (!ExtensionInfo::get_for_extension_class($class)->is_supported()) { + $this->markTestSkipped("$class not supported with this database"); + } + } catch (ExtensionNotFound $e) { + // ignore - this is a core test rather than an extension test } // Set up a clean environment for each test @@ -220,7 +224,7 @@ protected function assert_no_content(string $content): void protected function assert_search_results($tags, $results): void { - $images = Image::find_images(0, null, $tags); + $images = Search::find_images(0, null, $tags); $ids = []; foreach ($images as $image) { $ids[] = $image->id;