released 4.0.2

performance improvement tweaks

README.md

@@ -4108,8 +4108,10 @@ in markdown:
            -c, --count
                   Only a count of selected lines is written to standard output.  If
                   -o or -u is specified, counts the number of patterns matched.  If
-                  -v is specified, counts the number of non-matching lines.  If
-                  --tree is specified, outputs directories in a tree-like format.
+                  -v is specified, counts the number of non-matching lines.  If -m1,
+                  (with a comma or --min-count=1) is specified, counts only matching
+                  files without outputting zero matches.  If --tree is specified,
+                  outputs directories in a tree-like format.
 
            --color[=WHEN], --colour[=WHEN]
                   Mark up the matching text with the expression stored in the
@@ -5372,7 +5374,7 @@ in markdown:
 
 
 
-    ugrep 4.0.0                      August 18, 2023                        UGREP(1)
+    ugrep 4.0.2                      August 22, 2023                        UGREP(1)
 
 🔝 [Back to table of contents](#toc)
 

lib/matcher.cpp

@@ -1290,7 +1290,7 @@ bool Matcher::advance()
           break;
       }
     }
-    else if (pat_->pin_ > 8 && pat_->pin_ <= 16)
+    else if (pat_->pin_ == 16)
     {
       size_t lcp = pat_->lcp_;
       size_t lcs = pat_->lcs_;
@@ -2566,7 +2566,7 @@ bool Matcher::advance()
       }
     }
 #endif
-    if (pat_->npy_ < 16)
+    if (min >= 4 || pat_->npy_ < 16 || (min >= 2 && pat_->npy_ >= 56))
     {
       if (min >= 4)
       {
@@ -2730,64 +2730,32 @@ bool Matcher::advance()
         }
       }
     }
-    if (min >= 4)
+    while (true)
     {
-      while (true)
+      const char *s = buf_ + loc;
+      const char *e = buf_ + end_ - 6;
+      bool f = true;
+      while (s < e &&
+          (f = (Pattern::predict_match(pma, s) &&
+                Pattern::predict_match(pma, ++s) &&
+                Pattern::predict_match(pma, ++s) &&
+                Pattern::predict_match(pma, ++s))))
       {
-        const char *s = buf_ + loc;
-        const char *e = buf_ + end_ - min;
-        bool f = true;
-        while (s < e &&
-            (f = (!Pattern::predict_match(pmh, s, min) &&
-                  !Pattern::predict_match(pmh, ++s, min))))
-        {
-          ++s;
-        }
-        loc = s - buf_;
-        if (!f)
-        {
-          set_current(loc);
-          return true;
-        }
-        set_current_match(loc - 1);
-        (void)peek_more();
-        loc = cur_ + 1;
-        if (loc + min >= end_)
-        {
-          set_current(loc);
-          return loc + min <= end_;
-        }
+        ++s;
       }
-    }
-    else
-    {
-      while (true)
+      loc = s - buf_;
+      if (!f)
       {
-        const char *s = buf_ + loc;
-        const char *e = buf_ + end_ - 6;
-        bool f = true;
-        while (s < e &&
-            (f = (Pattern::predict_match(pma, s) &&
-                  Pattern::predict_match(pma, ++s) &&
-                  Pattern::predict_match(pma, ++s) &&
-                  Pattern::predict_match(pma, ++s))))
-        {
-          ++s;
-        }
-        loc = s - buf_;
-        if (!f)
-        {
-          set_current(loc);
-          return true;
-        }
-        set_current_match(loc - 1);
-        (void)peek_more();
-        loc = cur_ + 1;
-        if (loc + 6 >= end_)
-        {
-          set_current(loc);
-          return loc + min <= end_;
-        }
+        set_current(loc);
+        return true;
+      }
+      set_current_match(loc - 1);
+      (void)peek_more();
+      loc = cur_ + 1;
+      if (loc + 6 >= end_)
+      {
+        set_current(loc);
+        return loc + min <= end_;
       }
     }
   }

lib/pattern.cpp

@@ -291,7 +291,8 @@ void Pattern::init(const char *options, const uint8_t *pred)
       npy_ = 0;
     // needle count and frequency thresholds to enable needle-based search
     uint16_t pinmax = 8;
-    uint8_t freqmax = 251;
+    uint8_t freqmax1 = 91; // one position
+    uint8_t freqmax2 = 251; // two positions
 #if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2)
     if (have_HW_AVX512BW() || have_HW_AVX2())
       pinmax = 16;
@@ -310,55 +311,64 @@ void Pattern::init(const char *options, const uint8_t *pred)
     lcs_ = 0;
     uint16_t nlcp = 65535; // max and undefined
     uint16_t nlcs = 65535; // max and undefined
+    uint16_t freqsum = 0;
     uint8_t freqlcp = 255; // max
     uint8_t freqlcs = 255; // max
     for (uint16_t k = 0; k < min_; ++k)
     {
       Pred mask = 1 << k;
       uint16_t n = 0;
-      uint8_t freq = 0;
+      uint16_t sum = 0;
+      uint8_t max = 0;
       // at position k count the matching characters and find the max character frequency
       for (uint16_t i = 0; i < 256; ++i)
       {
         if ((bit_[i] & mask) == 0)
         {
           ++n;
-          if (frequency(static_cast<uint8_t>(i)) > freq)
-            freq = frequency(static_cast<uint8_t>(i));
+          uint8_t freq = frequency(static_cast<uint8_t>(i));
+          sum += freq;
+          if (freq > max)
+            max = freq;
         }
       }
       if (n <= pinmax)
       {
         // pick the fewest and rarest (least frequently occurring) needles to search
-        if (freq < freqlcp || (n < nlcp && freq == freqlcp))
+        if (max < freqlcp || (n < nlcp && max == freqlcp))
         {
           lcs_ = lcp_;
           nlcs = nlcp;
           freqlcs = freqlcp;
           lcp_ = static_cast<uint8_t>(k);
           nlcp = n;
-          freqlcp = freq;
+          freqsum = sum;
+          freqlcp = max;
         }
-        else if (n < nlcs || (n == nlcs && freq < freqlcs))
+        else if (n < nlcs ||
+            (n == nlcs &&
+             (max < freqlcs ||
+              abs(static_cast<int>(lcp_) - static_cast<int>(lcs_)) < abs(static_cast<int>(lcp_) - static_cast<int>(k)))))
         {
           lcs_ = static_cast<uint8_t>(k);
           nlcs = n;
-          freqlcs = freq;
+          freqlcs = max;
         }
       }
     }
-    // only one position to pin
-    if (min_ == 1)
+    // one position to pin: make lcp and lcs equal (compared and optimized later)
+    if (min_ == 1 || ((freqsum <= freqlcp || nlcs == 65535) && freqsum <= freqmax1))
     {
       nlcs = nlcp;
       lcs_ = lcp_;
     }
     // number of needles required
     uint16_t n = nlcp > nlcs ? nlcp : nlcs;
+    DBGLOG("min=%zu lcp=%hu(%hu) pin=%hu nlcp=%hu(%hu) freq=%hu(%hu) freqsum=%hu npy=%zu", min_, lcp_, lcs_, n, nlcp, nlcs, freqlcp, freqlcs, freqsum, npy_);
     // determine if a needle-based search is worthwhile, below or meeting the thresholds
-    if (n <= pinmax && freqlcp <= freqmax)
+    if (n <= pinmax && freqlcp <= freqmax2)
     {
-      // bridge the gap from 9 to 16
+      // bridge the gap from 9 to 16 to handle 9 to 16 combined
       if (n > 8)
         n = 16;
       uint16_t j = 0, k = n;

man/ugrep.1

@@ -1,4 +1,4 @@
-.TH UGREP "1" "August 18, 2023" "ugrep 4.0.0" "User Commands"
+.TH UGREP "1" "August 22, 2023" "ugrep 4.0.2" "User Commands"
 .SH NAME
 \fBugrep\fR, \fBug\fR -- file pattern searcher
 .SH SYNOPSIS
@@ -213,7 +213,9 @@ the match.  See also options \fB\-A\fR, \fB\-B\fR and \fB\-y\fR.
 Only a count of selected lines is written to standard output.
 If \fB\-o\fR or \fB\-u\fR is specified, counts the number of patterns matched.
 If \fB\-v\fR is specified, counts the number of non\-matching lines.  If
-\fB\-\-tree\fR is specified, outputs directories in a tree\-like format.
+\fB\-m\fR1, (with a comma or \fB\-\-min\-count\fR=1) is specified, counts only
+matching files without outputting zero matches.  If \fB\-\-tree\fR is
+specified, outputs directories in a tree\-like format.
 .TP
 \fB\-\-color\fR[=\fIWHEN\fR], \fB\-\-colour\fR[=\fIWHEN\fR]
 Mark up the matching text with the expression stored in the

src/ugrep.cpp

@@ -7732,24 +7732,28 @@ void ugrep()
       flag_invert_match = false;
     }
 
-    // -X (--hex) and --match: also output terminating \n in hexdumps (set LineMatcher option A)
-    if (flag_hex)
-      matcher_options.push_back('A');
+    // --match and -v matches nothing, but -y and also -c should still list files when --min-count > 0
+    if (!flag_invert_match || flag_any_line || (flag_count && flag_min_count == 0))
+    {
+      // -X (--hex) and --match: also output terminating \n in hexdumps (set LineMatcher option A)
+      if (flag_hex)
+        matcher_options.push_back('A');
 
-    // --match: match lines with the RE/flex Line Matcher
-    reflex::LineMatcher matcher(reflex::Input(), matcher_options.c_str());
+      // --match: match lines with the RE/flex Line Matcher
+      reflex::LineMatcher matcher(reflex::Input(), matcher_options.c_str());
 
-    if (threads > 1)
-    {
-      GrepMaster grep(output, &matcher, NULL);
-      grep.ugrep();
-    }
-    else
-    {
-      Grep grep(output, &matcher, NULL);
-      set_grep_handle(&grep);
-      grep.ugrep();
-      clear_grep_handle();
+      if (threads > 1)
+      {
+        GrepMaster grep(output, &matcher, NULL);
+        grep.ugrep();
+      }
+      else
+      {
+        Grep grep(output, &matcher, NULL);
+        set_grep_handle(&grep);
+        grep.ugrep();
+        clear_grep_handle();
+      }
     }
   }
   else if (flag_perl_regexp)
@@ -9194,7 +9198,7 @@ void Grep::search(const char *pathname, uint16_t cost)
   if (pathname == LABEL_STANDARD_INPUT)
     pathname = flag_label;
 
-  bool colorize = flag_apply_color || flag_tag != NULL;
+  bool colorize = flag_apply_color || flag_replace != NULL || flag_tag != NULL;
   bool matched = false;
 
   // -z: loop over extracted archive parts, when applicable
@@ -9233,10 +9237,6 @@ void Grep::search(const char *pathname, uint16_t cost)
       {
         // option -q, -l, or -L
 
-        // --match and -v matches nothing
-        if (flag_match && flag_invert_match)
-          goto exit_search;
-
         // --format: whether to out.acquire() early before Stats::found_part()
         bool acquire = flag_format != NULL && (flag_format_open != NULL || flag_format_close != NULL);
 
@@ -9395,6 +9395,13 @@ void Grep::search(const char *pathname, uint16_t cost)
                   break;
 
                 lineno = current_lineno;
+
+#if WITH_SKIP_NEWLINE // this is only valid if no \n is part of the patterns
+                // if the match does not span more than one line, then skip to end of the line (we count matching lines)
+                if (matcher->lines() == 1)
+                  if (!matcher->skip('\n'))
+                    break;
+#endif
               }
             }
 
@@ -10446,6 +10453,14 @@ void Grep::search(const char *pathname, uint16_t cost)
                 restline_last = matcher->last();
               }
             }
+
+#if WITH_SKIP_NEWLINE // this is only valid if no \n is part of the patterns
+            // no -u and no colors: if the match does not span more than one line, then skip to end of the line
+            if (!flag_ungroup && !colorize)
+              if (matcher->lines() == 1)
+                if (!matcher->skip('\n'))
+                  break;
+#endif
           }
           else
           {

src/ugrep.hpp

@@ -38,7 +38,7 @@
 #define UGREP_HPP
 
 // ugrep version
-#define UGREP_VERSION "4.0.1"
+#define UGREP_VERSION "4.0.2"
 
 // disable mmap because mmap is almost always slower than the file reading speed improvements since 3.0.0
 #define WITH_NO_MMAP