Merge pull request #62 from vincentdephily/follow_renames

Follow package moves

CHANGELOG.md

@@ -1,23 +1,30 @@
 # unreleased
 
+Feature release: Binary merges, package moves, emerge arguments, process tree, configurable colors,
+and other goodies.
+
 ## New features
 
 * `log`, `stats` and `predict` now take binary merges into account
   - Track merge time stats separately
   - Display bin merges in a different color
   - The `unknown` config has been split into `unknownc` (compiled) and `unknownb` (binary)
+* `stats` and `predict` now follow package moves
+  - Merge time predictions remain correct after a rename like `sys-devel/llvm` -> `llvm-core/llvm`
+  - `Stats` are aggregated using the latest name
 * `log` and `stat` can now show emerge (r)runs
   - Use `--show=r`/`-sr`/`-sa` to enable it
   - No duration given, as `emerge.log` doesn't provide enough info to make this reliable
 * `--from` and `--to` now accept more values
   - New `command` keyword resolves to the time of the nth emerge command
     (`-fc` is roughly equivalent to qlop's `--lastmerge`)
   - A single span (`day`/`y`/`command`/etc) without a count now means that span with a count of 1
+    (so `-fd` is equivalent to `-1d`)
 * `predict` now displays emerge proces tree instead of just top proces
-  - Bevahvior configurable with `--pdepth`, `--pwidth`
+  - Behavior configurable with `--pdepth`/`-D`/`--pwidth`/`-W`
   - Format is a bit nicer and more colorful
   - `--show=e` renamed `--show=r` (running emerge processes) for consistency
-* Display a placeholder for skipped rows, configurable with `--showskip`
+* Display a placeholder for skipped rows (`--first`/`--last`/`--pdepth`), configurable with `--showskip`
 * Colors are now configurable, to match your terminal's theme
   - Eg `theme = "count:0 duration:1;3;37"` in `emlop.toml` displays counts unstyled and durations in
     bright italic white.
@@ -31,6 +38,7 @@
 * Don't display child emerge processes as root ones
 * Fix off by one upper bound for some cli args
 * Allow alignment of wider columns
+* Fix bright/dim terminal colors for stats counts
 
 # 0.7.1 2024-09-30
 

docs/COMPARISON.md

@@ -131,7 +131,7 @@ estimate the resulting speedup factor.
 | Global ETA format                                 | total time    | total time    | total time, end date |
 | Estimation accuracy                               | ok            | better        | best, configurable   |
 | Recognize binary emerges                          | no            | no            | yes                  |
-| Follow package renames                            | yes           | no            | no                   |
+| Follow package moves                              | no            | no            | yes                  |
 | Query gentoo.linuxhowtos.org for unknown packages | yes           | no            | no                   |
 
 ## Speed

src/commands.rs

@@ -179,6 +179,7 @@ impl ArgKind {
 /// Then we compute the stats per ebuild, and print that.
 pub fn cmd_stats(gc: Conf, sc: ConfStats) -> Result<bool, Error> {
     let hist = get_hist(&gc.logfile, gc.from, gc.to, sc.show, &sc.search, sc.exact)?;
+    let moves = PkgMoves::new(&Mtimedb::new());
     let h = [sc.group.name(), "Logged emerges", "Install/Update", "Unmerge/Clean", "Sync"];
     let mut tblc = Table::new(&gc).margin(1, " ").header(h);
     let h = [sc.group.name(), "Repo", "Syncs", "Total time", "Predict time"];
@@ -237,7 +238,7 @@ pub fn cmd_stats(gc: Conf, sc: ConfStats) -> Result<bool, Error> {
                 *run_args.entry(ArgKind::new(&args)).or_insert(0) += 1;
             },
             Hist::MergeStart { ts, key, .. } => {
-                merge_start.insert(key, (ts, false));
+                merge_start.insert(moves.get(key), (ts, false));
             },
             Hist::MergeStep { kind, key, .. } => {
                 if matches!(kind, MergeStep::MergeBinary) {
@@ -247,9 +248,9 @@ pub fn cmd_stats(gc: Conf, sc: ConfStats) -> Result<bool, Error> {
                 }
             },
             Hist::MergeStop { ts, ref key, .. } => {
-                if let Some((start_ts, bin)) = merge_start.remove(key) {
+                if let Some((start_ts, bin)) = merge_start.remove(moves.get_ref(key)) {
                     let (tc, tb, _) =
-                        pkg_time.entry(p.take_ebuild())
+                        pkg_time.entry(moves.get(p.take_ebuild()))
                                 .or_insert((Times::new(), Times::new(), Times::new()));
                     if bin {
                         tb.insert(ts - start_ts);
@@ -259,12 +260,12 @@ pub fn cmd_stats(gc: Conf, sc: ConfStats) -> Result<bool, Error> {
                 }
             },
             Hist::UnmergeStart { ts, key, .. } => {
-                unmerge_start.insert(key, ts);
+                unmerge_start.insert(moves.get(key), ts);
             },
             Hist::UnmergeStop { ts, ref key, .. } => {
-                if let Some(start_ts) = unmerge_start.remove(key) {
+                if let Some(start_ts) = unmerge_start.remove(moves.get_ref(key)) {
                     let (_, _, times) =
-                        pkg_time.entry(p.take_ebuild())
+                        pkg_time.entry(moves.get(p.take_ebuild()))
                                 .or_insert((Times::new(), Times::new(), Times::new()));
                     times.insert(ts - start_ts);
                 }
@@ -450,23 +451,26 @@ pub fn cmd_predict(gc: Conf, mut sc: ConfPred) -> Result<bool, Error> {
 
     // Parse emerge log.
     let hist = get_hist(&gc.logfile, gc.from, gc.to, Show::m(), &vec![], false)?;
+    let mdb = Mtimedb::new();
+    let moves = PkgMoves::new(&mdb);
     let mut started: BTreeMap<String, (i64, bool)> = BTreeMap::new();
     let mut times: HashMap<(String, bool), Times> = HashMap::new();
     for p in hist {
         match p {
             Hist::MergeStart { ts, key, .. } => {
-                started.insert(key, (ts, false));
+                started.insert(moves.get(key), (ts, false));
             },
             Hist::MergeStep { kind, key, .. } => {
                 if matches!(kind, MergeStep::MergeBinary) {
-                    if let Some((_, bin)) = started.get_mut(&key) {
+                    if let Some((_, bin)) = started.get_mut(moves.get_ref(&key)) {
                         *bin = true;
                     }
                 }
             },
             Hist::MergeStop { ts, ref key, .. } => {
-                if let Some((start_ts, bin)) = started.remove(key.as_str()) {
-                    let timevec = times.entry((p.take_ebuild(), bin)).or_insert(Times::new());
+                if let Some((start_ts, bin)) = started.remove(moves.get_ref(key)) {
+                    let timevec =
+                        times.entry((moves.get(p.take_ebuild()), bin)).or_insert(Times::new());
                     timevec.insert(ts - start_ts);
                 }
             },
@@ -477,7 +481,7 @@ pub fn cmd_predict(gc: Conf, mut sc: ConfPred) -> Result<bool, Error> {
     // Build list of pending merges
     let pkgs: Vec<Pkg> = if std::io::stdin().is_terminal() {
         // From resume list
-        let mut r = get_resume(sc.resume);
+        let mut r = get_resume(sc.resume, &mdb);
         // Plus specific emerge processes
         for p in einfo.pkgs.iter() {
             if !r.contains(p) {

src/parse.rs

@@ -4,7 +4,7 @@ mod history;
 mod proces;
 
 pub use ansi::{Ansi, AnsiStr, Theme};
-pub use current::{get_buildlog, get_emerge, get_pretend, get_resume, Pkg};
+pub use current::{get_buildlog, get_emerge, get_pretend, get_resume, Mtimedb, Pkg, PkgMoves};
 pub use history::{get_hist, Hist, MergeStep};
 #[cfg(test)]
 pub use proces::tests::procs;

src/parse/current.rs

@@ -7,9 +7,11 @@ use log::*;
 use regex::Regex;
 use serde::Deserialize;
 use serde_json::from_reader;
-use std::{fs::File,
+use std::{collections::HashMap,
+          fs::File,
           io::{BufRead, BufReader, Read},
-          path::PathBuf};
+          path::PathBuf,
+          time::Instant};
 
 /// Package name and version
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
@@ -73,35 +75,45 @@ pub fn get_pretend<R: Read>(reader: R, filename: &str) -> Vec<Pkg> {
     out
 }
 
+
 #[derive(Deserialize)]
 struct Resume {
     mergelist: Vec<Vec<String>>,
 }
-#[derive(Deserialize)]
-struct Mtimedb {
+#[derive(Deserialize, Default)]
+pub struct Mtimedb {
     resume: Option<Resume>,
     resume_backup: Option<Resume>,
+    updates: Option<HashMap<String, i64>>,
+}
+impl Mtimedb {
+    pub fn new() -> Self {
+        Self::try_new("/var/cache/edb/mtimedb").unwrap_or_default()
+    }
+    fn try_new(file: &str) -> Option<Self> {
+        let now = Instant::now();
+        let reader = File::open(file).map_err(|e| warn!("Cannot open {file:?}: {e}")).ok()?;
+        let r = from_reader(reader).map_err(|e| warn!("Cannot parse {file:?}: {e}")).ok();
+        debug!("Loaded {file} in {:?}", now.elapsed());
+        r
+    }
 }
 
+
 /// Parse resume list from portage mtimedb
-pub fn get_resume(kind: ResumeKind) -> Vec<Pkg> {
-    let r = get_resume_priv(kind, "/var/cache/edb/mtimedb").unwrap_or_default();
+pub fn get_resume(kind: ResumeKind, db: &Mtimedb) -> Vec<Pkg> {
+    let r = try_get_resume(kind, db).unwrap_or_default();
     debug!("Loaded {kind:?} resume list: {r:?}");
     r
 }
-fn get_resume_priv(kind: ResumeKind, file: &str) -> Option<Vec<Pkg>> {
-    if matches!(kind, ResumeKind::No) {
-        return Some(vec![]);
-    }
-    let reader = File::open(file).map_err(|e| warn!("Cannot open {file:?}: {e}")).ok()?;
-    let db: Mtimedb = from_reader(reader).map_err(|e| warn!("Cannot parse {file:?}: {e}")).ok()?;
+fn try_get_resume(kind: ResumeKind, db: &Mtimedb) -> Option<Vec<Pkg>> {
     let r = match kind {
         ResumeKind::Either | ResumeKind::Auto => {
-            db.resume.filter(|o| !o.mergelist.is_empty()).or(db.resume_backup)?
+            db.resume.as_ref().filter(|o| !o.mergelist.is_empty()).or(db.resume_backup.as_ref())?
         },
-        ResumeKind::Main => db.resume?,
-        ResumeKind::Backup => db.resume_backup?,
-        ResumeKind::No => unreachable!(),
+        ResumeKind::Main => db.resume.as_ref()?,
+        ResumeKind::Backup => db.resume_backup.as_ref()?,
+        ResumeKind::No => return Some(vec![]),
     };
     Some(r.mergelist
           .iter()
@@ -112,6 +124,79 @@ fn get_resume_priv(kind: ResumeKind, file: &str) -> Option<Vec<Pkg>> {
 }
 
 
+pub struct PkgMoves(HashMap<String, String>);
+impl PkgMoves {
+    /// Parse package moves using file list from portagedb
+    pub fn new(db: &Mtimedb) -> Self {
+        let r = Self::try_new(db).unwrap_or_default();
+        trace!("Package moves: {r:?}");
+        Self(r)
+    }
+
+    pub fn get(&self, key: String) -> String {
+        self.0.get(&key).cloned().unwrap_or(key)
+    }
+
+    pub fn get_ref<'a>(&'a self, key: &'a String) -> &'a String {
+        self.0.get(key).unwrap_or(key)
+    }
+
+    fn try_new(db: &Mtimedb) -> Option<HashMap<String, String>> {
+        let now = Instant::now();
+        // Sort the files in reverse chronological order (compare year, then quarter)
+        let mut files: Vec<_> = db.updates.as_ref()?.keys().collect();
+        files.sort_by(|a, b| match (a.rsplit_once('/'), b.rsplit_once('/')) {
+                 (Some((_, a)), Some((_, b))) if a.len() == 7 && b.len() == 7 => {
+                     match a[3..].cmp(&b[3..]) {
+                         std::cmp::Ordering::Equal => a[..3].cmp(&b[..3]),
+                         o => o,
+                     }.reverse()
+                 },
+                 _ => {
+                     warn!("Using default sort for {a} <> {b}");
+                     a.cmp(b)
+                 },
+             });
+        // Read each file to populate the result map
+        let mut moves = HashMap::new();
+        for f in &files {
+            Self::parse(&mut moves, f);
+        }
+        debug!("Loaded {} package moves from {} files in {:?}",
+               moves.len(),
+               files.len(),
+               now.elapsed());
+        Some(moves)
+    }
+
+    fn parse(moves: &mut HashMap<String, String>, file: &str) -> Option<()> {
+        trace!("Parsing {file}");
+        let f = File::open(file).map_err(|e| warn!("Cannot open {file:?}: {e}")).ok()?;
+        for line in
+            BufReader::new(f).lines().map_while(Result::ok).filter(|l| l.starts_with("move "))
+        {
+            if let Some((from, to)) = line[5..].split_once(' ') {
+                // Portage rewrites each repo's update files so that entries point directly to the
+                // final name, but there can still be cross-repo chains, which we untangle
+                // here. Assumes we're parsing files newest-first.
+                if let Some(to_final) = moves.get(to) {
+                    if from != to_final {
+                        trace!("Using move {from} -> {to_final} instead -> {to} in {file}");
+                        moves.insert(from.to_owned(), to_final.clone());
+                    } else {
+                        trace!("Ignoring move {from} -> {to} in {file}");
+                    }
+                } else {
+                    // TODO: MSRV 1.?? try_insert https://github.com/rust-lang/rust/issues/82766
+                    moves.entry(from.to_owned()).or_insert_with(|| to.to_owned());
+                }
+            }
+        }
+        Some(())
+    }
+}
+
+
 /// Retrieve summary info from the build log
 pub fn get_buildlog(pkg: &Pkg, portdirs: &Vec<PathBuf>) -> Option<String> {
     for portdir in portdirs {
@@ -231,9 +316,9 @@ mod tests {
 
     /// Check that `get_resume()` has the expected output
     fn check_resume(kind: ResumeKind, file: &str, expect: Option<&[(&str, bool)]>) {
-        let expect_pkg =
+        let expect_pkg: Option<Vec<Pkg>> =
             expect.map(|o| o.into_iter().map(|(s, b)| Pkg::try_new(s, *b).unwrap()).collect());
-        let res = get_resume_priv(kind, &format!("tests/{file}"));
+        let res = Mtimedb::try_new(&format!("tests/{file}")).and_then(|m| try_get_resume(kind, &m));
         assert_eq!(expect_pkg, res, "Mismatch for {file}");
     }
 
@@ -291,4 +376,35 @@ mod tests {
         let einfo = get_emerge(&procs);
         assert_eq!(einfo.roots, vec![1, 5]);
     }
+
+    #[test]
+    fn pkgmoves() {
+        // It's interesting to run this test with RUST_LOG=trace. Expect:
+        // * "Cannot open tests/notfound: No such file or directory"
+        // * "Using default sort ..." (depending on random hashmap seed)
+        // * "Using move chain/v1 -> chain/v3 instead -> chain/v2 in tests/4Q-2022"
+        // * "Ignoring move loop/final -> loop/from in tests/4Q-2022"
+        let _ = env_logger::try_init();
+        let moves = PkgMoves::new(&Mtimedb::try_new("tests/mtimedb.updates").unwrap());
+        for (have, want, why) in
+            [// Basic cases
+             ("app-doc/doxygen", "app-text/doxygen", "simple move in 2024"),
+             ("x11-libs/libva", "media-libs/libva", "simple move in 2022"),
+             ("notmoved", "notmoved", "unknown string should return original string"),
+             ("dev-haskell/extra", "dev-haskell/extra", "slotmoves should be ignored"),
+             // Multi-moves where portage updated the old file
+             ("dev-util/lldb", "llvm-core/lldb", "1st lldb rename"),
+             ("dev-debug/lldb", "llvm-core/lldb", "2nd lldb rename"),
+             // Weird cases
+             ("duplicate/bar", "foo/bar", "duplicate update should prefer newest (no trace)"),
+             ("conflict/foo", "foo/2024", "conflicting update should prefer newest (no trace)"),
+             ("loop/from", "loop/final", "loops should prefer newest (trace \"ignore move...\")"),
+             ("chain/v2", "chain/v3", "chain from new should be taken as-is (no trace)"),
+             ("chain/v1",
+              "chain/v3",
+              "chain from old should point to new (trace \"using move...\")")]
+        {
+            assert_eq!(moves.get(String::from(have)), String::from(want), "{why}");
+        }
+    }
 }