Ensure early installation of critical built in packages

init.el

@@ -148,80 +148,81 @@ melpa-stable.")
   (message "Loading tapped before-init file: %s" tapped-file)
   (load (file-name-sans-extension tapped-file)))
 
+(eval-and-compile
+  (load (file-name-concat (locate-user-emacs-file "modules") "init-require")))
 
 ;;; Packages from Melpa
 ;; Use M-x `package-refresh-contents' to update the cache.
 ;; Use M-x `package-list-package' to load and display the list of packages,
 ;; then press I to mark for installation and X to execute (it's like `dired').
 
 ;; Initialize the package system
-(require 'seq)
 (require 'package)
 
 (add-to-list 'package-archives
              (cons "melpa" exordium-melpa-package-repo) t)
 
-(when (seq-filter (lambda (pkg)
-                    (string= "melpa-stable" (cdr pkg)))
-                  exordium-extra-pinned)
+(when (let (match)
+        (mapc (lambda (pkg)
+                (when (equal "melpa-stable" (cdr pkg))
+                  (setq match t)))
+              exordium-extra-pinned)
+        match)
   (add-to-list 'package-archives
                (cons "melpa-stable" exordium-stable-melpa-package-repo) t))
 
-
 (setq package-user-dir
       (locate-user-emacs-file (concat "elpa-" emacs-version)))
 
-(when (fboundp 'native-comp-available-p)
+(when (fboundp 'native-comp-available-p) ;; check needed before Emacs-28
   (setq package-native-compile (native-comp-available-p)))
 
 (package-initialize)
-
-;; Load the packages we need if they are not installed already
-(let ((package-pinned-packages (append
-                                '((use-package             . "gnu")
-                                  (diminish                . "gnu")
-                                  (bind-key                . "gnu"))
-                                exordium-extra-pinned))
-      has-refreshed)
-  (mapc (lambda (pkg)
-          (unless (package-installed-p pkg)
-            (unless has-refreshed
-              (message "Refreshing package database...")
-              (package-refresh-contents)
-              (setq has-refreshed t)
-              (message "Done."))
-            (package-install pkg)))
-        (append (mapcar #'car package-pinned-packages)
-                exordium-extra-packages)))
+
+(unless (package-installed-p 'use-package)  ;; Before Emacs-29
+  (package-refresh-contents)
+  (package-install 'use-package))
+
 ;; This is only needed once, near the top of the file
 (require 'use-package)
 
-(eval-and-compile
-  (load (file-name-concat (locate-user-emacs-file "modules") "init-require")))
-(exordium-require 'init-prefs)            ; defines variables that prefs.el can override
-(exordium-require 'init-lib)              ; utility functions - load this first
+(exordium-require 'init-force-elpa)
+
+;; Pin user extra packages early, in case they are dependencies of some other
+;; packages that are installed early.
+(dolist (pkg exordium-extra-pinned)
+  (use-package-pin-package (car pkg) (cdr pkg)))
 
 (use-package use-package
   :exordium-force-elpa gnu
   :custom
   (use-package-always-ensure t)
   (use-package-compute-statistics t))
-(use-package diminish
-  :exordium-force-elpa gnu)
-(use-package bind-key
-  :exordium-force-elpa gnu)
 
 ;; Some packages (i.e., magit, forge) require relatively new package `seq'.
 ;; Unfortunately, `package' is unable to bump the built-in `seq'.  Ensure it is
 ;; installed in the newest available version.
 (use-package seq
   :exordium-force-elpa gnu)
 
-(dolist (pkg-pin exordium-extra-pinned)
-  (use-package-pin-package (car pkg-pin) (cdr pkg-pin)))
+;; `org' may be upgraded from ELPA (for example, as a part of a first start)
+;; and some packages depend on it.  To prevent loading a built in version by
+;; such packages upgrade it early.
+(use-package org
+  :exordium-force-elpa gnu)
 
+(use-package diminish
+  :exordium-force-elpa gnu)
+
+(use-package bind-key
+  :exordium-force-elpa gnu)
+
+(dolist (pkg exordium-extra-pinned)
+  (unless (package-installed-p pkg)
+    (package-install pkg)))
 
-;;; Load Modules
+;; Byte recompile modules, if necessary
+
 (require 'bytecomp)
 (defun exordium-recompile-modules ()
   "Recompile modules.
@@ -353,6 +354,10 @@ after it's been byte compiled."
   (async-bytecomp-package-mode))
 
 
+;;; Load Modules
+
+(exordium-require 'init-prefs)            ; Defines variables that prefs.el can override
+(exordium-require 'init-lib)              ; Utility functions - load this first
 (exordium-require 'init-environment)      ; environment variables
 
 
@@ -365,11 +370,6 @@ after it's been byte compiled."
 (setq custom-theme-directory exordium-themes-dir)
 (exordium-require 'init-progress-bar nil)
 
-;; `org' may be upgraded from ELPA (for example, as a part of a first start)
-;; and some packages depend on it.  To prevent loading a built in version by
-;; such packages upgrade it early.
-(exordium-require 'init-org)
-
 (when exordium-nw
   (set-face-background 'highlight nil))
 (when exordium-theme
@@ -425,6 +425,7 @@ after it's been byte compiled."
 
 ;; Major modes
 (exordium-require 'init-markdown)
+(exordium-require 'init-org)
 (exordium-require 'init-xml)
 
 ;; OS-specific things

modules/init-force-elpa.el

@@ -0,0 +1,145 @@
+;;; init-force-elpa.el --- Force installation from ELPA -*- lexical-binding: t -*-
+
+;;; Commentary:
+;;
+;; This module is a keyword extension for `use-pacakage'.  It adds keyword
+;; `:exordium-force-elpa'.  It takes a single argument, that is an archive name
+;; from which the package should be installed, and pinned to.  When containing
+;; `use-package' is evaluated, and the package is a built in package, and the
+;; package has a newer version in the specified archive then it is installed
+;; from the specified archive.
+;;
+;; This alleviates a need for a manual installation in case of
+;; upgrading a built-in package as described in Info node Info node `(emacs)
+;; Package Installation'.
+;;
+;; N.B. Using this keyword achieves similar result to what `:pin' does.  Using
+;; the latter is not necessary, when `:exordium-force-elpa' is used.  Should
+;; the `:pin' be used pay extra care to make sure it points to the same archive
+;; as `:exordium-force-elpa'.
+
+;;; Code:
+(require 'package)
+
+(eval-when-compile
+  (use-package use-package-core
+    :ensure nil
+    :defer t
+    :autoload (use-package-only-one
+               use-package-process-keywords)))
+
+(defun use-package-normalize/:exordium-force-elpa (_name keyword args)
+                                        ; checkdoc-params: (keyword args)
+  "Allow either a single string or a single symbol."
+  (use-package-only-one (symbol-name keyword) args
+    #'(lambda (_label arg)
+        (cond
+         ((stringp arg) arg)
+         ((use-package-non-nil-symbolp arg) (symbol-name arg))
+         (t
+          (use-package-error
+           ":exordium-force-elpa wants an archive name (a string)"))))))
+
+(defun exordium--use-package-force-elpa (pkg archive)
+  "Return the form that enforces installation of a built-in PKG from ARCHIVE."
+  `(let ((package ',(use-package-as-symbol pkg)))
+     ;; Ensure package is pinned even if it won't be installed
+     (use-package-pin-package package ,archive)
+     (package-read-all-archive-contents)
+     (when-let* (((package-built-in-p package))
+                 (builtin-version (alist-get package package--builtin-versions))
+                 (pkg-desc (or
+                            (cl-find-if (lambda (desc)
+                                          (equal (package-desc-archive desc)
+                                                 ,archive))
+                                        (alist-get package
+                                                   package-archive-contents))
+                            (progn
+                              (package-refresh-contents)
+                              (cl-find-if (lambda (desc)
+                                            (equal (package-desc-archive desc)
+                                                   ,archive))
+                                          (alist-get package
+                                                     package-archive-contents)))))
+                 (archive-version (package-desc-version pkg-desc))
+                 ((not (package-installed-p package archive-version)))
+                 ((version-list-< builtin-version archive-version)))
+       (package--save-selected-packages (cons package
+                                              package-selected-packages))
+       (condition-case-unless-debug err
+           (let* ((package-install-upgrade-built-in t)
+                  (transaction (package-compute-transaction
+                                (list pkg-desc)
+                                (package-desc-reqs pkg-desc)))
+                  (transaction-load-path (mapcar
+                                          (lambda (desc)
+                                            (expand-file-name
+                                             (package-desc-full-name desc)
+                                             package-user-dir))
+                                          transaction))
+                  (with-load-path
+                   (lambda (orig-fun &rest args)
+                     (let (new-load-path)
+                       ;; Ensure the newly installed package and its
+                       ;; dependencies are is in `load-path', when they are
+                       ;; reloaded.
+                       (let ((load-path (append transaction-load-path load-path)))
+                         (apply orig-fun args)
+                         (setq new-load-path load-path))
+                       ;; After reload, ensure all directories that were added
+                       ;; during reload are in the original `load-path'.
+                       (dolist (dir new-load-path)
+                         (unless (or (member dir transaction-load-path)
+                                     (member dir load-path))
+                           (push dir load-path)))))))
+             (unwind-protect
+                 (progn
+                   ;; `packgage-activate-1' calls
+                   ;; `package--reload-previously-loaded' and then adds the
+                   ;; newly installed package's directory to `load-path'. This
+                   ;; however may be not sufficient when some files `require'
+                   ;; files from the package.  Ensure such `require'd files are
+                   ;; visible for the latter call, and allow the original
+                   ;; `load-path' to be updated by the former (likely when
+                   ;; loading package autoloads).
+                   (advice-add 'package--reload-previously-loaded
+                               :around
+                               with-load-path)
+                   (package-download-transaction transaction)
+                   (package--quickstart-maybe-refresh)
+                   t)
+               (advice-remove 'package--reload-previously-loaded
+                              with-load-path)))
+         (error
+          (display-warning 'use-package
+                           (format "Failed to force ELPA installation %s: %s"
+                                   name (error-message-string err))
+                           :error))))))
+
+(defun use-package-handler/:exordium-force-elpa (name _keyword archive-name rest state)
+                                        ; checkdoc-params: (rest state)
+  "Pin package NAME to ELPA archive ARCHIVE-NAME and install it from there.
+Installation and pinning only hapens when the package is a
+built-in package and the archive ARCHIVE-NAME has a newer
+version of it (according to `version-list-<').
+
+Note that after the package NAME has been forcefully installed
+from ELPA archive it shadows the built-in package and it becomes
+eligible for upgrading while, i.e., `package-upgrade' is called,
+see Info node `(emacs) Package Installation'."
+  (let ((body (use-package-process-keywords name rest state))
+        (force-elpa-form (when archive-name
+                           (exordium--use-package-force-elpa name
+                                                             archive-name))))
+    ;; Pinning should occur just before ensuring
+    ;; See `use-package-handler/:ensure'.
+    (if (bound-and-true-p byte-compile-current-file)
+        (eval force-elpa-form)              ; Eval when byte-compiling,
+      (push force-elpa-form body))          ; or else wait until runtime.
+    body))
+
+(add-to-list 'use-package-keywords :exordium-force-elpa)
+
+(provide 'init-force-elpa)
+
+;;; init-force-elpa.el ends here