fetchers: git: add submodules and cloning depth support

Add two new options: "depths" and "submodules" to support recursive
submodules cloning along with shallow clones. We need those two
features together because projects with submodules tend to be large
and it may be inefficient to download whole repositories.

Also, as "depth" option forces git to download only one
branch/revision, and switching branches with submodules can be tricky,
we force git to download branch pointed by "rev" only.

Signed-off-by: Volodymyr Babchuk <[email protected]>

docs/user-reference.rst

@@ -249,6 +249,8 @@ repositories. There is a full list of supported parameters:
   url: "url://for.repository/project.git"
   rev: revision_name
   dir: "directory/where/store/code"
+  depth: 1
+  submodules: true
 
 
 
@@ -262,6 +264,11 @@ repositories. There is a full list of supported parameters:
   cloning. If this option is missed, `moulin` will try to guess
   directory name from :code:`url`. This path is relative to
   component's build directory.
+* :code:`submodules` - optional - boolean. Fetch submodules along with
+  main repository.
+* :code:`depth` - optional - cloning depth. Corresponds to :code:`--depth`
+  option for :code:`git clone`. If used together with :code:`submodules`
+  enabled, it will call :code:`git`  with :code:`--shallow-submodules`
 
 repo fetcher
 ^^^^^^^^^^^^

moulin/fetchers/git.py

@@ -33,12 +33,12 @@ def gen_build_rules(generator: ninja_syntax.Writer):
     generator.rule("git_clone",
                    command="GIT_SSH_COMMAND='ssh -o BatchMode=yes' "
                            "GIT_TERMINAL_PROMPT=0 "
-                           "git clone -q $git_url $git_dir && touch $out",
+                           "git clone $git_clone_opts -q $git_url $git_dir  && touch $out",
                    description="git clone")
     generator.newline()
 
     generator.rule("git_checkout",
-                   command="git -C $git_dir checkout -q $git_rev && touch $out",
+                   command="git -C $git_dir checkout $git_checkout_opts -q $git_rev && touch $out",
                    description="git checkout")
     generator.newline()
 
@@ -65,6 +65,27 @@ def __init__(self, conf: YamlValue, build_dir: str, generator: ninja_syntax.Writ
         dirname = conf.get("dir", default=_guess_dirname(self.url)).as_str
         self.git_dir = os.path.join(build_dir, dirname)
         self.git_rev = conf.get("rev", default="master").as_str
+        self.clone_opts = []
+        self.checkout_opts = []
+        depth = conf.get("depth", 0).as_int
+        if depth:
+            self.clone_opts.append(f"--depth {depth}")
+        if conf.get("submodules", default=False).as_bool:
+            self.enable_submodules = True
+            self.clone_opts.append("--recurse-submodules")
+            self.checkout_opts.append("--recurse-submodules")
+            if depth:
+                self.clone_opts.append("--shallow-submodules")
+        else:
+            self.enable_submodules = False
+
+        # Download only requested revision. This is less flexible for
+        # developers, but it is bulletproof in terms of potential
+        # checkout issues
+        if depth or self.enable_submodules:
+            self.clone_opts.append(f"--branch {self.git_rev}")
+        else:
+            self.clone_opts.append("--no-checkout")
 
     def gen_fetch(self):
         """Generate instruction to fetch git repo"""
@@ -88,25 +109,38 @@ def gen_fetch(self):
                              "git_clone",
                              variables={
                                  "git_url": self.url,
-                                 "git_dir": self.git_dir
+                                 "git_dir": self.git_dir,
+                                 "git_clone_opts": " ".join(self.clone_opts),
                              })
         self.generator.newline()
         self.generator.build(checkout_stamp,
                              "git_checkout",
                              clone_stamp,
                              variables={
                                  "git_rev": self.git_rev,
-                                 "git_dir": self.git_dir
+                                 "git_dir": self.git_dir,
+                                 "git_checkout_opts": " ".join(self.checkout_opts),
                              })
         self.generator.newline()
         return checkout_stamp
 
     def get_file_list(self) -> List[str]:
         "Get list of files under git control"
+        files = []
         repo = pygit2.Repository(self.git_dir)
         index = repo.index
         index.read()
-        return [os.path.join(self.git_dir, entry.path) for entry in index]
+        files.extend([os.path.join(self.git_dir, entry.path) for entry in index])
+
+        if self.enable_submodules:
+            for submodule in repo.submodules:
+                sub_repo = submodule.open()
+                index = sub_repo.index
+                index.read()
+                files.extend(
+                    [os.path.join(self.git_dir, submodule.path, entry.path) for entry in index])
+
+        return files
 
     def capture_state(self):
         """
@@ -116,3 +150,6 @@ def capture_state(self):
         repo = pygit2.Repository(self.git_dir)
         head = repo.revparse_single("HEAD")
         self.conf["rev"] = str(head)
+        # TODO: Store submodules state
+        if self.enable_submodules:
+            raise NotImplementedError("Can't (yet) capture state for git submodules")