fetch package managers manifests

.gitignore

@@ -1,3 +1,6 @@
+### Solution
+/storage
+
 ### Neo4J
 /data
 /conf
@@ -417,3 +420,5 @@ FodyWeavers.xsd
 # JetBrains Rider
 *.sln.iml
 
+# Mac OS
+.DS_Store

README.md

@@ -47,6 +47,12 @@ Now you can visit neo4J browser at http://localhost:7474/.
 10. Allow to mark dependency as deprecated
 11. Visualise dependencies
 
+## Tasks
+
+ 1. Optimize repo scanning
+    1. use custom GitLab client
+    2. gitlab scanning infinite loop
+
 ## Docs
 
 ### node.js Yarn

Watchtower.Core/ProjectFile.cs

@@ -1,6 +1,11 @@
 namespace Watchtower.Core;
+using FsPath = Path;
 
 public class ProjectFile
 {
     public required string Path { get; init; }
+
+    public string FileName => FsPath.GetFileName(Path);
+
+    public string? DirPath => FsPath.GetDirectoryName(Path);
 }

Watchtower.Integrations/GitLab/GitLabIntegration.cs

@@ -12,7 +12,7 @@ public static GitLabIntegration Create(string url, string privateToken)
         return new GitLabIntegration(new GitLabClient(url, privateToken));
     }
 
-    public async IAsyncEnumerable<Project> ListProjectsAsync()
+    public async IAsyncEnumerable<Project> ListProjectsAsync(int maxDepth = 2)
     {
         var groupQuery = new GroupQuery
         {
@@ -39,26 +39,28 @@ public async IAsyncEnumerable<Project> ListProjectsAsync()
                     DefaultBranch = project.DefaultBranch,
                     Namespace = project.Namespace.FullPath,
                     GitUrl = project.SshUrl,
-                    Files = ScanTreeAsync(project.Id)
+                    Files = ScanTreeAsync(project.Id, maxDepth)
                 };
             }
         }
     }
 
-    private async IAsyncEnumerable<ProjectFile> ScanTreeAsync(int projectId, string path = "", int depth = 0, int maxDepth = 1)
+    private async IAsyncEnumerable<ProjectFile> ScanTreeAsync(int projectId, int maxDepth, string path = "", int depth = 0)
     {
         var treeOptions = new RepositoryGetTreeOptions
         {
             Path = path,
-            Recursive = false
+            Recursive = false,
+            PerPage = 100,
         };
-        await foreach (var file in client.GetRepository(projectId).GetTreeAsync(treeOptions))
+        var repository = client.GetRepository(projectId);
+        await foreach (var file in repository.GetTreeAsync(treeOptions))
         {
             switch (file.Type)
             {
                 case ObjectType.tree when depth < maxDepth:
                     {
-                        await foreach (var innerFile in ScanTreeAsync(projectId, file.Path, depth + 1, maxDepth))
+                        await foreach (var innerFile in ScanTreeAsync(projectId, maxDepth, file.Path, depth + 1))
                             yield return innerFile;
                         break;
                     }
@@ -73,4 +75,11 @@ private async IAsyncEnumerable<ProjectFile> ScanTreeAsync(int projectId, string
             }
         }
     }
+
+    public async Task<string> GetProjectFileAsync(Project project, ProjectFile file)
+    {
+        var repository = client.GetRepository(project.Id);
+        var fileData = await repository.Files.GetAsync(file.Path, project.DefaultBranch);
+        return System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(fileData.Content));
+    }
 }

Watchtower.Playground/Program.cs

@@ -5,6 +5,11 @@
 
 var gitLabUrl = Environment.GetEnvironmentVariable("GITLAB_URL") ?? "https://gitlab.com/";
 var gitLabToken = Environment.GetEnvironmentVariable("GITLAB_TOKEN") ?? "***";
+var dataRoot = Environment.GetEnvironmentVariable("STORAGE_DIR") ?? $"{Directory.GetCurrentDirectory()}/data";
+
+var storage = new Storage(dataRoot);
+
+Console.WriteLine($"String data to: {dataRoot}");
 
 var gitlab = GitLabIntegration.Create(gitLabUrl, gitLabToken);
 
@@ -13,23 +18,31 @@
 await foreach (var project in gitlab.ListProjectsAsync())
 {
     Console.WriteLine($"{project.Source} {project.Id}: {project.Namespace}/{project.Name} ({project.DefaultBranch})");
+    if (storage.IsProjectStoredAsync(project))
+    {
+        continue;
+    }
     await foreach (var file in project.Files)
     {
-        var fileName = Path.GetFileName(file.Path);
+        var fileName = file.FileName;
         if (fileName is "package.json" or "yarn.lock" or "package-lock.json" or "lerna.json")
         {
+            await storage.PutFileAsync(project, file, gitlab.GetProjectFileAsync(project, file));
             Console.WriteLine($"\t [ node ] {file.Path}");
         }
         else if (fileName is "Dockerfile" or "docker-compose.yaml" or "docker-compose.yml")
         {
+            await storage.PutFileAsync(project, file, gitlab.GetProjectFileAsync(project, file));
             Console.WriteLine($"\t [docker] {file.Path}");
         }
         else if (fileName is "pom.xml")
         {
+            await storage.PutFileAsync(project, file, gitlab.GetProjectFileAsync(project, file));
             Console.WriteLine($"\t [ java ] {file.Path}");
         }
         else if (fileName is "pyproject.toml")
         {
+            await storage.PutFileAsync(project, file, gitlab.GetProjectFileAsync(project, file));
             Console.WriteLine($"\t [python] {file.Path}");
         }
         else
@@ -38,3 +51,4 @@
         }
     }
 }
+

Watchtower.Playground/Storage.cs

@@ -0,0 +1,26 @@
+using Watchtower.Core;
+
+namespace Watchtower.Playground;
+
+public class Storage
+{
+    private string Root { get; }
+
+    public Storage(string root)
+    {
+        Root = root;
+        Directory.CreateDirectory(root);
+    }
+
+    public async Task PutFileAsync(Project project, ProjectFile file, Task<string> contents)
+    {
+        var dir = $"{GetProjectPath(project)}/{file.DirPath}";
+        Directory.CreateDirectory(dir);
+        await File.WriteAllTextAsync($"{dir}/{file.FileName}", await contents);
+    }
+
+    public bool IsProjectStoredAsync(Project project) => Directory.Exists(GetProjectPath(project));
+
+    private string GetProjectPath(Project project) =>
+        $"{Root}/@{project.Namespace}@/@@{project.Name}@@/__{project.DefaultBranch}__";
+}