Ускорение build-deps (#45)

* faster build-deps with caching

* another fix for no nuget restore for root while build-deps

* split test

Author: degtep

Commands/Build.cs

@@ -1,5 +1,6 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
+using System.Threading.Tasks;
 using Common;
 
 namespace Commands
@@ -52,12 +53,15 @@ protected override int Execute()
                 return -1;
             }
 
+            var builder = new ModuleBuilder(Log, buildSettings);
+            var builderInitTask = Task.Run(() => builder.Init());
+
             new BuildPreparer(Log).GetModulesOrder(moduleName, configuration, out topSortedDeps, out modulesToUpdate, out currentCommitHases);
 
             var builtStorage = BuiltInfoStorage.Deserialize();
             builtStorage.RemoveBuildInfo(moduleName);
 
-            var builder = new ModuleBuilder(Log, buildSettings);
+            builderInitTask.Wait();
             var module = new Dep(moduleName, null, configuration);
 
             BuildDeps.TryNugetRestore(new List<Dep> {module}, builder);

Commands/BuildDeps.cs

@@ -45,23 +45,25 @@ protected override int Execute()
             var moduleName = Path.GetFileName(cwd);
 
             configuration = string.IsNullOrEmpty(configuration) ? "full-build" : configuration;
-
             List<Dep> modulesToBuild;
             List<Dep> topSortedDeps;
             Dictionary<string, string> currentCommitHases;
-
+            var builder = new ModuleBuilder(Log, buildSettings);
+            var builderInitTask = Task.Run(() => builder.Init());
             new BuildPreparer(Log).GetModulesOrder(moduleName, configuration ?? "full-build", out topSortedDeps, out modulesToBuild, out currentCommitHases);
+            if (modulesToBuild.Count > 0 && modulesToBuild[modulesToBuild.Count - 1].Name == moduleName)
+            {
+                modulesToBuild.RemoveAt(modulesToBuild.Count - 1); //remove root
+            }
             if (rebuild)
                 modulesToBuild = topSortedDeps;
 
             var builtStorage = BuiltInfoStorage.Deserialize();
             foreach (var dep in modulesToBuild)
                 builtStorage.RemoveBuildInfo(dep.Name);
-
-            var builder = new ModuleBuilder(Log, buildSettings);
 
+            builderInitTask.Wait();
             TryNugetRestore(modulesToBuild, builder);
-
             int built = 1;
             for (var i = 0; i < topSortedDeps.Count - 1; i++)
             {
@@ -92,7 +94,6 @@ protected override int Execute()
                 built++;
             }
             builtStorage.Save();
-
             Log.Debug("msbuild time: " + ModuleBuilder.TotalMsbuildTime);
             return 0;
         }

Commands/PackCommand.cs

@@ -44,6 +44,7 @@ protected override int Execute()
             {
                 XmlDocumentHelper.Save(patchedDocument, projectPath, "\n");
                 var moduleBuilder = new ModuleBuilder(Log, buildSettings);
+                moduleBuilder.Init();
                 ConsoleWriter.WriteInfo("start pack");
                 moduleBuilder.DotnetPack(modulePath, projectPath, buildData?.Configuration ?? "Release");
             }

Common/BuildPreparer.cs

@@ -20,16 +20,14 @@ public void GetModulesOrder(string moduleName, string configuration, out List<De
         {
             log.Debug("Building configurations graph");
             ConsoleWriter.WriteProgress("Building configurations graph");
-
             var configsGraph = BuildConfigsGraph(moduleName, configuration);
             configsGraph = EraseExtraChildren(configsGraph);
             topSortedVertices = GetTopologicallySortedGraph(configsGraph, moduleName, configuration);
-
+            
             log.Debug("Getting current commit hashes");
             ConsoleWriter.WriteProgress("Getting current commit hashes");
             currentCommitHashes = GetCurrentCommitHashes(configsGraph);
             updatedModules = BuiltInfoStorage.Deserialize().GetUpdatedModules(topSortedVertices, currentCommitHashes);
-
             ConsoleWriter.ResetProgress();
         }
 
@@ -149,22 +147,33 @@ public static Dictionary<Dep, List<Dep>> BuildConfigsGraph(string moduleName, st
             return graph;
         }
 
-        private static void Dfs(Dep dep, Dictionary<Dep, List<Dep>> graph, HashSet<Dep> visitedConfigurations)
+        private static readonly Dictionary<string, bool> DepConfigurationExistsCache = new Dictionary<string, bool>();
+        private static void CheckAndUpdateDepConfiguration(Dep dep)
         {
             dep.UpdateConfigurationIfNull();
-
-            if (Yaml.Exists(dep.Name) && !Yaml.ConfigurationParser(dep.Name).ConfigurationExists(dep.Configuration))
+            var key = dep.ToString();
+            if (!DepConfigurationExistsCache.ContainsKey(key))
+            {
+                if (!Directory.Exists(Path.Combine(Helper.CurrentWorkspace, dep.Name)))
+                {
+                    throw new CementBuildException("Failed to find module '" + dep.Name + "'");
+                }
+                DepConfigurationExistsCache[key] = !Yaml.Exists(dep.Name) ||
+                    Yaml.ConfigurationParser(dep.Name).ConfigurationExists(dep.Configuration);
+            }
+            if (!DepConfigurationExistsCache[key])
             {
-                ConsoleWriter.WriteWarning($"Configuration '{dep.Configuration}' was not found in {dep.Name}. Will take full-build config");
+                ConsoleWriter.WriteWarning(
+                    $"Configuration '{dep.Configuration}' was not found in {dep.Name}. Will take full-build config");
                 dep.Configuration = "full-build";
             }
+        }
 
+        private static void Dfs(Dep dep, Dictionary<Dep, List<Dep>> graph, HashSet<Dep> visitedConfigurations)
+        {
+            CheckAndUpdateDepConfiguration(dep);
             visitedConfigurations.Add(dep);
             graph[dep] = new List<Dep>();
-            if (!Directory.Exists(Path.Combine(Helper.CurrentWorkspace, dep.Name)))
-            {
-                throw new CementBuildException("Failed to find module '" + dep.Name + "'");
-            }
             var currentDeps = new DepsParser(Path.Combine(Helper.CurrentWorkspace, dep.Name)).Get(dep.Configuration).Deps ?? new List<Dep>();
             currentDeps = currentDeps.Select(d => new Dep(d.Name, null, d.Configuration)).ToList();
             foreach (var d in currentDeps)

Common/BuiltInfoStorage.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -73,7 +73,7 @@ public void Save()
 
         public List<Dep> GetUpdatedModules(List<Dep> modules, Dictionary<string, string> currentCommitHashes)
         {
-            return modules.Where(module => IsModuleUpdate(currentCommitHashes, module)).ToList();
+            return modules.AsParallel().Where(module => IsModuleUpdate(currentCommitHashes, module)).ToList();
         }
 
         private bool IsModuleUpdate(Dictionary<string, string> currentCommitHashes, Dep module)

Common/Dep.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -13,6 +13,7 @@ public class Dep : IEquatable<Dep>
         public string Treeish { get; set; }
         public string Configuration { get; set; }
         public bool NeedSrc { get; set; }
+        static readonly Dictionary<string, string> DepDefaultConfigurationCache = new Dictionary<string, string>();
 
         public Dep(string name, string treeish = null, string configuration = null)
         {
@@ -53,16 +54,23 @@ public Dep(string fromYamlString)
                 Configuration = null;
         }
 
+        
         public void UpdateConfigurationIfNull()
         {
             UpdateConfigurationIfNull(Helper.CurrentWorkspace);
         }
 
         public void UpdateConfigurationIfNull(string workspace)
         {
-            Configuration = Configuration ??
-                            new ConfigurationParser(new FileInfo(Path.Combine(workspace, Name)))
-                                .GetDefaultConfigurationName();
+            if (!string.IsNullOrEmpty(Configuration)) return;
+            var path = Path.Combine(workspace, Name);
+            if (!DepDefaultConfigurationCache.ContainsKey(path))
+            {
+                DepDefaultConfigurationCache[path] =
+                    new ConfigurationParser(new FileInfo(Path.Combine(workspace, Name)))
+                        .GetDefaultConfigurationName();
+            }
+            Configuration = DepDefaultConfigurationCache[path];
         }
 
         private string UnEscapeBadChars(string str)

Common/ModuleBuilder.cs

@@ -18,6 +18,10 @@ public ModuleBuilder(ILog log, BuildSettings buildSettings)
         {
             this.log = log;
             this.buildSettings = buildSettings;
+        }
+
+        public void Init()
+        {
             VsDevHelper.ReplaceVariablesToVs();
         }
 

Tests/BuildTests/TestBuildDepsOrder.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Common;

Tests/CommandsTests/TestGet.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
 using Common;
 using Common.YamlParsers;
@@ -634,7 +634,7 @@ public void TestDoesNotThrowWhenDefaultTreeishAndNotDefaultTreeish2()
         }
 
         [Test]
-        public void TestGetDepsWithBranches()
+        public void TestGetDepsWithBranchesFullBuild()
         {
             using (var env = new TestEnvironment())
             {
@@ -657,12 +657,30 @@ public void TestGetDepsWithBranches()
                 env.Get("A");
                 Assert.IsTrue(Directory.Exists(Path.Combine(dir, "C")));
                 Assert.AreEqual("branch1", new GitRepository("C", dir, Log).CurrentLocalTreeish().Value);
+            }
+        }
+
+        [Test]
+        public void TestGetDepsWithBranchesClientBuild()
+        {
+            using (var env = new TestEnvironment())
+            {
+                var dir = env.WorkingDirectory.Path;
 
+                env.CreateRepo("A", new Dictionary<string, DepsContent>
+                {
+                    {"full-build", new DepsContent(null, new List<Dep> {new Dep("B@master")})}
+                });
                 env.CreateRepo("B", new Dictionary<string, DepsContent>
                 {
                     {"full-build", new DepsContent(null, new List<Dep> {new Dep("C@branch1")})},
                     {"client *default", new DepsContent(null, new List<Dep> {new Dep("C@branch2")})}
                 });
+                env.CreateRepo("C", new Dictionary<string, DepsContent>
+                {
+                    {"full-build", new DepsContent(null, new List<Dep>())}
+                }, new[] { "master", "branch1", "branch2" });
+                
                 env.Get("A");
                 Assert.IsTrue(Directory.Exists(Path.Combine(dir, "C")));
                 Assert.AreEqual("branch2", new GitRepository("C", dir, Log).CurrentLocalTreeish().Value);