diff --git a/src/TfsAdvanced.Updater/Tasks/BuildDefinitionUpdater.cs b/src/TfsAdvanced.Updater/Tasks/BuildDefinitionUpdater.cs index 91f777a..511258e 100644 --- a/src/TfsAdvanced.Updater/Tasks/BuildDefinitionUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/BuildDefinitionUpdater.cs @@ -7,79 +7,58 @@ using TfsAdvanced.Models; using TfsAdvanced.Models.Infrastructure; using TFSAdvanced.Models.DTO; +using TFSAdvanced.Updater.Tasks; namespace TfsAdvanced.Updater.Tasks { - public class BuildDefinitionUpdater + public class BuildDefinitionUpdater : UpdaterBase { private readonly BuildDefinitionRepository buildDefinitionRepository; private readonly UpdateStatusRepository updateStatusRepository; private readonly ProjectRepository projectRepository; private readonly RepositoryRepository repositoryRepository; private readonly RequestData requestData; - private readonly ILogger logger; - private bool IsRunning; - - public BuildDefinitionUpdater(BuildDefinitionRepository buildDefinitionRepository, RequestData requestData, ProjectRepository projectRepository, UpdateStatusRepository updateStatusRepository, RepositoryRepository repositoryRepository, ILogger logger) + + public BuildDefinitionUpdater(BuildDefinitionRepository buildDefinitionRepository, RequestData requestData, ProjectRepository projectRepository, UpdateStatusRepository updateStatusRepository, RepositoryRepository repositoryRepository, ILogger logger) : + base(logger) { this.buildDefinitionRepository = buildDefinitionRepository; this.requestData = requestData; this.projectRepository = projectRepository; this.updateStatusRepository = updateStatusRepository; this.repositoryRepository = repositoryRepository; - this.logger = logger; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try + var buildDefinitions = new ConcurrentBag(); + Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => { - - var buildDefinitions = new ConcurrentBag(); - Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => + var definitions = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/definitions?api=2.2").Result; + if (definitions == null) { - var definitions = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/definitions?api=2.2").Result; - if (definitions == null) - { - logger.LogInformation($"Unable to get the definitiosn for the project {project.Name}"); - return; - } - Parallel.ForEach(definitions, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, definition => - { - var populatedDefinition = GetAsync.Fetch(requestData, definition.url).Result; + logger.LogInformation($"Unable to get the definitiosn for the project {project.Name}"); + return; + } + Parallel.ForEach(definitions, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, definition => + { + var populatedDefinition = GetAsync.Fetch(requestData, definition.url).Result; - buildDefinitions.Add(new BuildDefinition - { - DefaultBranch = populatedDefinition.repository.defaultBranch, - Folder = populatedDefinition.path, - Id = populatedDefinition.id, - Name = populatedDefinition.name, - Url = populatedDefinition._links.web.href, - Repository = repositoryRepository.GetById(populatedDefinition.repository.id) - }); + buildDefinitions.Add(new BuildDefinition + { + DefaultBranch = populatedDefinition.repository.defaultBranch, + Folder = populatedDefinition.path, + Id = populatedDefinition.id, + Name = populatedDefinition.name, + Url = populatedDefinition._links.web.href, + Repository = repositoryRepository.GetById(populatedDefinition.repository.id) }); }); + }); - buildDefinitionRepository.Update(buildDefinitions); - - updateStatusRepository.UpdateStatus(new UpdateStatus { LastUpdate = DateTime.Now, UpdatedRecords = buildDefinitions.Count, UpdaterName = nameof(BuildDefinitionUpdater)}); - - } - catch (Exception ex) - { - throw new InvalidOperationException("Error processing the build definition updater.", ex); - } - finally - - { - IsRunning = false; - } - + buildDefinitionRepository.Update(buildDefinitions); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = buildDefinitions.Count, UpdaterName = nameof(BuildDefinitionUpdater)}); } } } diff --git a/src/TfsAdvanced.Updater/Tasks/BuildUpdater.cs b/src/TfsAdvanced.Updater/Tasks/BuildUpdater.cs index 18fe647..c651769 100644 --- a/src/TfsAdvanced.Updater/Tasks/BuildUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/BuildUpdater.cs @@ -10,84 +10,66 @@ using TfsAdvanced.Models.Infrastructure; using TFSAdvanced.Models.DTO; using TFSAdvanced.Updater.Models.Builds; +using TFSAdvanced.Updater.Tasks; using Build = TFSAdvanced.Models.DTO.Build; using BuildStatus = TFSAdvanced.Models.DTO.BuildStatus; namespace TfsAdvanced.Updater.Tasks { - public class BuildUpdater + public class BuildUpdater : UpdaterBase { private readonly BuildRepository buildRepository; private readonly UpdateStatusRepository updateStatusRepository; private readonly ProjectRepository projectRepository; private readonly RequestData requestData; - private readonly ILogger logger; - private bool IsRunning; - + public BuildUpdater(BuildRepository buildRepository, RequestData requestData, ProjectRepository projectRepository, UpdateStatusRepository updateStatusRepository, ILogger logger) + :base(logger) { this.buildRepository = buildRepository; this.requestData = requestData; this.projectRepository = projectRepository; this.updateStatusRepository = updateStatusRepository; - this.logger = logger; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try + DateTime yesterday = DateTime.Now.Date.AddDays(-1); + var builds = new ConcurrentStack(); + Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => { - - DateTime yesterday = DateTime.Now.Date.AddDays(-1); - var builds = new ConcurrentStack(); - Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => + // Finished PR builds + var projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&reasonFilter=validateShelveset&minFinishTime={yesterday:O}").Result; + if (projectBuilds != null && projectBuilds.Any()) { - // Finished PR builds - var projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&reasonFilter=validateShelveset&minFinishTime={yesterday:O}").Result; - if (projectBuilds != null && projectBuilds.Any()) - { - builds.PushRange(projectBuilds.ToArray()); - } + builds.PushRange(projectBuilds.ToArray()); + } - // Current active builds - projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&inProgress=notStarted").Result; - if (projectBuilds != null && projectBuilds.Any()) - { - builds.PushRange(projectBuilds.ToArray()); - } + // Current active builds + projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&statusFilter=inProgress&inProgress=notStarted").Result; + if (projectBuilds != null && projectBuilds.Any()) + { + builds.PushRange(projectBuilds.ToArray()); + } - DateTime twoHoursAgo = DateTime.Now.AddHours(-2); - // Because we want to capture the final state of any build that was running and just finished we are getting those too - // Finished builds within the last 2 hours - projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={twoHoursAgo:O}").Result; - if (projectBuilds != null && projectBuilds.Any()) - { - builds.PushRange(projectBuilds.ToArray()); - } - - }); + DateTime twoHoursAgo = DateTime.Now.AddHours(-2); + // Because we want to capture the final state of any build that was running and just finished we are getting those too + // Finished builds within the last 2 hours + projectBuilds = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/build/builds?api-version=2.2&minFinishTime={twoHoursAgo:O}").Result; + if (projectBuilds != null && projectBuilds.Any()) + { + builds.PushRange(projectBuilds.ToArray()); + } + }); - // The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't finished yet - var buildLists = builds.ToList(); - buildRepository.Update(buildLists.Select(CreateBuild)); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater)}); - } - catch (Exception ex) - { - throw new InvalidOperationException("Error running build updater", ex); - } - finally - { - IsRunning = false; - } + // The builds must be requested without the filter because the only filter available is minFinishTime, which will filter out those that haven't finished yet + var buildLists = builds.ToList(); + buildRepository.Update(buildLists.Select(CreateBuild)); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = buildLists.Count, UpdaterName = nameof(BuildUpdater)}); } private Build CreateBuild(TFSAdvanced.Updater.Models.Builds.Build build) diff --git a/src/TfsAdvanced.Updater/Tasks/JobRequestUpdater.cs b/src/TfsAdvanced.Updater/Tasks/JobRequestUpdater.cs index 50656f5..d66f58c 100644 --- a/src/TfsAdvanced.Updater/Tasks/JobRequestUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/JobRequestUpdater.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Hangfire; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TfsAdvanced.Models; using TfsAdvanced.Models.Infrastructure; @@ -10,11 +11,12 @@ using TFSAdvanced.Models.DTO; using TFSAdvanced.Updater.Models.Builds; using TFSAdvanced.Updater.Models.JobRequests; +using TFSAdvanced.Updater.Tasks; using BuildStatus = TFSAdvanced.Models.DTO.BuildStatus; namespace TfsAdvanced.Updater.Tasks { - public class JobRequestUpdater + public class JobRequestUpdater : UpdaterBase { private readonly JobRequestRepository jobRequestRepository; private readonly UpdateStatusRepository updateStatusRepository; @@ -24,10 +26,8 @@ public class JobRequestUpdater private readonly ProjectRepository projectRepository; private readonly ReleaseDefinitionRepository releaseDefinitionRepository; private readonly RequestData requestData; - private bool IsRunning; - - - public JobRequestUpdater(JobRequestRepository jobRequestRepository, RequestData requestData, PoolRepository poolRepository, BuildRepository buildRepository, UpdateStatusRepository updateStatusRepository, BuildDefinitionRepository buildDefinitionRepository, ProjectRepository projectRepository, ReleaseDefinitionRepository releaseDefinitionRepository) + + public JobRequestUpdater(JobRequestRepository jobRequestRepository, RequestData requestData, PoolRepository poolRepository, BuildRepository buildRepository, UpdateStatusRepository updateStatusRepository, BuildDefinitionRepository buildDefinitionRepository, ProjectRepository projectRepository, ReleaseDefinitionRepository releaseDefinitionRepository, ILogger logger) : base(logger) { this.jobRequestRepository = jobRequestRepository; this.requestData = requestData; @@ -39,161 +39,143 @@ public JobRequestUpdater(JobRequestRepository jobRequestRepository, RequestData this.releaseDefinitionRepository = releaseDefinitionRepository; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try - { + ConcurrentBag jobRequests = new ConcurrentBag(); - ConcurrentBag jobRequests = new ConcurrentBag(); + Parallel.ForEach(poolRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, pool => + { - Parallel.ForEach(poolRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, pool => + var poolJobRequests = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools/{pool.id}/jobrequests?api-version=1.0").Result; + if (poolJobRequests != null) { - - var poolJobRequests = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools/{pool.id}/jobrequests?api-version=1.0").Result; - if (poolJobRequests != null) + foreach (var poolJobRequest in poolJobRequests) { - foreach (var poolJobRequest in poolJobRequests) + + QueueJob queueJob = new QueueJob { - - QueueJob queueJob = new QueueJob - { - - RequestId = poolJobRequest.requestId, - QueuedTime = poolJobRequest.queueTime, - AssignedTime = poolJobRequest.assignTime, - FinishedTime = poolJobRequest.finishTime, - Name = poolJobRequest.definition.name, - Url = poolJobRequest.definition._links.self.href - }; - - if (poolJobRequest.planType == PlanTypes.Build) + + RequestId = poolJobRequest.requestId, + QueuedTime = poolJobRequest.queueTime, + AssignedTime = poolJobRequest.assignTime, + FinishedTime = poolJobRequest.finishTime, + Name = poolJobRequest.definition.name, + Url = poolJobRequest.definition._links.self.href + }; + + if (poolJobRequest.planType == PlanTypes.Build) + { + var build = buildRepository.GetBuild(poolJobRequest.owner.id); + if (build != null) { - var build = buildRepository.GetBuild(poolJobRequest.owner.id); - if (build != null) - { - queueJob.LaunchedBy = build.Creator; - queueJob.StartedTime = build.StartedDate; - queueJob.FinishedTime = build.FinishedDate; - queueJob.BuildFolder = build.Folder; + queueJob.LaunchedBy = build.Creator; + queueJob.StartedTime = build.StartedDate; + queueJob.FinishedTime = build.FinishedDate; + queueJob.BuildFolder = build.Folder; - switch (build.BuildStatus) - { - case BuildStatus.NotStarted: - queueJob.QueueJobStatus = QueueJobStatus.Queued; - break; - case BuildStatus.Abandonded: - queueJob.QueueJobStatus = QueueJobStatus.Abandonded; - break; - case BuildStatus.Building: - queueJob.QueueJobStatus = QueueJobStatus.Building; - break; - case BuildStatus.Cancelled: - queueJob.QueueJobStatus = QueueJobStatus.Cancelled; - break; - case BuildStatus.Expired: - case BuildStatus.Failed: - queueJob.QueueJobStatus = QueueJobStatus.Failed; - break; - case BuildStatus.Succeeded: - queueJob.QueueJobStatus = QueueJobStatus.Succeeded; - break; - } + switch (build.BuildStatus) + { + case BuildStatus.NotStarted: + queueJob.QueueJobStatus = QueueJobStatus.Queued; + break; + case BuildStatus.Abandonded: + queueJob.QueueJobStatus = QueueJobStatus.Abandonded; + break; + case BuildStatus.Building: + queueJob.QueueJobStatus = QueueJobStatus.Building; + break; + case BuildStatus.Cancelled: + queueJob.QueueJobStatus = QueueJobStatus.Cancelled; + break; + case BuildStatus.Expired: + case BuildStatus.Failed: + queueJob.QueueJobStatus = QueueJobStatus.Failed; + break; + case BuildStatus.Succeeded: + queueJob.QueueJobStatus = QueueJobStatus.Succeeded; + break; } + } - var buildDefinition = buildDefinitionRepository.GetBuildDefinition(poolJobRequest.definition.id); - if (buildDefinition != null) + var buildDefinition = buildDefinitionRepository.GetBuildDefinition(poolJobRequest.definition.id); + if (buildDefinition != null) + { + var project = projectRepository.GetProject(buildDefinition.Repository.Project.Id); + if (project != null) { - var project = projectRepository.GetProject(buildDefinition.Repository.Project.Id); - if (project != null) - { - queueJob.Project = new Project - { - Id = project.Id, - Name = project.Name, - Url = project.Url - }; - } - else + queueJob.Project = new Project { - queueJob.Project = new Project - { - Name = "Unknown Project" - }; - } + Id = project.Id, + Name = project.Name, + Url = project.Url + }; } else { queueJob.Project = new Project { - Name = "Unknown Build Definition" + Name = "Unknown Project" }; } - } - else if (poolJobRequest.planType == PlanTypes.Release) + else { - if (poolJobRequest.finishTime.HasValue) + queueJob.Project = new Project { - switch (poolJobRequest.result) - { - case BuildResult.succeeded: - queueJob.QueueJobStatus = QueueJobStatus.Succeeded; - break; - case BuildResult.abandoned: - queueJob.QueueJobStatus = QueueJobStatus.Abandonded; - break; - case BuildResult.canceled: - queueJob.QueueJobStatus = QueueJobStatus.Cancelled; - break; - case BuildResult.failed: - case BuildResult.partiallySucceeded: - queueJob.QueueJobStatus = QueueJobStatus.Failed; - break; - } - } + Name = "Unknown Build Definition" + }; + } - var releaseDefinition = releaseDefinitionRepository.GetReleaseDefinition(poolJobRequest.definition.id); - if (releaseDefinition != null) + } + else if (poolJobRequest.planType == PlanTypes.Release) + { + if (poolJobRequest.finishTime.HasValue) + { + switch (poolJobRequest.result) { - queueJob.Project = releaseDefinition.Project; + case BuildResult.succeeded: + queueJob.QueueJobStatus = QueueJobStatus.Succeeded; + break; + case BuildResult.abandoned: + queueJob.QueueJobStatus = QueueJobStatus.Abandonded; + break; + case BuildResult.canceled: + queueJob.QueueJobStatus = QueueJobStatus.Cancelled; + break; + case BuildResult.failed: + case BuildResult.partiallySucceeded: + queueJob.QueueJobStatus = QueueJobStatus.Failed; + break; } - else - { - queueJob.Project = new Project - { - Name = "Unknown Release Definition" - }; - } - - } + var releaseDefinition = releaseDefinitionRepository.GetReleaseDefinition(poolJobRequest.definition.id); + if (releaseDefinition != null) + { + queueJob.Project = releaseDefinition.Project; + } + else + { + queueJob.Project = new Project + { + Name = "Unknown Release Definition" + }; + } - jobRequests.Add(queueJob); } - } - }); - DateTime yesterday = DateTime.Now.Date.AddDays(-1); - var jobRequestsLists = jobRequests.Where(x => !x.StartedTime.HasValue || x.StartedTime.Value >= yesterday).ToList(); - jobRequestRepository.Update(jobRequestsLists); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = jobRequestsLists.Count, UpdaterName = nameof(JobRequestUpdater)}); - } - catch (Exception ex) - { - throw new InvalidOperationException("Error running Job Request Updater ", ex); - } - finally - { - IsRunning = false; - } + jobRequests.Add(queueJob); + } + } + }); + + DateTime yesterday = DateTime.Now.Date.AddDays(-1); + var jobRequestsLists = jobRequests.Where(x => !x.StartedTime.HasValue || x.StartedTime.Value >= yesterday).ToList(); + jobRequestRepository.Update(jobRequestsLists); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = jobRequestsLists.Count, UpdaterName = nameof(JobRequestUpdater)}); } } } diff --git a/src/TfsAdvanced.Updater/Tasks/PoolUpdater.cs b/src/TfsAdvanced.Updater/Tasks/PoolUpdater.cs index 0d6ff2d..b8e5cb0 100644 --- a/src/TfsAdvanced.Updater/Tasks/PoolUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/PoolUpdater.cs @@ -1,50 +1,34 @@ using System; using Hangfire; +using Hangfire.Logging; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TfsAdvanced.Models; using TFSAdvanced.Models.DTO; +using TFSAdvanced.Updater.Tasks; namespace TfsAdvanced.Updater.Tasks { - public class PoolUpdater + public class PoolUpdater : UpdaterBase { private readonly RequestData requestData; private readonly PoolRepository poolRepository; private readonly UpdateStatusRepository updateStatusRepository; - private bool IsRunning; - - - public PoolUpdater(PoolRepository poolRepository, RequestData requestData, UpdateStatusRepository updateStatusRepository) + + public PoolUpdater(PoolRepository poolRepository, RequestData requestData, UpdateStatusRepository updateStatusRepository, ILogger logger) : base(logger) { this.requestData = requestData; this.updateStatusRepository = updateStatusRepository; this.poolRepository = poolRepository; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try - { - - var pools = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools?api-version=1.0").Result; - if (pools != null) - { - poolRepository.Update(pools); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = pools.Count, UpdaterName = nameof(PoolUpdater)}); - } - - } - catch (Exception ex) - { - throw new InvalidOperationException("Error running Pool Updater", ex); - } - finally + var pools = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/distributedtask/pools?api-version=1.0").Result; + if (pools != null) { - IsRunning = false; + poolRepository.Update(pools); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = pools.Count, UpdaterName = nameof(PoolUpdater)}); } } } diff --git a/src/TfsAdvanced.Updater/Tasks/ProjectUpdater.cs b/src/TfsAdvanced.Updater/Tasks/ProjectUpdater.cs index 414003e..d48cae1 100644 --- a/src/TfsAdvanced.Updater/Tasks/ProjectUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/ProjectUpdater.cs @@ -1,58 +1,41 @@ using System; using System.Linq; using Hangfire; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TfsAdvanced.Models; using TFSAdvanced.Updater.Models.Projects; +using TFSAdvanced.Updater.Tasks; namespace TfsAdvanced.Updater.Tasks { - public class ProjectUpdater + public class ProjectUpdater : UpdaterBase { private readonly ProjectRepository projectRepository; private readonly UpdateStatusRepository updateStatusRepository; private readonly RequestData requestData; - private bool IsRunning; - public ProjectUpdater(ProjectRepository projectRepository, RequestData requestData, UpdateStatusRepository updateStatusRepository) + public ProjectUpdater(ProjectRepository projectRepository, RequestData requestData, UpdateStatusRepository updateStatusRepository, ILogger logger) : base(logger) { this.projectRepository = projectRepository; this.requestData = requestData; this.updateStatusRepository = updateStatusRepository; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try + var projects = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/projects?api-version=1.0").Result; + if (projects != null) { - var projects = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/_apis/projects?api-version=1.0").Result; - if (projects != null) + projectRepository.Update(projects.Select(x => new TFSAdvanced.Models.DTO.Project { - projectRepository.Update(projects.Select(x => new TFSAdvanced.Models.DTO.Project - { - Id = x.id, - Name = x.name, - Url = x.remoteUrl - })); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = projects.Count, UpdaterName = nameof(ProjectUpdater)}); - } - - - } - catch (Exception ex) - { - throw new InvalidOperationException("Error running project updater.", ex); + Id = x.id, + Name = x.name, + Url = x.remoteUrl + })); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = projects.Count, UpdaterName = nameof(ProjectUpdater)}); } - finally - { - IsRunning = false; - } - } } } diff --git a/src/TfsAdvanced.Updater/Tasks/PullRequestUpdater.cs b/src/TfsAdvanced.Updater/Tasks/PullRequestUpdater.cs index e72d2ab..d9ee03f 100644 --- a/src/TfsAdvanced.Updater/Tasks/PullRequestUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/PullRequestUpdater.cs @@ -11,102 +11,83 @@ using TfsAdvanced.Models.Infrastructure; using TFSAdvanced.Models.DTO; using TFSAdvanced.Updater.Models.PullRequests; +using TFSAdvanced.Updater.Tasks; using PullRequest = TFSAdvanced.Models.DTO.PullRequest; using Reviewer = TFSAdvanced.Models.DTO.Reviewer; namespace TfsAdvanced.Updater.Tasks { - public class PullRequestUpdater + public class PullRequestUpdater : UpdaterBase { private readonly RequestData requestData; private readonly PullRequestRepository pullRequestRepository; private readonly RepositoryRepository repositoryRepository; private readonly UpdateStatusRepository updateStatusRepository; private readonly BuildRepository buildRepository; - private readonly ILogger logger; - private bool IsRunning; - + public PullRequestUpdater(PullRequestRepository pullRequestRepository, RequestData requestData, RepositoryRepository repositoryRepository, - UpdateStatusRepository updateStatusRepository, BuildRepository buildRepository, ILogger logger) + UpdateStatusRepository updateStatusRepository, BuildRepository buildRepository, ILogger logger) : base(logger) { this.requestData = requestData; this.repositoryRepository = repositoryRepository; this.updateStatusRepository = updateStatusRepository; this.buildRepository = buildRepository; - this.logger = logger; this.pullRequestRepository = pullRequestRepository; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - - IsRunning = true; - try + ConcurrentBag allPullRequests = new ConcurrentBag(); + Parallel.ForEach(repositoryRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, repository => { - ConcurrentBag allPullRequests = new ConcurrentBag(); - Parallel.ForEach(repositoryRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, repository => + if (string.IsNullOrEmpty(repository.PullRequestUrl)) + return; + var pullRequests = GetAsync.FetchResponseList(requestData, repository.PullRequestUrl).Result; + if (pullRequests == null) + return; + Parallel.ForEach(pullRequests, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, pullRequest => { - if (string.IsNullOrEmpty(repository.PullRequestUrl)) - return; - var pullRequests = GetAsync.FetchResponseList(requestData, repository.PullRequestUrl).Result; - if (pullRequests == null) - return; - Parallel.ForEach(pullRequests, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, pullRequest => + try { - try + if (pullRequest.lastMergeCommit == null) { - if (pullRequest.lastMergeCommit == null) - { - logger.LogWarning($"Unable to get last merge commit for the pullrequest ({pullRequest.pullRequestId}) {pullRequest.description}"); - return; - } - - if (string.IsNullOrEmpty(pullRequest.lastMergeCommit.commitId)) - { - logger.LogWarning($"Unable to get the last commitID for the pull request ({pullRequest.pullRequestId}) {pullRequest.description}"); - return; - } - var build = buildRepository.GetBuildBySourceVersion(pullRequest.lastMergeCommit.commitId); - - - var pullRequestDto = BuildPullRequest(pullRequest, build); - pullRequestDto.Repository = repository; - pullRequestDto.Url = BuildPullRequestUrl(pullRequest, requestData.BaseAddress); - pullRequestDto.RequiredReviewers = repository.MinimumApproverCount; + logger.LogWarning($"Unable to get last merge commit for the pullrequest ({pullRequest.pullRequestId}) {pullRequest.description}"); + return; + } - foreach (var reviewer in pullRequest.reviewers) - { - // Container reviewers do not count - if (reviewer.isContainer) - continue; - if (reviewer.vote == (int) Vote.Approved) - pullRequestDto.AcceptedReviewers++; - } - allPullRequests.Add(pullRequestDto); + if (string.IsNullOrEmpty(pullRequest.lastMergeCommit.commitId)) + { + logger.LogWarning($"Unable to get the last commitID for the pull request ({pullRequest.pullRequestId}) {pullRequest.description}"); + return; } - catch (Exception e) + var build = buildRepository.GetBuildBySourceVersion(pullRequest.lastMergeCommit.commitId); + + + var pullRequestDto = BuildPullRequest(pullRequest, build); + pullRequestDto.Repository = repository; + pullRequestDto.Url = BuildPullRequestUrl(pullRequest, requestData.BaseAddress); + pullRequestDto.RequiredReviewers = repository.MinimumApproverCount; + + foreach (var reviewer in pullRequest.reviewers) { - logger.LogError("Error parsing pull request", e); + // Container reviewers do not count + if (reviewer.isContainer) + continue; + if (reviewer.vote == (int) Vote.Approved) + pullRequestDto.AcceptedReviewers++; } - }); + allPullRequests.Add(pullRequestDto); + } + catch (Exception e) + { + logger.LogError("Error parsing pull request", e); + } }); - var pullRequestsList = allPullRequests.ToList(); - pullRequestRepository.Update(pullRequestsList); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = pullRequestsList.Count, UpdaterName = nameof(PullRequestUpdater)}); + }); + var pullRequestsList = allPullRequests.ToList(); + pullRequestRepository.Update(pullRequestsList); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = pullRequestsList.Count, UpdaterName = nameof(PullRequestUpdater)}); - - } - catch (Exception ex) - { - throw new InvalidOperationException("Error runnign Pull Request Updater", ex); - } - finally - { - IsRunning = false; - } } private PullRequest BuildPullRequest(TFSAdvanced.Updater.Models.PullRequests.PullRequest x, Build build) diff --git a/src/TfsAdvanced.Updater/Tasks/ReleaseDefinitionUpdater.cs b/src/TfsAdvanced.Updater/Tasks/ReleaseDefinitionUpdater.cs index 8a74d4e..f8565d4 100644 --- a/src/TfsAdvanced.Updater/Tasks/ReleaseDefinitionUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/ReleaseDefinitionUpdater.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Hangfire; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TfsAdvanced.Models; using TfsAdvanced.Models.Infrastructure; @@ -12,57 +13,37 @@ namespace TFSAdvanced.Updater.Tasks { - public class ReleaseDefinitionUpdater + public class ReleaseDefinitionUpdater : UpdaterBase { private readonly ReleaseDefinitionRepository releaseDefinitionRepository; private readonly ProjectRepository projectRepository; private readonly RequestData requestData; - private bool IsRunning; - - public ReleaseDefinitionUpdater(ReleaseDefinitionRepository releaseDefinitionRepository, ProjectRepository projectRepository, RequestData requestData) + + public ReleaseDefinitionUpdater(ReleaseDefinitionRepository releaseDefinitionRepository, ProjectRepository projectRepository, RequestData requestData, ILogger logger) : base(logger) { this.releaseDefinitionRepository = releaseDefinitionRepository; this.projectRepository = projectRepository; this.requestData = requestData; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try + var releases = new ConcurrentStack(); + Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => { - var releases = new ConcurrentStack(); - Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => - { - var releaseDefinitions = GetAsync.FetchResponseList(requestData, $"{requestData.BaseReleaseManagerAddress}/{project.Id}/_apis/Release/definitions?api-version=3.0-preview.1").Result; - if (releaseDefinitions != null) + var releaseDefinitions = GetAsync.FetchResponseList(requestData, $"{requestData.BaseReleaseManagerAddress}/{project.Id}/_apis/Release/definitions?api-version=3.0-preview.1").Result; + if (releaseDefinitions != null) + { + Parallel.ForEach(releaseDefinitions, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, releaseDefinition => { - Parallel.ForEach(releaseDefinitions, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, releaseDefinition => - { - var populatedReleaseDefinition = GetAsync.Fetch(requestData, releaseDefinition.url).Result; - releases.Push(CreateReleaseDefinition(populatedReleaseDefinition)); - }); - } - }); - - releaseDefinitionRepository.Update(releases); - - } - catch (Exception ex) - { - throw new InvalidOperationException("Error processing the release definition updater.", ex); - } - finally - - { - IsRunning = false; - } - + var populatedReleaseDefinition = GetAsync.Fetch(requestData, releaseDefinition.url).Result; + releases.Push(CreateReleaseDefinition(populatedReleaseDefinition)); + }); + } + }); + releaseDefinitionRepository.Update(releases); } private ReleaseDefinition CreateReleaseDefinition(Models.Releases.ReleaseDefinition releaseDefinition) diff --git a/src/TfsAdvanced.Updater/Tasks/RepositoryUpdater.cs b/src/TfsAdvanced.Updater/Tasks/RepositoryUpdater.cs index f0b0556..102ea21 100644 --- a/src/TfsAdvanced.Updater/Tasks/RepositoryUpdater.cs +++ b/src/TfsAdvanced.Updater/Tasks/RepositoryUpdater.cs @@ -3,25 +3,25 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Hangfire; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TfsAdvanced.Models; using TfsAdvanced.Models.Infrastructure; using TFSAdvanced.Models.DTO; using TFSAdvanced.Updater.Models.Policy; +using TFSAdvanced.Updater.Tasks; namespace TfsAdvanced.Updater.Tasks { - public class RepositoryUpdater + public class RepositoryUpdater : UpdaterBase { private readonly ProjectRepository projectRepository; private readonly RepositoryRepository repositoryRepository; private readonly UpdateStatusRepository updateStatusRepository; private readonly RequestData requestData; - private bool IsRunning; - + public RepositoryUpdater(ProjectRepository projectRepository, RequestData requestData, RepositoryRepository repositoryRepository, - UpdateStatusRepository updateStatusRepository) + UpdateStatusRepository updateStatusRepository, ILogger logger) : base(logger) { this.projectRepository = projectRepository; this.requestData = requestData; @@ -29,65 +29,52 @@ public RepositoryUpdater(ProjectRepository projectRepository, RequestData reques this.updateStatusRepository = updateStatusRepository; } - [AutomaticRetry(Attempts = 0)] - public void Update() + protected override void Update() { - if (IsRunning) - return; - IsRunning = true; - try + ConcurrentBag populatedRepositories = new ConcurrentBag(); + Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => { - ConcurrentBag populatedRepositories = new ConcurrentBag(); - Parallel.ForEach(projectRepository.GetAll(), new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, project => + IList repositories = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0").Result; + if (repositories == null) + return; + Parallel.ForEach(repositories, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, repo => { - IList repositories = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories?api=1.0").Result; - if (repositories == null) - return; - Parallel.ForEach(repositories, new ParallelOptions {MaxDegreeOfParallelism = AppSettings.MAX_DEGREE_OF_PARALLELISM}, repo => + try { - try - { - var populatedRepository = GetAsync.Fetch(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result; + var populatedRepository = GetAsync.Fetch(requestData, $"{requestData.BaseAddress}/{project.Name}/_apis/git/repositories/{repo.name}?api=1.0").Result; - var repositoryDto = new Repository + var repositoryDto = new Repository + { + Id = populatedRepository.id, + Name = populatedRepository.name, + PullRequestUrl = populatedRepository._links.pullRequests.href, + Url = populatedRepository.remoteUrl, + Project = new Project { - Id = populatedRepository.id, - Name = populatedRepository.name, - PullRequestUrl = populatedRepository._links.pullRequests.href, - Url = populatedRepository.remoteUrl, - Project = new Project - { - Id = populatedRepository.project.id, - Name = populatedRepository.project.name, - Url = populatedRepository.project.url - } - }; - var policyConfigurations = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1").Result; + Id = populatedRepository.project.id, + Name = populatedRepository.project.name, + Url = populatedRepository.project.url + } + }; + var policyConfigurations = GetAsync.FetchResponseList(requestData, $"{requestData.BaseAddress}/defaultcollection/{project.Id}/_apis/policy/configurations?api-version=2.0-preview.1").Result; - foreach (var configuration in policyConfigurations) + foreach (var configuration in policyConfigurations) + { + if (configuration.type.displayName == "Minimum number of reviewers") { - if (configuration.type.displayName == "Minimum number of reviewers") - { - repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount; - } + repositoryDto.MinimumApproverCount = configuration.settings.minimumApproverCount; } - populatedRepositories.Add(repositoryDto); } - catch (Exception){} - }); + populatedRepositories.Add(repositoryDto); + } + catch (Exception) + { + } }); - var repositoryList = populatedRepositories.ToList(); - repositoryRepository.Update(repositoryList); - updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater)}); - } - catch (Exception ex) - { - throw new InvalidOperationException("Error running repository updater", ex); - } - finally - { - IsRunning = false; - } + }); + var repositoryList = populatedRepositories.ToList(); + repositoryRepository.Update(repositoryList); + updateStatusRepository.UpdateStatus(new UpdateStatus {LastUpdate = DateTime.Now, UpdatedRecords = repositoryList.Count, UpdaterName = nameof(RepositoryUpdater)}); } } } diff --git a/src/TfsAdvanced.Updater/Tasks/Updater.cs b/src/TfsAdvanced.Updater/Tasks/Updater.cs index 1e13306..0b79789 100644 --- a/src/TfsAdvanced.Updater/Tasks/Updater.cs +++ b/src/TfsAdvanced.Updater/Tasks/Updater.cs @@ -2,6 +2,7 @@ using System.Threading; using Hangfire; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using TfsAdvanced.DataStore.Repository; using TFSAdvanced.Updater.Tasks; @@ -9,65 +10,73 @@ namespace TfsAdvanced.Updater.Tasks { public class Updater { + private readonly ILogger logger; private Timer fiveSecondTimer; private Timer thirtySecondTimer; private readonly IServiceProvider serviceProvider; - public Updater(IServiceProvider serviceProvider) + public Updater(IServiceProvider serviceProvider, ILogger logger) { + this.logger = logger; this.serviceProvider = serviceProvider; } public void Start() { + + logger.LogInformation("Starting bootstrapping app"); + DateTime startTime = DateTime.Now; // Initialize the updaters in order - double stepSize = 1.0/7.0; + double stepSize = 1.0/8.0; double percentLoaded = 0.0; var hangFireStatusRepository = serviceProvider.GetService(); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; - hangFireStatusRepository.SetPercentLoaded(percentLoaded); - serviceProvider.GetService().Update(); - percentLoaded += stepSize; + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); + percentLoaded = RunUpdate(hangFireStatusRepository, percentLoaded, stepSize); hangFireStatusRepository.SetPercentLoaded(1); hangFireStatusRepository.SetIsLoaded(true); // Slow moving things only need to be updated once an hour - RecurringJob.AddOrUpdate(updater => updater.Update(), Cron.Hourly); - RecurringJob.AddOrUpdate(updater => updater.Update(), Cron.Hourly); - RecurringJob.AddOrUpdate(updater => updater.Update(), Cron.Hourly); + ScheduleJob(Cron.Hourly()); + ScheduleJob(Cron.Hourly()); + ScheduleJob(Cron.Hourly()); thirtySecondTimer = new Timer(state => { - BackgroundJob.Enqueue(updater => updater.Update()); - BackgroundJob.Enqueue(updater => updater.Update()); - BackgroundJob.Enqueue(updater => updater.Update()); + EnqueueJob(); + EnqueueJob(); + EnqueueJob(); }, null, TimeSpan.Zero, TimeSpan.FromSeconds(30)); fiveSecondTimer = new Timer(state => { - BackgroundJob.Enqueue(updater => updater.Update()); - BackgroundJob.Enqueue(updater => updater.Update()); + EnqueueJob(); + EnqueueJob(); }, null, TimeSpan.Zero, TimeSpan.FromSeconds(5)); + logger.LogInformation($"Finished bootstrapping app in {DateTime.Now-startTime:g}"); + } + + private double RunUpdate(HangFireStatusRepository hangFireStatusRepository, double percentLoaded, double stepSize) where T : UpdaterBase + { + hangFireStatusRepository.SetPercentLoaded(percentLoaded); + serviceProvider.GetService().Run(); + return percentLoaded + stepSize; + } + + private void ScheduleJob(string cron) where T : UpdaterBase + { + RecurringJob.AddOrUpdate((T updater) => updater.Run(), cron); + } + + private void EnqueueJob() where T : UpdaterBase + { + BackgroundJob.Enqueue((T updater) => updater.Run()); } public void Stop() diff --git a/src/TfsAdvanced.Updater/Tasks/UpdaterBase.cs b/src/TfsAdvanced.Updater/Tasks/UpdaterBase.cs new file mode 100644 index 0000000..c1c33c0 --- /dev/null +++ b/src/TfsAdvanced.Updater/Tasks/UpdaterBase.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Hangfire; +using Microsoft.Extensions.Logging; + +namespace TFSAdvanced.Updater.Tasks +{ + public abstract class UpdaterBase + { + protected readonly ILogger logger; + private bool isRunning; + + protected UpdaterBase(ILogger logger) + { + this.logger = logger; + } + + [AutomaticRetry(Attempts = 0)] + public void Run() + { + var className = GetType().Name; + + if (isRunning) + { + logger.LogInformation($"Skipping scheduled run for {className} because it is currently running."); + return; + } + + isRunning = true; + + logger.LogInformation($"Starting {className}"); + var start = DateTime.Now; + try + { + Update(); + } + catch (Exception e) + { + logger.LogError($"Error running update for {className}.", e); + } + logger.LogInformation($"Finished Running {className} {DateTime.Now-start:g}"); + + isRunning = false; + } + + + protected abstract void Update(); + } +}