diff --git a/OsmNightWatch.Lib/IssuesData.cs b/OsmNightWatch.Lib/IssuesData.cs index 684ee49..e1935bd 100644 --- a/OsmNightWatch.Lib/IssuesData.cs +++ b/OsmNightWatch.Lib/IssuesData.cs @@ -1,94 +1,91 @@ -using System.Collections.Generic; - -namespace OsmNightWatch.Lib -{ - public class IssuesData - { - public static HashSet LastKnownGoodIssueTypes { get; } = new HashSet() - { - "OpenAdminPolygon", - //"OpenAdminPolygon7", - //"OpenAdminPolygon8", - //"OpenAdminPolygon9", - //"OpenAdminPolygon10", - "BrokenCoastLine" - }; - - public DateTime DateTime { get; set; } - public int MinutelySequenceNumber { get; set; } - - public DateTime LastKnownGoodDateTime { get; set; } - public int LastKnownGoodMinutelySequenceNumber { get; set; } - - public List AllIssues { get; init; } = new List(); - - public void SetTimestampsAndLastKnownGood(IssuesData? oldIssuesData) - { - var hashSet = oldIssuesData == null ? new HashSet() : new HashSet(oldIssuesData.AllIssues); - bool canBeLastKnownGood = true; - foreach (var issue in AllIssues) - { - // Check if issue is in last known good category... - if (LastKnownGoodIssueTypes.Contains(issue.IssueType)) - { - canBeLastKnownGood = false; - } - - if (hashSet.TryGetValue(issue, out var oldIssue)) - { - issue.FirstTimeSeen = oldIssue.FirstTimeSeen; - } - else - { - issue.FirstTimeSeen = DateTime; - } - } - if (canBeLastKnownGood) - { - LastKnownGoodDateTime = DateTime; - LastKnownGoodMinutelySequenceNumber = MinutelySequenceNumber; - } - else - { - LastKnownGoodDateTime = oldIssuesData?.LastKnownGoodDateTime ?? default; - LastKnownGoodMinutelySequenceNumber = oldIssuesData?.LastKnownGoodMinutelySequenceNumber ?? -1; - } - } - } - - public class IssueData - { - public string FriendlyName { get; set; } - - /// - /// The issue type, e.g: OpenAdminPolygon, BrokenCoastLine... - /// - public string IssueType { get; set; } - - public string OsmType { get; set; } - - public long OsmId { get; set; } - - public string Details { get; set; } - - public DateTime? FirstTimeSeen { get; set; } - - public override int GetHashCode() - { - return HashCode.Combine( - IssueType?.GetHashCode(), - OsmType?.GetHashCode(), - OsmId.GetHashCode(), - Details?.GetHashCode()); - } - - public override bool Equals(object? obj) - { - if (obj is not IssueData other) return false; - return other.IssueType == IssueType && - other.OsmType == OsmType && - other.OsmId == OsmId && - other.Details == Details; - } - } +using System.Collections.Generic; + +namespace OsmNightWatch.Lib +{ + public class IssuesData + { + public static HashSet LastKnownGoodIssueTypes { get; } = new HashSet() + { + "OpenAdminPolygon", + "BrokenCoastLine", + "AdminsState" + }; + + public DateTime DateTime { get; set; } + public int MinutelySequenceNumber { get; set; } + + public DateTime LastKnownGoodDateTime { get; set; } + public int LastKnownGoodMinutelySequenceNumber { get; set; } + + public List AllIssues { get; init; } = new List(); + + public void SetTimestampsAndLastKnownGood(IssuesData? oldIssuesData) + { + var hashSet = oldIssuesData == null ? new HashSet() : new HashSet(oldIssuesData.AllIssues); + bool canBeLastKnownGood = true; + foreach (var issue in AllIssues) + { + // Check if issue is in last known good category... + if (LastKnownGoodIssueTypes.Contains(issue.IssueType)) + { + canBeLastKnownGood = false; + } + + if (hashSet.TryGetValue(issue, out var oldIssue)) + { + issue.FirstTimeSeen = oldIssue.FirstTimeSeen; + } + else + { + issue.FirstTimeSeen = DateTime; + } + } + if (canBeLastKnownGood) + { + LastKnownGoodDateTime = DateTime; + LastKnownGoodMinutelySequenceNumber = MinutelySequenceNumber; + } + else + { + LastKnownGoodDateTime = oldIssuesData?.LastKnownGoodDateTime ?? default; + LastKnownGoodMinutelySequenceNumber = oldIssuesData?.LastKnownGoodMinutelySequenceNumber ?? -1; + } + } + } + + public class IssueData + { + public string FriendlyName { get; set; } + + /// + /// The issue type, e.g: OpenAdminPolygon, BrokenCoastLine... + /// + public string IssueType { get; set; } + + public string OsmType { get; set; } + + public long OsmId { get; set; } + + public string Details { get; set; } + + public DateTime? FirstTimeSeen { get; set; } + + public override int GetHashCode() + { + return HashCode.Combine( + IssueType?.GetHashCode(), + OsmType?.GetHashCode(), + OsmId.GetHashCode(), + Details?.GetHashCode()); + } + + public override bool Equals(object? obj) + { + if (obj is not IssueData other) return false; + return other.IssueType == IssueType && + other.OsmType == OsmType && + other.OsmId == OsmId && + other.Details == Details; + } + } } \ No newline at end of file diff --git a/OsmNightWatch.Web/Helper.cs b/OsmNightWatch.Web/Helper.cs index 1f0c7cc..baf7639 100644 --- a/OsmNightWatch.Web/Helper.cs +++ b/OsmNightWatch.Web/Helper.cs @@ -12,9 +12,10 @@ public class Helper //("OpenAdminPolygon10", "Open admins level 10.","/OpenAdminPolygons/10" ), ("BrokenCoastLine", "Broken coastlines","/BrokenWaterCoastlines" ), ("AdminsState", "Admins State","/AdminsState" ), + ("AdminCentre","Admin Centres", "/AdminCentre" ), ("MissingWays","Admins without ways", "/AdminsWithoutWays" ), ("ImportantFeatures","Important Features", "/ImportantFeatures" ) - + }; public static string ConvertToFullOsmType(string osmType) diff --git a/OsmNightWatch.Web/Pages/AdminCentre.razor b/OsmNightWatch.Web/Pages/AdminCentre.razor new file mode 100644 index 0000000..aef34a1 --- /dev/null +++ b/OsmNightWatch.Web/Pages/AdminCentre.razor @@ -0,0 +1,77 @@ +@page "/AdminCentre" + +Admin Centre + +@inject IJSRuntime JSRuntime +@using OsmNightWatch.Web.Data +@using OsmNightWatch.Lib +@inject IssuesDataService IssuesDataService + +

Ensures that admins have admin_centre that has place tag.

+ +@if (issuesData == null) +{ +

Loading...

+} +else +{ + + + + + + + + + + + @foreach (var issuesByDay in issuesData.AllIssues.Where(issue => issue.IssueType == "AdminCentre").GroupBy(i => i.FirstTimeSeen.Value.Date).OrderByDescending(ig => ig.Key)) + { + + + + + + + foreach (var issue in issuesByDay.OrderByDescending(i => i.FirstTimeSeen).ThenBy(i => i.OsmType).ThenBy(i => i.OsmId)) + { + + + + + + + } + } + +
Day/Name of relationLinksFirst time seenMore details
@issuesByDay.Key.ToString("yyyy-MM-dd - dddd")
@issue.OsmType.ToUpper()[0]@issue.OsmId - @issue.FriendlyName + RA + JOSM + @issue.FirstTimeSeen!.Value.ToString("yyyy-MM-dd HH:mm:ss")@issue.Details
+} + +@code { + [Parameter] + public int? AdminLevel { get; set; } + + public long lastClickedOsmId = -1; + + private IssuesData? issuesData; + + protected override async Task OnInitializedAsync() + { + issuesData = await IssuesDataService.GetIssuesDataAsync(); + } + + public async Task HighligthMe(long osmId) + { + lastClickedOsmId = osmId; + this.StateHasChanged(); + } + + public async Task OpenJosm(long osmId) + { + await JSRuntime.InvokeVoidAsync("CallJosm.Invoke", $"http://localhost:8111/load_object?new_layer=true&objects=r{osmId}&relation_members=true"); + await HighligthMe(osmId); + } +} diff --git a/OsmNightWatch.sln b/OsmNightWatch.sln index f2f7cd9..4f12a24 100644 --- a/OsmNightWatch.sln +++ b/OsmNightWatch.sln @@ -1,43 +1,43 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.3.32507.5 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch", "OsmNightWatch\OsmNightWatch.csproj", "{4C392BB0-4E33-47A3-B56D-2D3B02F368C0}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch.Web", "OsmNightWatch.Web\OsmNightWatch.Web.csproj", "{59939C38-3AC9-489A-9FFF-1B7A028CA0A7}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch.Lib", "OsmNightWatch.Lib\OsmNightWatch.Lib.csproj", "{3E802EBA-A872-42B2-A54B-6632152288A2}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PbfParser", "PbfParser\PbfParser.csproj", "{E83ECE87-C64E-42B0-9817-4E72AEB63C47}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Release|Any CPU.Build.0 = Release|Any CPU - {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Release|Any CPU.Build.0 = Release|Any CPU - {3E802EBA-A872-42B2-A54B-6632152288A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3E802EBA-A872-42B2-A54B-6632152288A2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3E802EBA-A872-42B2-A54B-6632152288A2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3E802EBA-A872-42B2-A54B-6632152288A2}.Release|Any CPU.Build.0 = Release|Any CPU - {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {5DB192A9-3C95-4FF3-AAD0-EB37D9974B4B} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.3.32507.5 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch", "OsmNightWatch\OsmNightWatch.csproj", "{4C392BB0-4E33-47A3-B56D-2D3B02F368C0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch.Web", "OsmNightWatch.Web\OsmNightWatch.Web.csproj", "{59939C38-3AC9-489A-9FFF-1B7A028CA0A7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OsmNightWatch.Lib", "OsmNightWatch.Lib\OsmNightWatch.Lib.csproj", "{3E802EBA-A872-42B2-A54B-6632152288A2}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PbfParser", "PbfParser\PbfParser.csproj", "{E83ECE87-C64E-42B0-9817-4E72AEB63C47}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C392BB0-4E33-47A3-B56D-2D3B02F368C0}.Release|Any CPU.Build.0 = Release|Any CPU + {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {59939C38-3AC9-489A-9FFF-1B7A028CA0A7}.Release|Any CPU.Build.0 = Release|Any CPU + {3E802EBA-A872-42B2-A54B-6632152288A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E802EBA-A872-42B2-A54B-6632152288A2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E802EBA-A872-42B2-A54B-6632152288A2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E802EBA-A872-42B2-A54B-6632152288A2}.Release|Any CPU.Build.0 = Release|Any CPU + {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E83ECE87-C64E-42B0-9817-4E72AEB63C47}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5DB192A9-3C95-4FF3-AAD0-EB37D9974B4B} + EndGlobalSection +EndGlobal diff --git a/OsmNightWatch/Analyzers/AdminCountPerCountry/AdminCountPerCountryAnalyzer.cs b/OsmNightWatch/Analyzers/AdminCountPerCountry/AdminCountPerCountryAnalyzer.cs index 4187c12..e4e9740 100644 --- a/OsmNightWatch/Analyzers/AdminCountPerCountry/AdminCountPerCountryAnalyzer.cs +++ b/OsmNightWatch/Analyzers/AdminCountPerCountry/AdminCountPerCountryAnalyzer.cs @@ -104,6 +104,10 @@ public void EmptyUpsertAdmins(object param) private IEnumerable UpdateRelations((uint Id, Relation Relation)[] relevantThings, IOsmGeoSource newOsmSource) { + var expectedState = GetExpectedState(); + var relationsToCheckAdminCenter = new HashSet(expectedState.Countries.SelectMany(c => c.Admins.OrderBy(a => a.Key).FirstOrDefault().Value ?? [])); + expectedState.Countries.ForEach(c => relationsToCheckAdminCenter.Add(c.RelationId)); + StartAdminsUpsertTransaction(); try { @@ -117,7 +121,7 @@ private IEnumerable UpdateRelations((uint Id, Relation Relation)[] re return; } var result = BuildPolygonFromRelation.BuildPolygon(admin.Relation, newOsmSource); - database.RelationChangesTracker.AddRelationToTrack(admin.Id, result.Ways); + database.RelationChangesTracker.AddRelationToTrack(admin.Id, result.Ways, admin.Relation.Members.Where(m => m.Type == OsmGeoType.Node).Select(m => m.Id)); NetTopologySuite.Geometries.Geometry? geom; if (result.reason != null) { @@ -137,6 +141,12 @@ private IEnumerable UpdateRelations((uint Id, Relation Relation)[] re { geom = result.Polygon; } + + if (string.IsNullOrEmpty(result.reason) && relationsToCheckAdminCenter.Contains(admin.Id) && CheckAdminCentre(admin.Relation, newOsmSource) is string adminCentreReason) + { + result.reason = adminCentreReason; + } + if (!admin.Relation.Tags.TryGetValue("name:en", out var friendlyName)) { if (!admin.Relation.Tags.TryGetValue("name", out friendlyName)) @@ -154,8 +164,6 @@ private IEnumerable UpdateRelations((uint Id, Relation Relation)[] re } EndAdminsUpsertTransaction(true); - var expectedState = GetExpectedState(); - if (currentState == null) { currentState = CreateCurrentState(expectedState!); @@ -165,6 +173,20 @@ private IEnumerable UpdateRelations((uint Id, Relation Relation)[] re currentState = UpdateCurrentState(currentState, relevantThings); } + foreach (var (relationId, name, adminLevel, reason) in GetAdminsWithReason()) + { + if (reason.Contains("admin_centre")) + { + yield return new IssueData() { + IssueType = "AdminCentre", + FriendlyName = name, + OsmType = "R", + OsmId = relationId, + Details = reason + }; + } + } + foreach (var (relationId, name, adminLevel, reason) in GetBrokenAdmins()) { if (reason == "Missing ways") @@ -238,6 +260,39 @@ private IEnumerable UpdateRelations((uint Id, Relation Relation)[] re } } + private string? CheckAdminCentre(Relation relation, IOsmGeoSource newOsmSource) + { + var adminCentreCount = relation.Members.Count(m => m.Role == "admin_centre"); + if (adminCentreCount == 0) + { + return "Missing admin_centre role"; + } + if (adminCentreCount > 1) + { + return "Too many admin_centre role roles"; + } + var member = relation.Members.First(m => m.Role == "admin_centre"); + if (member.Type != OsmGeoType.Node) + { + return "admin_centre member is not Node"; + } + var node = newOsmSource.GetNode(member.Id); + if (!(node.Tags?.TryGetValue("place", out var placeValue) ?? false)) + return $"admin_centre node({member.Id}) does not have 'place' tag."; + //switch (placeValue) + //{ + // case "town": + // case "village": + // case "city": + // case "suburb": + // return null; + + // default: + // return $"admin_centre node has place value set to {placeValue}, but must be 'village', 'town' or 'city'."; + //} + return null; + } + private StateOfTheAdmins? cachedExpectedState = null; private EntityTagHeaderValue? latestEtag; @@ -282,6 +337,9 @@ public IEnumerable ProcessChangeset(MergedChangeset changeSet, IOsmGe changedRelations.Add((uint)relation.Id); } } + + var relevantThings = newOsmSource.BatchLoad(relationIds: new HashSet(changedRelations.Select(r => (long)r))).relations; + Utils.BatchLoad(relevantThings, newOsmSource, true, true); //changedRelations.Add(3286892);// for testing specific relation build polygon logic return UpdateRelations(changedRelations.Select(id => (id, newOsmSource.GetRelation(id))).ToArray(), newOsmSource); } @@ -524,6 +582,19 @@ public void UpsertAdmin(long id, string friendlyName, int adminLevel, Geometry? } } } + public IEnumerable<(long RelationId, string name, int adminLevel, string reason)> GetAdminsWithReason() + { + using (var comm = sqlConnection.CreateCommand("SELECT id, friendlyname, adminlevel, reason FROM admins WHERE reason IS NOT NULL")) + { + using var reader = comm.ExecuteReader(); + { + while (reader.Read()) + { + yield return (reader.GetInt64(0), reader.GetString(1), reader.GetInt32(2), reader.GetString(3)); + } + } + } + } public bool DoesCountryExist(long relationId) { @@ -541,9 +612,9 @@ public bool DoesCountryExist(long relationId) } } - public List GetCountryAdmins(long relationId, int adminLevel) + public List GetCountryAdmins(long relationId, int adminLevel) { - var result = new List(); + var result = new List(); byte[] buffer = new byte[15 * 1024 * 1024]; var gaiaReader = new GaiaGeoReader(); PreparedPolygon polygon = null; @@ -594,11 +665,11 @@ public List GetCountryAdmins(long relationId, int adminLevel) { if (polygon.Contains(adminGeometry)) { - result.Add(reader.GetInt64(0)); + result.Add((uint)reader.GetInt64(0)); } else if (polygon.Overlaps(adminGeometry)) { - result.Add(reader.GetInt64(0)); + result.Add((uint)reader.GetInt64(0)); } } } diff --git a/OsmNightWatch/Analyzers/AdminCountPerCountry/Country.cs b/OsmNightWatch/Analyzers/AdminCountPerCountry/Country.cs index 0fe1f4a..1823ace 100644 --- a/OsmNightWatch/Analyzers/AdminCountPerCountry/Country.cs +++ b/OsmNightWatch/Analyzers/AdminCountPerCountry/Country.cs @@ -10,7 +10,7 @@ public class Country public string EnglishName { get; set; } public string Iso2 { get; set; } public string Iso3 { get; set; } - public Dictionary> Admins { get; set; } = new(); + public Dictionary> Admins { get; set; } = new(); [JsonIgnore] public PreparedPolygon Polygon { get; set; } [JsonIgnore] diff --git a/OsmNightWatch/Analyzers/AdminCountPerCountry/RelationChangesTracker.cs b/OsmNightWatch/Analyzers/AdminCountPerCountry/RelationChangesTracker.cs index ee94761..2b2a39f 100644 --- a/OsmNightWatch/Analyzers/AdminCountPerCountry/RelationChangesTracker.cs +++ b/OsmNightWatch/Analyzers/AdminCountPerCountry/RelationChangesTracker.cs @@ -6,6 +6,7 @@ namespace OsmNightWatch.Analyzers.AdminCountPerCountry; public class RelationChangesTracker { public Dictionary> NodeToWay = new(); + public Dictionary> NodeToRelation = new(); public Dictionary> WayToRelation = new(); public HashSet Relations = new(); public HashSet TrackedRelations = new(); @@ -16,7 +17,7 @@ public RelationChangesTracker(KeyValueDatabase database) this.database = database; } - public void AddRelationToTrack(uint relationId, List ways) + public void AddRelationToTrack(uint relationId, List ways, IEnumerable nodes) { lock (Relations) { @@ -44,6 +45,18 @@ public void AddRelationToTrack(uint relationId, List ways) } } } + + foreach (var nodeId in nodes) + { + if (NodeToRelation.TryGetValue(nodeId, out var nodeRelations)) + { + nodeRelations.Add(relationId); + } + else + { + NodeToRelation.Add(nodeId, new HashSet() { relationId }); + } + } } } @@ -100,6 +113,15 @@ public HashSet GetChangedRelations(MergedChangeset changeSet) foreach (var wayId in waysSet) { database.GetWayToRelation(wayId, result); + } + + foreach (var node in changeSet.OsmNodes) + { + if (node.Value == null || node.Value.Version < 2) + { + continue; + } + database.GetNodeToRelation(node.Key, result); } foreach (var relation in changeSet.Relations.Keys) diff --git a/OsmNightWatch/Analyzers/BrokenCoastline/BrokenCoastlineAnalyzer.cs b/OsmNightWatch/Analyzers/BrokenCoastline/BrokenCoastlineAnalyzer.cs index 8513af6..b94f43a 100644 --- a/OsmNightWatch/Analyzers/BrokenCoastline/BrokenCoastlineAnalyzer.cs +++ b/OsmNightWatch/Analyzers/BrokenCoastline/BrokenCoastlineAnalyzer.cs @@ -284,8 +284,7 @@ public IEnumerable ProcessChangeset(MergedChangeset changeSet, IOsmGe } } database.RelationChangesTracker.AddWaysToTrack(coastlinesToUpsert); - // I think since we are fetching QueryAllCoastlinesWithCrossesReasonInDatabase this is not needed anymore - //ExpandListOfAllCoastlinesWithCoastlinesIntersectingBbox(waysToCheck); + Utils.BatchLoad(coastlinesToUpsert, newOsmSource, true, true); using (var transaction = sqlConnection.BeginTransaction()) { foreach (var wayId in coastlinesToDelete) @@ -305,22 +304,6 @@ public IEnumerable ProcessChangeset(MergedChangeset changeSet, IOsmGe return ProcessAllWays(newOsmSource); } - private void ExpandListOfAllCoastlinesWithCoastlinesIntersectingBbox(HashSet waysToCheck) - { - using (var comm = sqlConnection.CreateCommand(@$"SELECT searchedCoastline.id FROM Coastlines searchedCoastline, Coastlines modifiedCoastline WHERE - modifiedCoastline.id IN ({string.Join("", "", waysToCheck)}) AND - searchedCoastline.rowid IN (SELECT ROWID FROM SpatialIndex WHERE f_table_name='Coastlines' AND search_frame=modifiedCoastline.geom);")) - { - using var reader = comm.ExecuteReader(); - { - while (reader.Read()) - { - waysToCheck.Add((uint)reader.GetInt32(0)); - } - } - } - } - public IEnumerable GetProblematicCoastlines() { using (var comm = sqlConnection.CreateCommand("SELECT id, reason FROM coastlines WHERE reason IS NOT NULL")) diff --git a/OsmNightWatch/KeyValueDatabase.cs b/OsmNightWatch/KeyValueDatabase.cs index f82b891..e914fab 100644 --- a/OsmNightWatch/KeyValueDatabase.cs +++ b/OsmNightWatch/KeyValueDatabase.cs @@ -182,6 +182,18 @@ public void WriteRelationChangesTracker(RelationChangesTracker tracker) } tx.Put(db, keyBuffer, buffer[..^span.Length]); } + db = OpenDb(tx, "Tracker_NodeToRelation"); + keyBuffer = stackalloc byte[8]; + foreach (var relations in tracker.NodeToRelation.OrderBy(n => n.Key)) + { + BinaryPrimitives.WriteInt64BigEndian(keyBuffer, relations.Key); + var span = buffer; + foreach (var relationId in relations.Value) + { + BinSerialize.WriteUInt(ref span, relationId); + } + tx.Put(db, keyBuffer, buffer[..^span.Length]); + } db = OpenDb(tx, "Tracker_Relations"); keyBuffer = stackalloc byte[4]; foreach (var relation in tracker.Relations.OrderBy(n => n)) @@ -218,6 +230,19 @@ public void WriteRelationChangesTracker(RelationChangesTracker tracker) } tx.Put(db, keyBuffer, buffer[..^span.Length]); } + db = OpenDb(tx, "Tracker_NodeToRelation"); + keyBuffer = stackalloc byte[8]; + foreach (var relations in tracker.NodeToRelation) + { + BinaryPrimitives.WriteInt64BigEndian(keyBuffer, relations.Key); + GetNodeToRelation(relations.Key, relations.Value); + var span = buffer; + foreach (var relationId in relations.Value) + { + BinSerialize.WriteUInt(ref span, relationId); + } + tx.Put(db, keyBuffer, buffer[..^span.Length]); + } db = OpenDb(tx, "Tracker_Relations"); keyBuffer = stackalloc byte[4]; foreach (var relation in tracker.Relations) @@ -444,6 +469,27 @@ public void Dispose() dbEnv.Dispose(); } + public void GetNodeToRelation(long nodeId, HashSet relations) + { + if (transaction is not LightningTransaction tx) + { + throw new InvalidOperationException("Transaction not started!"); + } + var db = OpenDb(tx, "Tracker_NodeToRelation"); + Span keyBuffer = stackalloc byte[8]; + BinaryPrimitives.WriteInt64BigEndian(keyBuffer, nodeId); + + var read = tx.Get(db, keyBuffer); + if (read.resultCode == MDBResultCode.Success) + { + var buffer = read.value.AsSpan(); + while (buffer.Length > 0) + { + relations.Add(BinSerialize.ReadUInt(ref buffer)); + } + } + } + public void GetNodeToWay(long nodeId, HashSet ways) { if (transaction is not LightningTransaction tx) diff --git a/OsmNightWatch/OsmDatabaseWithReplicationData.cs b/OsmNightWatch/OsmDatabaseWithReplicationData.cs index 21b8a2f..43bf2c6 100644 --- a/OsmNightWatch/OsmDatabaseWithReplicationData.cs +++ b/OsmNightWatch/OsmDatabaseWithReplicationData.cs @@ -98,23 +98,75 @@ public OsmGeo Get(OsmGeoType type, long id) public (IReadOnlyCollection nodes, IReadOnlyCollection ways, IReadOnlyCollection relations) BatchLoad(HashSet? nodeIds, HashSet? wayIds, HashSet? relationIds) { - var (nodes, ways, relations) = baseSource.BatchLoad(nodeIds, wayIds, relationIds); - foreach (var node in nodes) - { - Put(node); + var nodes = new List(); + if (nodeIds != null) + { + foreach (var nodeId in nodeIds.ToArray()) + { + if (changesetNodes.TryGetValue(nodeId, out var node)) + { + nodeIds.Remove(nodeId); + if (node != null) + nodes.Add(node); + } + else if (keyValueDatabase.TryGetNode(nodeId, out var nodeFromDb)) + { + nodeIds.Remove(nodeId); + if (nodeFromDb != null) + nodes.Add(nodeFromDb); + } + } } - foreach (var way in ways) - { - Put(way); + var ways = new List(); + if (wayIds != null) + { + foreach (var wayId in wayIds.ToArray()) + { + if (changesetWays.TryGetValue(wayId, out var way)) + { + wayIds.Remove(wayId); + if (way != null) + ways.Add(way); + } + else if (keyValueDatabase.TryGetWay(wayId, out var wayFromDb)) + { + wayIds.Remove(wayId); + if (wayFromDb != null) + ways.Add(wayFromDb); + } + } } - foreach (var relation in relations) - { - Put(relation); + var relations = new List(); + if (relationIds != null) + { + foreach (var relationId in relationIds.ToArray()) + { + if (changesetRelations.TryGetValue(relationId, out var relation)) + { + relationIds.Remove(relationId); + if (relation != null) + relations.Add(relation); + } + else if (keyValueDatabase.TryGetRelation(relationId, out var relationFromDb)) + { + relationIds.Remove(relationId); + if (relationFromDb != null) + relations.Add(relationFromDb); + } + } } - + var baseResults = baseSource.BatchLoad(nodeIds, wayIds, relationIds); + nodes.AddRange(baseResults.nodes); + ways.AddRange(baseResults.ways); + relations.AddRange(baseResults.relations); return (nodes, ways, relations); } + public void ClearBatchCache() + { + baseSource.ClearBatchCache(); + } + public void StoreCache() { keyValueDatabase.UpdateNodes(changesetNodes, true); diff --git a/OsmNightWatch/OsmNightWatch.csproj b/OsmNightWatch/OsmNightWatch.csproj index 6e0fd5c..9ae4143 100644 --- a/OsmNightWatch/OsmNightWatch.csproj +++ b/OsmNightWatch/OsmNightWatch.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 enable enable True diff --git a/OsmNightWatch/PbfDatabase.cs b/OsmNightWatch/PbfDatabase.cs index a4a813f..5a9e8fd 100644 --- a/OsmNightWatch/PbfDatabase.cs +++ b/OsmNightWatch/PbfDatabase.cs @@ -6,7 +6,9 @@ internal class PbfDatabase : IOsmValidateSource { private PbfIndex index; - + Dictionary nodesCache = new(); + Dictionary waysCache = new(); + Dictionary relationsCache = new(); public PbfDatabase(PbfIndex index) { @@ -15,7 +17,26 @@ public PbfDatabase(PbfIndex index) public (IReadOnlyCollection nodes, IReadOnlyCollection ways, IReadOnlyCollection relations) BatchLoad(HashSet? nodeIds = null, HashSet? wayIds = null, HashSet? relationIds = null) { - return (NodesParser.LoadNodes(nodeIds, index), WaysParser.LoadWays(wayIds, index), RelationsParser.LoadRelations(relationIds, index)); + //TODO: Optimize this, by passing just nodeIds/wayIds/relationIds that are not already in cache + var parsedNodes = NodesParser.LoadNodes(nodeIds, index); + foreach (var node in parsedNodes) + { + nodesCache[node.Id] = node; + nodeIds!.Remove(node.Id); + } + var parsedWays = WaysParser.LoadWays(wayIds, index); + foreach (var way in parsedWays) + { + waysCache[way.Id] = way; + wayIds!.Remove(way.Id); + } + var parsedRelations = RelationsParser.LoadRelations(relationIds, index); + foreach (var relation in parsedRelations) + { + relationsCache[relation.Id] = relation; + relationIds!.Remove(relation.Id); + } + return (parsedNodes, parsedWays, parsedRelations); } public IEnumerable Filter(FilterSettings filterSettings) @@ -108,11 +129,11 @@ public OsmGeo Get(OsmGeoType type, long id) switch (type) { case OsmGeoType.Node: - return NodesParser.LoadNodes(new HashSet() { id }, index).Single(); + return GetNode(id); case OsmGeoType.Way: - return WaysParser.LoadWays(new HashSet() { id }, index).Single(); + return GetWay(id); case OsmGeoType.Relation: - return RelationsParser.LoadRelations(new HashSet() { id }, index).Single(); + return GetRelation(id); default: throw new NotImplementedException(); } @@ -120,16 +141,37 @@ public OsmGeo Get(OsmGeoType type, long id) public Node GetNode(long id) { + if (nodesCache.TryGetValue(id, out var node)) + { + return node; + } return NodesParser.LoadNodes(new HashSet() { id }, index).Single(); } public Way GetWay(long id) { + if (waysCache.TryGetValue(id, out var way)) + { + return way; + } + return WaysParser.LoadWays(new HashSet() { id }, index).Single(); } public Relation GetRelation(long id) { + if (relationsCache.TryGetValue(id, out var relation)) + { + return relation; + } + return RelationsParser.LoadRelations(new HashSet() { id }, index).Single(); - } + } + + public void ClearBatchCache() + { + nodesCache.Clear(); + waysCache.Clear(); + relationsCache.Clear(); + } } \ No newline at end of file diff --git a/OsmNightWatch/Program.cs b/OsmNightWatch/Program.cs index d29671d..b84cc3c 100644 --- a/OsmNightWatch/Program.cs +++ b/OsmNightWatch/Program.cs @@ -6,7 +6,7 @@ using OsmNightWatch.Lib; using OsmNightWatch.PbfParsing; using OsmSharp.Changesets; -using OsmSharp.Replication; +using OsmSharp.Replication; using System.IO.Compression; using System.Xml.Serialization; @@ -101,7 +101,7 @@ database.SetTimestamp(replicationState.EndTimestamp); Log($"Analyzing changeset..."); var newIssuesData = Analyze(analyzers, mergedChangeset, dbWithChanges, replicationState); - + dbWithChanges.ClearBatchCache(); newIssuesData.SetTimestampsAndLastKnownGood(oldIssuesData); oldIssuesData = newIssuesData; UploadIssues(replicationState, newIssuesData); diff --git a/OsmNightWatch/Utils.cs b/OsmNightWatch/Utils.cs index ca49370..416fa6c 100644 --- a/OsmNightWatch/Utils.cs +++ b/OsmNightWatch/Utils.cs @@ -74,11 +74,11 @@ public static void BatchLoad(IEnumerable relevantThings, IOsmGeoBatchSou nodesToLoad.Add(member.Id); } } - osmSource.BatchLoad(wayIds: waysToLoad); + var loadedWays = osmSource.BatchLoad(wayIds: waysToLoad); if (nodes) { - foreach (var way in relevantThings.Union(waysToLoad.Select(id => osmSource.Get(OsmGeoType.Way, id))).OfType()) - nodesToLoad.UnionWith(way.Nodes); + foreach (var way in relevantThings.OfType().Union(loadedWays.ways)) + nodesToLoad.UnionWith(way.Nodes); osmSource.BatchLoad(nodeIds: nodesToLoad); } } @@ -89,7 +89,7 @@ private static List RecursivelyLoadAllRelations(IEnumerable re Dictionary dictionaryOfLoadedRelations; while (true) { - dictionaryOfLoadedRelations = relationsBag.ToDictionary(r => r.Id!, r => r); + dictionaryOfLoadedRelations = relationsBag.ToDictionary(r => r.Id, r => r); var unloadedChildren = new HashSet(); foreach (var relation in dictionaryOfLoadedRelations.Values) { @@ -106,10 +106,19 @@ private static List RecursivelyLoadAllRelations(IEnumerable re { break; } - osmSource.BatchLoad(null, null, unloadedChildren); + var batchLoadedRelations = osmSource.BatchLoad(relationIds: unloadedChildren).relations; + foreach (var loadedRelation in batchLoadedRelations) + { + relationsBag.Add(loadedRelation); + } foreach (var relationId in unloadedChildren) { - relationsBag.Add((Relation)osmSource.Get(OsmGeoType.Relation, relationId)); + var relation = (Relation)osmSource.Get(OsmGeoType.Relation, relationId); + if (relation == null) + { + continue; + } + relationsBag.Add(relation); } } return relationsBag.ToList(); diff --git a/PbfParser/IOsmGeoBatchSource.cs b/PbfParser/IOsmGeoBatchSource.cs index a1a8c16..8ac0e65 100644 --- a/PbfParser/IOsmGeoBatchSource.cs +++ b/PbfParser/IOsmGeoBatchSource.cs @@ -96,6 +96,7 @@ public interface IOsmGeoSource public interface IOsmGeoBatchSource : IOsmGeoSource { //TODO: Changes this weird API to GetNodes, GetWays and GetRelations - public (IReadOnlyCollection nodes, IReadOnlyCollection ways, IReadOnlyCollection relations) BatchLoad(HashSet? nodeIds = null, HashSet? wayIds = null, HashSet? relationIds = null); + (IReadOnlyCollection nodes, IReadOnlyCollection ways, IReadOnlyCollection relations) BatchLoad(HashSet? nodeIds = null, HashSet? wayIds = null, HashSet? relationIds = null); + void ClearBatchCache(); } }