diff --git a/backend/Omnikeeper/Model/BaseAttributeModel.cs b/backend/Omnikeeper/Model/BaseAttributeModel.cs index 24007002..cc193361 100644 --- a/backend/Omnikeeper/Model/BaseAttributeModel.cs +++ b/backend/Omnikeeper/Model/BaseAttributeModel.cs @@ -36,68 +36,13 @@ private string CIIDSelection2WhereClause(ICIIDSelection selection) }; } - // NOTE: postgres-based implementation of filtering by attribute value - // must work equivalent to AttributeScalarTextFilter.Contains() - //private string NamedAttributesWithValueFiltersSelection2WhereClause(NamedAttributesWithValueFiltersSelection s) - //{ - // var attributeFilterClauses = new List(); - // IList filterParameters = new List(); - // var index = 0; - // foreach (var (attributeName, filter) in s.NamesAndFilters) - // { - // string valueClause; - // if (filter.Exact != null) - // { - // valueClause = $"value_text = @filterAttributeValue{index}"; - // filterParameters.Add(new NpgsqlParameter($"filterAttributeValue{index}", filter.Exact)); - // } - // else if (filter.Regex != null) - // { - // // TODO: either support, or restrict other regex options - // var ignoreCase = (filter.Regex.Options & System.Text.RegularExpressions.RegexOptions.IgnoreCase) != 0; - // valueClause = $"value_text ~{(ignoreCase ? "*" : "")} @filterAttributeValue{index}"; - // filterParameters.Add(new NpgsqlParameter($"filterAttributeValue{index}", filter.Regex.Pattern)); - // } - // else - // throw new Exception("Invalid filter detected"); - // // TODO: support other attribute value types - // var filterClause = $"((type = 'text' OR type = 'multiline_text') AND {AttributeValueHelper.BuildSQLIsScalarCheckClause()} AND name = @filterAttributeName{index} AND {valueClause})"; - - // filterParameters.Add(new NpgsqlParameter($"filterAttributeName{index}", attributeName)); - // attributeFilterClauses.Add(filterClause); - - // index++; - // } - - // var attributeFilterClause = string.Join(" AND ", attributeFilterClauses); - // return $"({attributeFilterClause})"; - //} - //private IEnumerable NamedAttributesWithValueFiltersSelection2Parameters(NamedAttributesWithValueFiltersSelection selection) - //{ - // var index = 0; - // foreach (var (attributeName, filter) in selection.NamesAndFilters) - // { - // if (filter.Exact != null) - // yield return new NpgsqlParameter($"filterAttributeValue{index}", filter.Exact); - // else if (filter.Regex != null) - // yield return new NpgsqlParameter($"filterAttributeValue{index}", filter.Regex.Pattern); - // else - // throw new Exception("Invalid filter detected"); - - // yield return new NpgsqlParameter($"filterAttributeName{index}", attributeName); - - // index++; - // } - //} - - private string AttributeSelection2WhereClause(IAttributeSelection selection) + private string AttributeSelection2WhereClause(IAttributeSelection selection, ref int parameterIndex) { return selection switch { AllAttributeSelection _ => "1=1", NoAttributesSelection _ => "1=0", - NamedAttributesSelection _ => "name = ANY(@names)", - //NamedAttributesWithValueFiltersSelection f => NamedAttributesWithValueFiltersSelection2WhereClause(f), + NamedAttributesSelection _ => $"name = ANY({SetParameter(ref parameterIndex)})", _ => throw new NotImplementedException("") }; } @@ -110,13 +55,8 @@ private IEnumerable AttributeSelection2Parameters(IAttributeSel case NoAttributesSelection _: break; case NamedAttributesSelection n: - yield return new NpgsqlParameter("@names", n.AttributeNames.ToArray()); + yield return new NpgsqlParameter(null, n.AttributeNames.ToArray()); break; - //case NamedAttributesWithValueFiltersSelection f: - // var t = NamedAttributesWithValueFiltersSelection2Parameters(f); - // foreach (var tt in t) - // yield return tt; - // break; default: throw new NotImplementedException(""); }; @@ -184,21 +124,28 @@ private async Task OptimizeCIIDSelection(ICIIDSelection selectio return selection; } + private string SetParameter(ref int cur) + { + return $"${++cur}"; + } + private async IAsyncEnumerable<(CIAttribute attribute, string layerID)> _GetAttributes(ICIIDSelection selection, string[] layerIDs, IModelContext trans, TimeThreshold atTime, IAttributeSelection attributeSelection, bool fullBinary) { NpgsqlCommand command; var ciidSelection2CTEClause = CIIDSelection2CTEClause(selection); + + int parameterIndex = 0; if (atTime.IsLatest && _USE_LATEST_TABLE) { command = new NpgsqlCommand($@" {ciidSelection2CTEClause} select id, name, a.ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id FROM attribute_latest a {CIIDSelection2JoinClause(selection)} - where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY(@layer_ids) - and ({AttributeSelection2WhereClause(attributeSelection)})", trans.DBConnection, trans.DBTransaction); - command.Parameters.AddWithValue("layer_ids", layerIDs); + where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY({SetParameter(ref parameterIndex)}) + and ({AttributeSelection2WhereClause(attributeSelection, ref parameterIndex)})", trans.DBConnection, trans.DBTransaction); + command.Parameters.Add(new NpgsqlParameter { Value = layerIDs }); foreach (var p in AttributeSelection2Parameters(attributeSelection)) command.Parameters.Add(p); } @@ -211,14 +158,14 @@ private async Task OptimizeCIIDSelection(ICIIDSelection selectio select id, name, ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id from ( select distinct on(a.ci_id, name, layer_id) removed, id, name, a.ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id FROM attribute a {CIIDSelection2JoinClause(selection)} - where ({CIIDSelection2WhereClause(selection)}) and timestamp <= @time_threshold and layer_id = ANY(@layer_ids) and partition_index >= @partition_index - and ({AttributeSelection2WhereClause(attributeSelection)}) + where ({CIIDSelection2WhereClause(selection)}) and timestamp <= {SetParameter(ref parameterIndex)} and layer_id = ANY({SetParameter(ref parameterIndex)}) and partition_index >= {SetParameter(ref parameterIndex)} + and ({AttributeSelection2WhereClause(attributeSelection, ref parameterIndex)}) order by a.ci_id, name, layer_id, timestamp DESC NULLS LAST ) i where removed = false ", trans.DBConnection, trans.DBTransaction); - command.Parameters.AddWithValue("layer_ids", layerIDs); - command.Parameters.AddWithValue("time_threshold", atTime.Time.ToUniversalTime()); - command.Parameters.AddWithValue("partition_index", partitionIndex); + command.Parameters.AddWithValue(atTime.Time.ToUniversalTime()); + command.Parameters.AddWithValue(layerIDs); + command.Parameters.AddWithValue(partitionIndex); foreach (var p in AttributeSelection2Parameters(attributeSelection)) command.Parameters.Add(p); } @@ -283,19 +230,19 @@ public async Task>[]> GetAttr return ret; } - // TODO: test public async Task> GetCIIDsWithAttributes(ICIIDSelection selection, string[] layerIDs, IModelContext trans, TimeThreshold atTime) { NpgsqlCommand command; + int parameterIndex = 0; if (atTime.IsLatest && _USE_LATEST_TABLE) { command = new NpgsqlCommand($@" {CIIDSelection2CTEClause(selection)} select distinct a.ci_id FROM attribute_latest a {CIIDSelection2JoinClause(selection)} - where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY(@layer_ids) + where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY({SetParameter(ref parameterIndex)}) ", trans.DBConnection, trans.DBTransaction); - command.Parameters.AddWithValue("layer_ids", layerIDs); + command.Parameters.AddWithValue(layerIDs); } else { @@ -306,13 +253,13 @@ select distinct a.ci_id FROM attribute_latest a select distinct i.ci_id from ( select distinct on(a.ci_id, name, layer_id) a.ci_id as ci_id, removed FROM attribute a {CIIDSelection2JoinClause(selection)} - where ({CIIDSelection2WhereClause(selection)}) and timestamp <= @time_threshold and layer_id = ANY(@layer_ids) and partition_index >= @partition_index + where ({CIIDSelection2WhereClause(selection)}) and timestamp <= {SetParameter(ref parameterIndex)} and layer_id = ANY({SetParameter(ref parameterIndex)}) and partition_index >= {SetParameter(ref parameterIndex)} order by a.ci_id, name, layer_id, timestamp DESC NULLS LAST ) i WHERE i.removed = false ", trans.DBConnection, trans.DBTransaction); - command.Parameters.AddWithValue("layer_ids", layerIDs); - command.Parameters.AddWithValue("time_threshold", atTime.Time.ToUniversalTime()); - command.Parameters.AddWithValue("partition_index", partitionIndex); + command.Parameters.AddWithValue(atTime.Time.ToUniversalTime()); + command.Parameters.AddWithValue(layerIDs); + command.Parameters.AddWithValue(partitionIndex); } command.Prepare(); @@ -330,16 +277,17 @@ select distinct on(a.ci_id, name, layer_id) a.ci_id as ci_id, removed FROM attri return ret; } - + // TODO: test public async Task> GetAttributesOfChangeset(Guid changesetID, bool getRemoved, IModelContext trans) { + int parameterIndex = 0; var ret = new List(); using var command = new NpgsqlCommand($@" select id, name, ci_id, type, value_text, value_binary, value_control FROM attribute - where changeset_id = @changeset_id AND removed = @removed + where changeset_id = {SetParameter(ref parameterIndex)} AND removed = {SetParameter(ref parameterIndex)} ", trans.DBConnection, trans.DBTransaction); - command.Parameters.AddWithValue("changeset_id", changesetID); - command.Parameters.AddWithValue("removed", getRemoved); + command.Parameters.AddWithValue(changesetID); + command.Parameters.AddWithValue(getRemoved); command.Prepare(); diff --git a/backend/Tests/Integration/Model/AttributeModelTest.cs b/backend/Tests/Integration/Model/AttributeModelTest.cs index d91bec3f..dc455b41 100644 --- a/backend/Tests/Integration/Model/AttributeModelTest.cs +++ b/backend/Tests/Integration/Model/AttributeModelTest.cs @@ -578,5 +578,64 @@ public async Task TestAttributeValueDateTimeWithOffset() trans.Commit(); } } + + [Test] + public async Task TestGetCIIDsWithAttributes() + { + var transI = ModelContextBuilder.BuildImmediate(); + Guid ciid1, ciid2, ciid3; + using (var trans = ModelContextBuilder.BuildDeferred()) + { + ciid1 = await GetService().CreateCI(trans); + ciid2 = await GetService().CreateCI(trans); + ciid3 = await GetService().CreateCI(trans); + trans.Commit(); + } + + string layerID1, layerID2; + using (var trans = ModelContextBuilder.BuildDeferred()) + { + var (layer1, _) = await GetService().CreateLayerIfNotExists("l1", trans); + layerID1 = layer1.ID; + var (layer2, _) = await GetService().CreateLayerIfNotExists("l2", trans); + layerID2 = layer2.ID; + trans.Commit(); + } + + var layerset1 = await GetService().BuildLayerSet(new string[] { "l1" }, transI); + var layerset2 = await GetService().BuildLayerSet(new string[] { "l2" }, transI); + var layerset12 = await GetService().BuildLayerSet(new string[] { "l1", "l2" }, transI); + + + TimeThreshold insert1TimeThreshold; + using (var trans = ModelContextBuilder.BuildDeferred()) + { + var changeset = await CreateChangesetProxy(); + await GetService().InsertAttribute("a1", new AttributeScalarValueText("value_a1"), ciid1, layerID1, changeset, new DataOriginV1(DataOriginType.Manual), trans, OtherLayersValueHandlingForceWrite.Instance); + trans.Commit(); + + insert1TimeThreshold = TimeThreshold.BuildAtTime(changeset.TimeThreshold.Time); + } + + using (var trans = ModelContextBuilder.BuildDeferred()) + { + var changeset = await CreateChangesetProxy(); + await GetService().InsertAttribute("a2", new AttributeScalarValueText("value_a2"), ciid2, layerID2, changeset, new DataOriginV1(DataOriginType.Manual), trans, OtherLayersValueHandlingForceWrite.Instance); + trans.Commit(); + } + + var r1 = await GetService().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset1.LayerIDs, transI, TimeThreshold.BuildLatest()); + r1.Should().BeEquivalentTo(new List() { ciid1 }, options => options.WithoutStrictOrdering()); + + var r2 = await GetService().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset2.LayerIDs, transI, TimeThreshold.BuildLatest()); + r2.Should().BeEquivalentTo(new List() { ciid2 }, options => options.WithoutStrictOrdering()); + + var r3 = await GetService().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset12.LayerIDs, transI, TimeThreshold.BuildLatest()); + r3.Should().BeEquivalentTo(new List() { ciid1, ciid2 }, options => options.WithoutStrictOrdering()); + + // check historic + var r4 = await GetService().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset12.LayerIDs, transI, insert1TimeThreshold); + r4.Should().BeEquivalentTo(new List() { ciid1 }, options => options.WithoutStrictOrdering()); + } } }