Skip to content

Commit

Permalink
Reworked BaseAttributeModel to use Npgsql raw mode
Browse files Browse the repository at this point in the history 
Relates to #226
  • Loading branch information
Maximilian Csuk committed Jul 26, 2022
1 parent 42dcc40 commit d693a30
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 82 deletions.
112 changes: 30 additions & 82 deletions backend/Omnikeeper/Model/BaseAttributeModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,68 +36,13 @@ private string CIIDSelection2WhereClause(ICIIDSelection selection)
};
}

// NOTE: postgres-based implementation of filtering by attribute value
// must work equivalent to AttributeScalarTextFilter.Contains()
//private string NamedAttributesWithValueFiltersSelection2WhereClause(NamedAttributesWithValueFiltersSelection s)
//{
// var attributeFilterClauses = new List<string>();
// IList<NpgsqlParameter> filterParameters = new List<NpgsqlParameter>();
// var index = 0;
// foreach (var (attributeName, filter) in s.NamesAndFilters)
// {
// string valueClause;
// if (filter.Exact != null)
// {
// valueClause = $"value_text = @filterAttributeValue{index}";
// filterParameters.Add(new NpgsqlParameter($"filterAttributeValue{index}", filter.Exact));
// }
// else if (filter.Regex != null)
// {
// // TODO: either support, or restrict other regex options
// var ignoreCase = (filter.Regex.Options & System.Text.RegularExpressions.RegexOptions.IgnoreCase) != 0;
// valueClause = $"value_text ~{(ignoreCase ? "*" : "")} @filterAttributeValue{index}";
// filterParameters.Add(new NpgsqlParameter($"filterAttributeValue{index}", filter.Regex.Pattern));
// }
// else
// throw new Exception("Invalid filter detected");
// // TODO: support other attribute value types
// var filterClause = $"((type = 'text' OR type = 'multiline_text') AND {AttributeValueHelper.BuildSQLIsScalarCheckClause()} AND name = @filterAttributeName{index} AND {valueClause})";

// filterParameters.Add(new NpgsqlParameter($"filterAttributeName{index}", attributeName));
// attributeFilterClauses.Add(filterClause);

// index++;
// }

// var attributeFilterClause = string.Join(" AND ", attributeFilterClauses);
// return $"({attributeFilterClause})";
//}
//private IEnumerable<NpgsqlParameter> NamedAttributesWithValueFiltersSelection2Parameters(NamedAttributesWithValueFiltersSelection selection)
//{
// var index = 0;
// foreach (var (attributeName, filter) in selection.NamesAndFilters)
// {
// if (filter.Exact != null)
// yield return new NpgsqlParameter($"filterAttributeValue{index}", filter.Exact);
// else if (filter.Regex != null)
// yield return new NpgsqlParameter($"filterAttributeValue{index}", filter.Regex.Pattern);
// else
// throw new Exception("Invalid filter detected");

// yield return new NpgsqlParameter($"filterAttributeName{index}", attributeName);

// index++;
// }
//}

private string AttributeSelection2WhereClause(IAttributeSelection selection)
private string AttributeSelection2WhereClause(IAttributeSelection selection, ref int parameterIndex)
{
return selection switch
{
AllAttributeSelection _ => "1=1",
NoAttributesSelection _ => "1=0",
NamedAttributesSelection _ => "name = ANY(@names)",
//NamedAttributesWithValueFiltersSelection f => NamedAttributesWithValueFiltersSelection2WhereClause(f),
NamedAttributesSelection _ => $"name = ANY({SetParameter(ref parameterIndex)})",
_ => throw new NotImplementedException("")
};
}
Expand All @@ -110,13 +55,8 @@ private IEnumerable<NpgsqlParameter> AttributeSelection2Parameters(IAttributeSel
case NoAttributesSelection _:
break;
case NamedAttributesSelection n:
yield return new NpgsqlParameter("@names", n.AttributeNames.ToArray());
yield return new NpgsqlParameter(null, n.AttributeNames.ToArray());
break;
//case NamedAttributesWithValueFiltersSelection f:
// var t = NamedAttributesWithValueFiltersSelection2Parameters(f);
// foreach (var tt in t)
// yield return tt;
// break;
default:
throw new NotImplementedException("");
};
Expand Down Expand Up @@ -184,21 +124,28 @@ private async Task<ICIIDSelection> OptimizeCIIDSelection(ICIIDSelection selectio
return selection;
}

private string SetParameter(ref int cur)
{
return $"${++cur}";
}

private async IAsyncEnumerable<(CIAttribute attribute, string layerID)> _GetAttributes(ICIIDSelection selection, string[] layerIDs, IModelContext trans, TimeThreshold atTime, IAttributeSelection attributeSelection, bool fullBinary)
{
NpgsqlCommand command;

var ciidSelection2CTEClause = CIIDSelection2CTEClause(selection);


int parameterIndex = 0;
if (atTime.IsLatest && _USE_LATEST_TABLE)
{
command = new NpgsqlCommand($@"
{ciidSelection2CTEClause}
select id, name, a.ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id FROM attribute_latest a
{CIIDSelection2JoinClause(selection)}
where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY(@layer_ids)
and ({AttributeSelection2WhereClause(attributeSelection)})", trans.DBConnection, trans.DBTransaction);
command.Parameters.AddWithValue("layer_ids", layerIDs);
where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY({SetParameter(ref parameterIndex)})
and ({AttributeSelection2WhereClause(attributeSelection, ref parameterIndex)})", trans.DBConnection, trans.DBTransaction);
command.Parameters.Add(new NpgsqlParameter { Value = layerIDs });
foreach (var p in AttributeSelection2Parameters(attributeSelection))
command.Parameters.Add(p);
}
Expand All @@ -211,14 +158,14 @@ private async Task<ICIIDSelection> OptimizeCIIDSelection(ICIIDSelection selectio
select id, name, ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id from (
select distinct on(a.ci_id, name, layer_id) removed, id, name, a.ci_id, type, value_text, value_binary, value_control, changeset_id, layer_id FROM attribute a
{CIIDSelection2JoinClause(selection)}
where ({CIIDSelection2WhereClause(selection)}) and timestamp <= @time_threshold and layer_id = ANY(@layer_ids) and partition_index >= @partition_index
and ({AttributeSelection2WhereClause(attributeSelection)})
where ({CIIDSelection2WhereClause(selection)}) and timestamp <= {SetParameter(ref parameterIndex)} and layer_id = ANY({SetParameter(ref parameterIndex)}) and partition_index >= {SetParameter(ref parameterIndex)}
and ({AttributeSelection2WhereClause(attributeSelection, ref parameterIndex)})
order by a.ci_id, name, layer_id, timestamp DESC NULLS LAST
) i where removed = false
", trans.DBConnection, trans.DBTransaction);
command.Parameters.AddWithValue("layer_ids", layerIDs);
command.Parameters.AddWithValue("time_threshold", atTime.Time.ToUniversalTime());
command.Parameters.AddWithValue("partition_index", partitionIndex);
command.Parameters.AddWithValue(atTime.Time.ToUniversalTime());
command.Parameters.AddWithValue(layerIDs);
command.Parameters.AddWithValue(partitionIndex);
foreach (var p in AttributeSelection2Parameters(attributeSelection))
command.Parameters.Add(p);
}
Expand Down Expand Up @@ -283,19 +230,19 @@ public async Task<IDictionary<Guid, IDictionary<string, CIAttribute>>[]> GetAttr
return ret;
}

// TODO: test
public async Task<IReadOnlySet<Guid>> GetCIIDsWithAttributes(ICIIDSelection selection, string[] layerIDs, IModelContext trans, TimeThreshold atTime)
{
NpgsqlCommand command;
int parameterIndex = 0;
if (atTime.IsLatest && _USE_LATEST_TABLE)
{
command = new NpgsqlCommand($@"
{CIIDSelection2CTEClause(selection)}
select distinct a.ci_id FROM attribute_latest a
{CIIDSelection2JoinClause(selection)}
where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY(@layer_ids)
where ({CIIDSelection2WhereClause(selection)}) and layer_id = ANY({SetParameter(ref parameterIndex)})
", trans.DBConnection, trans.DBTransaction);
command.Parameters.AddWithValue("layer_ids", layerIDs);
command.Parameters.AddWithValue(layerIDs);
}
else
{
Expand All @@ -306,13 +253,13 @@ select distinct a.ci_id FROM attribute_latest a
select distinct i.ci_id from (
select distinct on(a.ci_id, name, layer_id) a.ci_id as ci_id, removed FROM attribute a
{CIIDSelection2JoinClause(selection)}
where ({CIIDSelection2WhereClause(selection)}) and timestamp <= @time_threshold and layer_id = ANY(@layer_ids) and partition_index >= @partition_index
where ({CIIDSelection2WhereClause(selection)}) and timestamp <= {SetParameter(ref parameterIndex)} and layer_id = ANY({SetParameter(ref parameterIndex)}) and partition_index >= {SetParameter(ref parameterIndex)}
order by a.ci_id, name, layer_id, timestamp DESC NULLS LAST
) i WHERE i.removed = false
", trans.DBConnection, trans.DBTransaction);
command.Parameters.AddWithValue("layer_ids", layerIDs);
command.Parameters.AddWithValue("time_threshold", atTime.Time.ToUniversalTime());
command.Parameters.AddWithValue("partition_index", partitionIndex);
command.Parameters.AddWithValue(atTime.Time.ToUniversalTime());
command.Parameters.AddWithValue(layerIDs);
command.Parameters.AddWithValue(partitionIndex);
}

command.Prepare();
Expand All @@ -330,16 +277,17 @@ select distinct on(a.ci_id, name, layer_id) a.ci_id as ci_id, removed FROM attri
return ret;
}


// TODO: test
public async Task<IReadOnlyList<CIAttribute>> GetAttributesOfChangeset(Guid changesetID, bool getRemoved, IModelContext trans)
{
int parameterIndex = 0;
var ret = new List<CIAttribute>();
using var command = new NpgsqlCommand($@"
select id, name, ci_id, type, value_text, value_binary, value_control FROM attribute
where changeset_id = @changeset_id AND removed = @removed
where changeset_id = {SetParameter(ref parameterIndex)} AND removed = {SetParameter(ref parameterIndex)}
", trans.DBConnection, trans.DBTransaction);
command.Parameters.AddWithValue("changeset_id", changesetID);
command.Parameters.AddWithValue("removed", getRemoved);
command.Parameters.AddWithValue(changesetID);
command.Parameters.AddWithValue(getRemoved);

command.Prepare();

Expand Down
59 changes: 59 additions & 0 deletions backend/Tests/Integration/Model/AttributeModelTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,5 +578,64 @@ public async Task TestAttributeValueDateTimeWithOffset()
trans.Commit();
}
}

[Test]
public async Task TestGetCIIDsWithAttributes()
{
var transI = ModelContextBuilder.BuildImmediate();
Guid ciid1, ciid2, ciid3;
using (var trans = ModelContextBuilder.BuildDeferred())
{
ciid1 = await GetService<ICIModel>().CreateCI(trans);
ciid2 = await GetService<ICIModel>().CreateCI(trans);
ciid3 = await GetService<ICIModel>().CreateCI(trans);
trans.Commit();
}

string layerID1, layerID2;
using (var trans = ModelContextBuilder.BuildDeferred())
{
var (layer1, _) = await GetService<ILayerModel>().CreateLayerIfNotExists("l1", trans);
layerID1 = layer1.ID;
var (layer2, _) = await GetService<ILayerModel>().CreateLayerIfNotExists("l2", trans);
layerID2 = layer2.ID;
trans.Commit();
}

var layerset1 = await GetService<ILayerModel>().BuildLayerSet(new string[] { "l1" }, transI);
var layerset2 = await GetService<ILayerModel>().BuildLayerSet(new string[] { "l2" }, transI);
var layerset12 = await GetService<ILayerModel>().BuildLayerSet(new string[] { "l1", "l2" }, transI);


TimeThreshold insert1TimeThreshold;
using (var trans = ModelContextBuilder.BuildDeferred())
{
var changeset = await CreateChangesetProxy();
await GetService<IAttributeModel>().InsertAttribute("a1", new AttributeScalarValueText("value_a1"), ciid1, layerID1, changeset, new DataOriginV1(DataOriginType.Manual), trans, OtherLayersValueHandlingForceWrite.Instance);
trans.Commit();

insert1TimeThreshold = TimeThreshold.BuildAtTime(changeset.TimeThreshold.Time);
}

using (var trans = ModelContextBuilder.BuildDeferred())
{
var changeset = await CreateChangesetProxy();
await GetService<IAttributeModel>().InsertAttribute("a2", new AttributeScalarValueText("value_a2"), ciid2, layerID2, changeset, new DataOriginV1(DataOriginType.Manual), trans, OtherLayersValueHandlingForceWrite.Instance);
trans.Commit();
}

var r1 = await GetService<IBaseAttributeModel>().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset1.LayerIDs, transI, TimeThreshold.BuildLatest());
r1.Should().BeEquivalentTo(new List<Guid>() { ciid1 }, options => options.WithoutStrictOrdering());

var r2 = await GetService<IBaseAttributeModel>().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset2.LayerIDs, transI, TimeThreshold.BuildLatest());
r2.Should().BeEquivalentTo(new List<Guid>() { ciid2 }, options => options.WithoutStrictOrdering());

var r3 = await GetService<IBaseAttributeModel>().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset12.LayerIDs, transI, TimeThreshold.BuildLatest());
r3.Should().BeEquivalentTo(new List<Guid>() { ciid1, ciid2 }, options => options.WithoutStrictOrdering());

// check historic
var r4 = await GetService<IBaseAttributeModel>().GetCIIDsWithAttributes(AllCIIDsSelection.Instance, layerset12.LayerIDs, transI, insert1TimeThreshold);
r4.Should().BeEquivalentTo(new List<Guid>() { ciid1 }, options => options.WithoutStrictOrdering());
}
}
}

0 comments on commit d693a30

Please sign in to comment.