Skip to content

Commit

Permalink
Optimize DB query performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@rampaa
rampaa committed Dec 14, 2023
1 parent a8094bd commit 9d94d9a
Show file tree
Hide file tree
Showing 7 changed files with 42 additions and 49 deletions.
12 changes: 7 additions & 5 deletions JL.Core/Dicts/EDICT/JMdict/JmdictDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,20 +174,22 @@ public static Dictionary<string, IList<IDictRecord>> GetRecordsFromDB(string dbN
r.cross_references AS relatedTerms,
r.antonyms AS antonyms
FROM record r
INNER JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term1
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR rsk.search_key = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down Expand Up @@ -238,7 +240,7 @@ SELECT json_array(rsk.search_key) AS searchKeys,
r.cross_references AS relatedTerms,
r.antonyms AS antonyms
FROM record r
INNER JOIN record_search_key rsk ON r.id = rsk.record_id
JOIN record_search_key rsk ON r.id = rsk.record_id
GROUP BY r.id
""";

Expand Down
8 changes: 5 additions & 3 deletions JL.Core/Dicts/EDICT/JMnedict/JmnedictDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,19 +100,21 @@ public static Dictionary<string, IList<IDictRecord>> GetRecordsFromDB(string dbN
r.glossary AS definitions,
r.name_types AS nameTypes
FROM record r
WHERE r.primary_spelling_in_hiragana = @term1
WHERE r.primary_spelling_in_hiragana IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR r.primary_spelling_in_hiragana = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down
8 changes: 5 additions & 3 deletions JL.Core/Dicts/EPWING/Nazeka/EpwingNazekaDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,19 +132,21 @@ public static Dictionary<string, IList<IDictRecord>> GetRecordsFromDB(string dbN
r.glossary AS definitions
FROM record r
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term1
WHERE rsk.search_key IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR rsk.search_key = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down
11 changes: 6 additions & 5 deletions JL.Core/Dicts/EPWING/Yomichan/EpwingYomichanDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,20 +135,21 @@ public static Dictionary<string, IList<IDictRecord>> GetRecordsFromDB(string dbN
r.glossary_tags AS definitionTags
FROM record r
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term1
"""
);
WHERE rsk.search_key IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR rsk.search_key = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down
8 changes: 5 additions & 3 deletions JL.Core/Dicts/PitchAccent/YomichanPitchAccentDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,19 +129,21 @@ public static Dictionary<string, IList<IDictRecord>> GetRecordsFromDB(string dbN
r.position AS position
FROM record r
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term1
WHERE rsk.search_key IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR rsk.search_key = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down
14 changes: 8 additions & 6 deletions JL.Core/Freqs/FreqDBManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,22 @@ public static Dictionary<string, List<FrequencyRecord>> GetRecordsFromDB(string
r.spelling as spelling,
r.frequency AS frequency
FROM record r
INNER JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term1
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key IN (@1
""");

for (int i = 1; i < terms.Count; i++)
{
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $"\nOR rsk.search_key = @term{i + 1}");
_ = queryBuilder.Append(CultureInfo.InvariantCulture, $", @{i + 1}");
}

_ = queryBuilder.Append(')');

command.CommandText = queryBuilder.ToString();

for (int i = 0; i < terms.Count; i++)
{
_ = command.Parameters.AddWithValue($"@term{i + 1}", terms[i]);
_ = command.Parameters.AddWithValue($"@{i + 1}", terms[i]);
}

using SqliteDataReader dataReader = command.ExecuteReader();
Expand Down Expand Up @@ -146,7 +148,7 @@ public static List<FrequencyRecord> GetRecordsFromDB(string dbName, string term)
"""
SELECT r.spelling as spelling, r.frequency AS frequency
FROM record r
INNER JOIN record_search_key rsk ON r.id = rsk.record_id
JOIN record_search_key rsk ON r.id = rsk.record_id
WHERE rsk.search_key = @term
""";

Expand All @@ -171,7 +173,7 @@ public static void LoadFromDB(Freq freq)
"""
SELECT json_array(rsk.search_key) AS searchKeys, r.spelling as spelling, r.frequency AS frequency
FROM record r
INNER JOIN record_search_key rsk ON r.id = rsk.record_id
JOIN record_search_key rsk ON r.id = rsk.record_id
GROUP BY r.id
""";

Expand Down
30 changes: 6 additions & 24 deletions JL.Core/Lookup/LookupUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1135,21 +1135,15 @@ private static List<LookupResult> BuildEpwingYomichanResult(
if (dbFreqs.Count > 0)
{
List<string> searchKeys = GetSearchKeyForEpwingYomichanRecord(epwingResults);
if (searchKeys.Count < 950)
{
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
},
() =>
{
if (useDBForPitchDict)
{
List<string> searchKeys = GetSearchKeyForEpwingYomichanRecord(epwingResults);
if (searchKeys.Count < 950)
{
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
});

Expand Down Expand Up @@ -1198,21 +1192,15 @@ private static ConcurrentBag<LookupResult> BuildEpwingNazekaResult(
if (dbFreqs.Count > 0)
{
List<string> searchKeys = GetSearchKeysFromEpwingNazekaRecord(epwingNazekaResults, true);
if (searchKeys.Count < 950)
{
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
},
() =>
{
if (useDBForPitchDict)
{
List<string> searchKeys = GetSearchKeysFromEpwingNazekaRecord(epwingNazekaResults, false);
if (searchKeys.Count < 950)
{
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
});

Expand Down Expand Up @@ -1262,21 +1250,15 @@ private static ConcurrentBag<LookupResult> BuildCustomWordResult(
if (dbFreqs.Count > 0)
{
List<string> searchKeys = GetSearchKeysFromEpwingNazekaRecord(customWordResults, true);
if (searchKeys.Count < 950)
{
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
frequencyDicts = GetFrequencyDictsFromDB(dbFreqs, searchKeys);
}
},
() =>
{
if (useDBForPitchDict)
{
List<string> searchKeys = GetSearchKeysFromEpwingNazekaRecord(customWordResults, false);
if (searchKeys.Count < 950)
{
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
pitchAccentDict = YomichanPitchAccentDBManager.GetRecordsFromDB(pitchDict!.Name, searchKeys);
}
});

Expand Down

0 comments on commit 9d94d9a

Please sign in to comment.