Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(seeding): add parallalization to seeding #1204

Closed
wants to merge 1 commit into from
Closed

feat(seeding): add parallalization to seeding #1204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopeMapperUpdater.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,27 +37,31 @@ public async Task UpdateClientScopeMapper(string instanceName, CancellationToken
var seederConfig = seedDataHandler.GetSpecificConfiguration(ConfigurationKey.ClientScopes);

var clients = await keycloak.GetClientsAsync(realm, null, true, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
foreach (var (clientName, mappingModels) in seedDataHandler.ClientScopeMappings)
{
var client = clients.SingleOrDefault(x => x.ClientId == clientName);
if (client?.Id is null)
await Parallel.ForEachAsync(seedDataHandler.ClientScopeMappings,
ParallelOptionsExtensions.CreateParallelOptions(cancellationToken),
async (mappings, token) =>
{
throw new ConflictException($"No client id found with name {clientName}");
}

var roles = await keycloak.GetRolesAsync(realm, client.Id, cancellationToken: cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
foreach (var mappingModel in mappingModels)
{
var clientScope = clients.SingleOrDefault(x => x.ClientId == mappingModel.Client);
if (clientScope?.Id is null)
var (clientName, mappingModels) = mappings;
var client = clients.SingleOrDefault(x => x.ClientId == clientName);
if (client?.Id is null)
{
throw new ConflictException($"No client id found with name {clientName}");
}
var clientRoles = await keycloak.GetClientRolesScopeMappingsForClientAsync(realm, clientScope.Id, client.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
var mappingModelRoles = mappingModel.Roles?.Select(roleName => roles.SingleOrDefault(r => r.Name == roleName) ?? throw new ConflictException($"No role with name {roleName} found")) ?? Enumerable.Empty<Role>();
await AddAndDeleteRoles(keycloak, realm, clientScope.Id, client.Id, clientRoles, mappingModelRoles, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
}

var roles = await keycloak.GetRolesAsync(realm, client.Id, cancellationToken: token).ConfigureAwait(ConfigureAwaitOptions.None);
foreach (var mappingModel in mappingModels)
{
var clientScope = clients.SingleOrDefault(x => x.ClientId == mappingModel.Client);
if (clientScope?.Id is null)
{
throw new ConflictException($"No client id found with name {clientName}");
}

var clientRoles = await keycloak.GetClientRolesScopeMappingsForClientAsync(realm, clientScope.Id, client.Id, token).ConfigureAwait(ConfigureAwaitOptions.None);
var mappingModelRoles = mappingModel.Roles?.Select(roleName => roles.SingleOrDefault(r => r.Name == roleName) ?? throw new ConflictException($"No role with name {roleName} found")) ?? Enumerable.Empty<Role>();
await AddAndDeleteRoles(keycloak, realm, clientScope.Id, client.Id, clientRoles, mappingModelRoles, seederConfig, token).ConfigureAwait(ConfigureAwaitOptions.None);
}
}).ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task AddAndDeleteRoles(KeycloakClient keycloak, string realm, string clientScopeId, string clientId, IEnumerable<Role> roles, IEnumerable<Role> updateRoles, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
Expand Down
64 changes: 36 additions & 28 deletions src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopesUpdater.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,51 +39,59 @@ public async Task UpdateClientScopes(string instanceName, CancellationToken canc
var clientScopes = await keycloak.GetClientScopesAsync(realm, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
var seedClientScopes = seedDataHandler.ClientScopes;

await CheckAndExecute(ModificationType.Delete, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, RemoveObsoleteClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
await CheckAndExecute(ModificationType.Create, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, CreateMissingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
await CheckAndExecute(ModificationType.Update, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, UpdateExistingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
var parallelOptions = ParallelOptionsExtensions.CreateParallelOptions(cancellationToken);
await CheckAndExecute(ModificationType.Delete, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, RemoveObsoleteClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
await CheckAndExecute(ModificationType.Create, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, CreateMissingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
await CheckAndExecute(ModificationType.Update, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, UpdateExistingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
}

private static Task CheckAndExecute(ModificationType modificationType, KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken, Func<KeycloakClient, string, IEnumerable<ClientScope>, IEnumerable<ClientScopeModel>, KeycloakSeederConfigModel, CancellationToken, Task> executeLogic) =>
private static Task CheckAndExecute(ModificationType modificationType, KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions, Func<KeycloakClient, string, IEnumerable<ClientScope>, IEnumerable<ClientScopeModel>, KeycloakSeederConfigModel, ParallelOptions, Task> executeLogic) =>
seederConfig.ModificationAllowed(modificationType)
? executeLogic(keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken)
? executeLogic(keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions)
: Task.CompletedTask;

private static async Task RemoveObsoleteClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task RemoveObsoleteClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
foreach (var deleteScope in clientScopes
.Where(x => seederConfig.ModificationAllowed(ModificationType.Delete, x.Name))
.ExceptBy(seedClientScopes.Select(x => x.Name), x => x.Name))
{
await keycloak.DeleteClientScopeAsync(
realm,
deleteScope.Id ?? throw new ConflictException($"clientScope.Id is null: {deleteScope.Name}"),
cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
var deleteScopes = clientScopes
.Where(x => seederConfig.ModificationAllowed(ModificationType.Delete, x.Name))
.ExceptBy(seedClientScopes.Select(x => x.Name), x => x.Name);
await Parallel.ForEachAsync(deleteScopes, parallelOptions, async (scope, cancellationToken) =>
{
await keycloak.DeleteClientScopeAsync(
realm,
scope.Id ?? throw new ConflictException($"clientScope.Id is null: {scope.Name}"),
cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
})
.ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task CreateMissingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task CreateMissingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
foreach (var addScope in seedClientScopes
.Where(x => seederConfig.ModificationAllowed(ModificationType.Create, x.Name))
.ExceptBy(clientScopes.Select(x => x.Name), x => x.Name))
{
await keycloak.CreateClientScopeAsync(realm, CreateClientScope(null, addScope, true), cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
var clientScopeModels = seedClientScopes
.Where(x => seederConfig.ModificationAllowed(ModificationType.Create, x.Name))
.ExceptBy(clientScopes.Select(x => x.Name), x => x.Name);
await Parallel.ForEachAsync(clientScopeModels, parallelOptions, async (scope, cancellationToken) =>
{
await keycloak.CreateClientScopeAsync(realm, CreateClientScope(null, scope, true), cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
})
.ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task UpdateExistingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task UpdateExistingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
foreach (var (clientScope, update) in clientScopes
var clientScopeModels = clientScopes
.Where(x => seederConfig.ModificationAllowed(ModificationType.Update, x.Name))
.Join(
seedClientScopes,
x => x.Name,
x => x.Name,
(clientScope, update) => (ClientScope: clientScope, Update: update)))
{
await UpdateClientScopeWithProtocolMappers(keycloak, realm, clientScope, update, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
(clientScope, update) => (ClientScope: clientScope, Update: update));
await Parallel.ForEachAsync(clientScopeModels, parallelOptions, async (scope, cancellationToken) =>
{
var (clientScope, update) = scope;
await UpdateClientScopeWithProtocolMappers(keycloak, realm, clientScope, update, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
})
.ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task UpdateClientScopeWithProtocolMappers(KeycloakClient keycloak, string realm, ClientScope clientScope, ClientScopeModel update, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
Expand Down
37 changes: 20 additions & 17 deletions src/keycloak/Keycloak.Seeding/BusinessLogic/IdentityProvidersUpdater.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public async Task UpdateIdentityProviders(string keycloakInstanceName, Cancellat
var keycloak = keycloakFactory.CreateKeycloakClient(keycloakInstanceName);
var realm = seedDataHandler.Realm;
var seederConfig = seedDataHandler.GetSpecificConfiguration(ConfigurationKey.IdentityProviders);
var parallelOptions = ParallelOptionsExtensions.CreateParallelOptions(cancellationToken);

foreach (var updateIdentityProvider in seedDataHandler.IdentityProviders)
{
Expand Down Expand Up @@ -64,17 +65,19 @@ public async Task UpdateIdentityProviders(string keycloakInstanceName, Cancellat
var updateMappers = seedDataHandler.IdentityProviderMappers.Where(x => x.IdentityProviderAlias == updateIdentityProvider.Alias);
var mappers = await keycloak.GetIdentityProviderMappersAsync(realm, updateIdentityProvider.Alias, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);

await DeleteObsoleteIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
await CreateMissingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
await UpdateExistingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
await DeleteObsoleteIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
await CreateMissingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
await UpdateExistingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
}
}

private static async Task CreateMissingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task CreateMissingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
foreach (var mapper in updateMappers
.Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Create, x.Name))
.ExceptBy(mappers.Select(x => x.Name), x => x.Name))
var addMappers = updateMappers
.Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers,
ModificationType.Create, x.Name))
.ExceptBy(mappers.Select(x => x.Name), x => x.Name);
await Parallel.ForEachAsync(addMappers, parallelOptions, async (mapper, cancellationToken) =>
{
await keycloak.AddIdentityProviderMapperAsync(
realm,
Expand All @@ -87,44 +90,46 @@ await keycloak.AddIdentityProviderMapperAsync(
},
mapper),
cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
}).ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task UpdateExistingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task UpdateExistingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
foreach (var (mapper, update) in mappers
var mappersToUpdate = mappers
.Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Update, x.Name))
.Join(
updateMappers,
x => x.Name,
x => x.Name,
(mapper, update) => (Mapper: mapper, Update: update))
.Where(x => !CompareIdentityProviderMapper(x.Mapper, x.Update)))
.Where(x => !CompareIdentityProviderMapper(x.Mapper, x.Update));
await Parallel.ForEachAsync(mappersToUpdate, parallelOptions, async (mapperModel, cancellationToken) =>
{
var (mapper, update) = mapperModel;
await keycloak.UpdateIdentityProviderMapperAsync(
realm,
alias,
mapper.Id ?? throw new ConflictException($"identityProviderMapper.id must never be null {mapper.Name} {mapper.IdentityProviderAlias}"),
UpdateIdentityProviderMapper(mapper, update),
cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
}).ConfigureAwait(ConfigureAwaitOptions.None);
}

private static async Task DeleteObsoleteIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
private static async Task DeleteObsoleteIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
{
if (mappers
.Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Delete, x.Name))
.ExceptBy(updateMappers.Select(x => x.Name), x => x.Name)
.IfAny(async deleteMappers =>
{
foreach (var mapper in deleteMappers)
await Parallel.ForEachAsync(deleteMappers, parallelOptions, async (mapper, cancellationToken) =>
{
await keycloak.DeleteIdentityProviderMapperAsync(
realm,
alias,
mapper.Id ?? throw new ConflictException($"identityProviderMapper.id must never be null {mapper.Name} {mapper.IdentityProviderAlias}"),
cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
}
}).ConfigureAwait(ConfigureAwaitOptions.None);
},
out var deleteMappersTask))
{
Expand Down Expand Up @@ -206,7 +211,6 @@ private static bool CompareIdentityProviderConfig(Config? config, IdentityProvid
config == null && update == null ||
config != null && update != null &&
config.HideOnLoginPage == update.HideOnLoginPage &&
//ClientSecret = update.ClientSecret &&
config.DisableUserInfo == update.DisableUserInfo &&
config.ValidateSignature == update.ValidateSignature &&
config.ClientId == update.ClientId &&
Expand All @@ -220,7 +224,6 @@ private static bool CompareIdentityProviderConfig(Config? config, IdentityProvid
config.UseJwksUrl == update.UseJwksUrl &&
config.UserInfoUrl == update.UserInfoUrl &&
config.Issuer == update.Issuer &&
// for Saml:
config.NameIDPolicyFormat == update.NameIDPolicyFormat &&
config.PrincipalType == update.PrincipalType &&
config.SignatureAlgorithm == update.SignatureAlgorithm &&
Expand Down
Loading
Loading