eliott/authentik
1
0
Fork 0
mirror of https://github.com/goauthentik/authentik synced 2026-04-25 17:15:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

lib/sync/outgoing: store sync settings in database (#17630)

Browse Source
This commit is contained in:
Marc 'risson' Schmitt
2025-10-22 17:15:37 +02:00
committed by GitHub
parent 6a594355d3
commit e593933bca
14 changed files with 324 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
authentik/enterprise/providers/google_workspace/api/providers.py
View File

@@ -37,6 +37,8 @@ class GoogleWorkspaceProviderSerializer(EnterpriseRequiredMixin, ProviderSeriali
"user_delete_action",
"group_delete_action",
"default_group_email_domain",
"sync_page_size",
"sync_page_timeout",
"dry_run",
]
extra_kwargs = {}

33
authentik/enterprise/providers/google_workspace/migrations/0005_googleworkspaceprovider_sync_page_size_and_more.py Normal file
View File

@@ -0,0 +1,33 @@
# Generated by Django 5.2.7 on 2025-10-21 12:35
import authentik.lib.utils.time
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentik_providers_google_workspace", "0004_googleworkspaceprovider_dry_run"),
]
operations = [
migrations.AddField(
model_name="googleworkspaceprovider",
name="sync_page_size",
field=models.PositiveIntegerField(
default=100,
help_text="Controls the number of objects synced in a single task",
validators=[django.core.validators.MinValueValidator(1)],
),
),
migrations.AddField(
model_name="googleworkspaceprovider",
name="sync_page_timeout",
field=models.TextField(
default="minutes=30",
help_text="Timeout for synchronization of a single page",
validators=[authentik.lib.utils.time.timedelta_string_validator],
),
),
]

2
authentik/enterprise/providers/microsoft_entra/api/providers.py
View File

@@ -36,6 +36,8 @@ class MicrosoftEntraProviderSerializer(EnterpriseRequiredMixin, ProviderSerializ
"filter_group",
"user_delete_action",
"group_delete_action",
"sync_page_size",
"sync_page_timeout",
"dry_run",
]
extra_kwargs = {}

33
authentik/enterprise/providers/microsoft_entra/migrations/0004_microsoftentraprovider_sync_page_size_and_more.py Normal file
View File

@@ -0,0 +1,33 @@
# Generated by Django 5.2.7 on 2025-10-21 12:35
import authentik.lib.utils.time
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentik_providers_microsoft_entra", "0003_microsoftentraprovider_dry_run"),
]
operations = [
migrations.AddField(
model_name="microsoftentraprovider",
name="sync_page_size",
field=models.PositiveIntegerField(
default=100,
help_text="Controls the number of objects synced in a single task",
validators=[django.core.validators.MinValueValidator(1)],
),
),
migrations.AddField(
model_name="microsoftentraprovider",
name="sync_page_timeout",
field=models.TextField(
default="minutes=30",
help_text="Timeout for synchronization of a single page",
validators=[authentik.lib.utils.time.timedelta_string_validator],
),
),
]

2
authentik/lib/sync/outgoing/__init__.py
View File

@@ -1,7 +1,5 @@
"""Sync constants"""
PAGE_SIZE = 100
PAGE_TIMEOUT_MS = 60 * 60 * 0.5 * 1000 # Half an hour
HTTP_CONFLICT = 409
HTTP_NO_CONTENT = 204
HTTP_SERVICE_UNAVAILABLE = 503

20
authentik/lib/sync/outgoing/models.py
View File

@@ -2,15 +2,15 @@ from typing import Any, Self
import pglock
from django.core.paginator import Paginator
from django.core.validators import MinValueValidator
from django.db import connection, models
from django.db.models import Model, QuerySet, TextChoices
from django.utils.translation import gettext_lazy as _
from dramatiq.actor import Actor
from authentik.core.models import Group, User
from authentik.lib.sync.outgoing import PAGE_SIZE, PAGE_TIMEOUT_MS
from authentik.lib.sync.outgoing.base import BaseOutgoingSyncClient
from authentik.lib.utils.time import fqdn_rand
from authentik.lib.utils.time import fqdn_rand, timedelta_from_string, timedelta_string_validator
from authentik.tasks.schedules.common import ScheduleSpec
from authentik.tasks.schedules.models import ScheduledModel
@@ -27,6 +27,17 @@ class OutgoingSyncDeleteAction(TextChoices):
class OutgoingSyncProvider(ScheduledModel, Model):
"""Base abstract models for providers implementing outgoing sync"""
sync_page_size = models.PositiveIntegerField(
help_text=_("Controls the number of objects synced in a single task"),
default=100,
validators=[MinValueValidator(1)],
)
sync_page_timeout = models.TextField(
help_text=_("Timeout for synchronization of a single page"),
default="minutes=30",
validators=[timedelta_string_validator],
)
dry_run = models.BooleanField(
default=False,
help_text=_(
@@ -46,11 +57,12 @@ class OutgoingSyncProvider(ScheduledModel, Model):
raise NotImplementedError
def get_paginator[T: User | Group](self, type: type[T]) -> Paginator:
return Paginator(self.get_object_qs(type), PAGE_SIZE)
return Paginator(self.get_object_qs(type), self.sync_page_size)
def get_object_sync_time_limit_ms[T: User | Group](self, type: type[T]) -> int:
num_pages: int = self.get_paginator(type).num_pages
return int(num_pages * PAGE_TIMEOUT_MS * 1.5)
page_timeout_ms = timedelta_from_string(self.sync_page_timeout).total_seconds() * 1000
return int(num_pages * page_timeout_ms * 1.5)
def get_sync_time_limit_ms(self) -> int:
return int(

10
authentik/lib/sync/outgoing/tasks.py
View File

@@ -9,7 +9,6 @@ from structlog.stdlib import BoundLogger, get_logger
from authentik.core.expression.exceptions import SkipObjectException
from authentik.core.models import Group, User
from authentik.events.utils import sanitize_item
from authentik.lib.sync.outgoing import PAGE_SIZE, PAGE_TIMEOUT_MS
from authentik.lib.sync.outgoing.base import Direction
from authentik.lib.sync.outgoing.exceptions import (
BadRequestSyncException,
@@ -20,6 +19,7 @@ from authentik.lib.sync.outgoing.exceptions import (
from authentik.lib.sync.outgoing.models import OutgoingSyncProvider
from authentik.lib.utils.errors import exception_to_dict
from authentik.lib.utils.reflection import class_to_path, path_to_class
from authentik.lib.utils.time import timedelta_from_string
from authentik.tasks.middleware import CurrentTask
from authentik.tasks.models import Task
@@ -44,10 +44,11 @@ class SyncTasks:
**options,
):
tasks = []
time_limit = timedelta_from_string(provider.sync_page_timeout).total_seconds() * 1000
for page in paginator.page_range:
page_sync = sync_objects.message_with_options(
args=(class_to_path(object_type), page, provider.pk),
time_limit=PAGE_TIMEOUT_MS,
time_limit=time_limit,
# Assign tasks to the same schedule as the current one
rel_obj=current_task.rel_obj,
uid=f"{provider.name}:{object_type._meta.model_name}:{page}",
@@ -139,7 +140,10 @@ class SyncTasks:
client = provider.client_for_model(_object_type)
except TransientSyncException:
return
paginator = Paginator(provider.get_object_qs(_object_type).filter(**filter), PAGE_SIZE)
paginator = Paginator(
provider.get_object_qs(_object_type).filter(**filter),
provider.sync_page_size,
)
if client.can_discover:
self.logger.debug("starting discover")
client.discover()

2
authentik/providers/scim/api/providers.py
View File

@@ -38,6 +38,8 @@ class SCIMProviderSerializer(
"compatibility_mode",
"exclude_users_service_account",
"filter_group",
"sync_page_size",
"sync_page_timeout",
"dry_run",
]
extra_kwargs = {}

33
authentik/providers/scim/migrations/0016_scimprovider_sync_page_size_and_more.py Normal file
View File

@@ -0,0 +1,33 @@
# Generated by Django 5.2.7 on 2025-10-21 12:35
import authentik.lib.utils.time
import django.core.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentik_providers_scim", "0015_alter_scimprovider_compatibility_mode"),
]
operations = [
migrations.AddField(
model_name="scimprovider",
name="sync_page_size",
field=models.PositiveIntegerField(
default=100,
help_text="Controls the number of objects synced in a single task",
validators=[django.core.validators.MinValueValidator(1)],
),
),
migrations.AddField(
model_name="scimprovider",
name="sync_page_timeout",
field=models.TextField(
default="minutes=30",
help_text="Timeout for synchronization of a single page",
validators=[authentik.lib.utils.time.timedelta_string_validator],
),
),
]

39
blueprints/schema.json
View File

@@ -5871,6 +5871,19 @@
"minLength": 1,
"title": "Default group email domain"
},
"sync_page_size": {
"type": "integer",
"minimum": 1,
"maximum": 2147483647,
"title": "Sync page size",
"description": "Controls the number of objects synced in a single task"
},
"sync_page_timeout": {
"type": "string",
"minLength": 1,
"title": "Sync page timeout",
"description": "Timeout for synchronization of a single page"
},
"dry_run": {
"type": "boolean",
"title": "Dry run",
@@ -6024,6 +6037,19 @@
],
"title": "Group delete action"
},
"sync_page_size": {
"type": "integer",
"minimum": 1,
"maximum": 2147483647,
"title": "Sync page size",
"description": "Controls the number of objects synced in a single task"
},
"sync_page_timeout": {
"type": "string",
"minLength": 1,
"title": "Sync page timeout",
"description": "Timeout for synchronization of a single page"
},
"dry_run": {
"type": "boolean",
"title": "Dry run",
@@ -9676,6 +9702,19 @@
"format": "uuid",
"title": "Filter group"
},
"sync_page_size": {
"type": "integer",
"minimum": 1,
"maximum": 2147483647,
"title": "Sync page size",
"description": "Controls the number of objects synced in a single task"
},
"sync_page_timeout": {
"type": "string",
"minLength": 1,
"title": "Sync page timeout",
"description": "Timeout for synchronization of a single page"
},
"dry_run": {
"type": "boolean",
"title": "Dry run",

78
schema.yml
View File

@@ -36067,6 +36067,14 @@ components:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
default_group_email_domain:
type: string
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -36238,6 +36246,15 @@ components:
default_group_email_domain:
type: string
minLength: 1
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -38570,6 +38587,14 @@ components:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
group_delete_action:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -38736,6 +38761,15 @@ components:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
group_delete_action:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -43743,6 +43777,15 @@ components:
default_group_email_domain:
type: string
minLength: 1
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -44408,6 +44451,15 @@ components:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
group_delete_action:
$ref: '#/components/schemas/OutgoingSyncDeleteAction'
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -45693,6 +45745,15 @@ components:
type: string
format: uuid
nullable: true
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -49360,6 +49421,14 @@ components:
type: string
format: uuid
nullable: true
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in
@@ -49469,6 +49538,15 @@ components:
type: string
format: uuid
nullable: true
sync_page_size:
type: integer
maximum: 2147483647
minimum: 1
description: Controls the number of objects synced in a single task
sync_page_timeout:
type: string
minLength: 1
description: Timeout for synchronization of a single page
dry_run:
type: boolean
description: When enabled, provider will not modify or create objects in

26
web/src/admin/providers/google_workspace/GoogleWorkspaceProviderForm.ts
View File

@@ -1,4 +1,7 @@
import "#elements/CodeMirror";
import "#components/ak-number-input";
import "#elements/utils/TimeDeltaHelp";
import "#components/ak-text-input";
import "#elements/ak-dual-select/ak-dual-select-dynamic-selected-provider";
import "#elements/ak-dual-select/ak-dual-select-provider";
import "#elements/forms/FormGroup";
@@ -272,6 +275,29 @@ export class GoogleWorkspaceProviderFormPage extends BaseProviderForm<GoogleWork
</p>
</ak-form-element-horizontal>
</div>
</ak-form-group>
<ak-form-group label="${msg("Sync settings")}">
<div class="pf-c-form">
<ak-number-input
label=${msg("Page size")}
required
name="pageSize"
value="${this.instance?.syncPageSize ?? 100}"
help=${msg("Controls the number of objects synced in a single task.")}
></ak-number-input>
<ak-text-input
name="syncPageTimeout"
label=${msg("Page timeout")}
input-hint="code"
required
value="${ifDefined(this.instance?.syncPageTimeout ?? "minutes=30")}"
.bighelp=${html`<p class="pf-c-form__helper-text">
${msg("Timeout for synchronization of a single page.")}
</p>
<ak-utils-time-delta-help></ak-utils-time-delta-help>`}
>
</ak-text-input>
</div>
</ak-form-group>`;
}
}

26
web/src/admin/providers/microsoft_entra/MicrosoftEntraProviderForm.ts
View File

@@ -1,4 +1,7 @@
import "#components/ak-hidden-text-input";
import "#components/ak-number-input";
import "#elements/utils/TimeDeltaHelp";
import "#components/ak-text-input";
import "#elements/ak-dual-select/ak-dual-select-dynamic-selected-provider";
import "#elements/ak-dual-select/ak-dual-select-provider";
import "#elements/forms/FormGroup";
@@ -248,6 +251,29 @@ export class MicrosoftEntraProviderFormPage extends BaseProviderForm<MicrosoftEn
</p>
</ak-form-element-horizontal>
</div>
</ak-form-group>
<ak-form-group label="${msg("Sync settings")}">
<div class="pf-c-form">
<ak-number-input
label=${msg("Page size")}
required
name="pageSize"
value="${this.instance?.syncPageSize ?? 100}"
help=${msg("Controls the number of objects synced in a single task.")}
></ak-number-input>
<ak-text-input
name="syncPageTimeout"
label=${msg("Page timeout")}
input-hint="code"
required
value="${ifDefined(this.instance?.syncPageTimeout ?? "minutes=30")}"
.bighelp=${html`<p class="pf-c-form__helper-text">
${msg("Timeout for synchronization of a single page.")}
</p>
<ak-utils-time-delta-help></ak-utils-time-delta-help>`}
>
</ak-text-input>
</div>
</ak-form-group>`;
}
}

27
web/src/admin/providers/scim/SCIMProviderFormForm.ts
View File

@@ -7,6 +7,9 @@ import "#elements/forms/Radio";
import "#elements/forms/SearchSelect/index";
import "#elements/CodeMirror";
import "#admin/common/ak-license-notice";
import "#components/ak-number-input";
import "#elements/utils/TimeDeltaHelp";
import "#components/ak-text-input";
import { propertyMappingsProvider, propertyMappingsSelector } from "./SCIMProviderFormHelpers.js";
@@ -301,5 +304,29 @@ export function renderForm({ provider = {}, errors = {}, update }: SCIMProviderF
</ak-form-element-horizontal>
</div>
</ak-form-group>
<ak-form-group label="${msg("Sync settings")}">
<div class="pf-c-form">
<ak-number-input
label=${msg("Page size")}
required
name="pageSize"
value="${provider.syncPageSize ?? 100}"
help=${msg("Controls the number of objects synced in a single task.")}
></ak-number-input>
<ak-text-input
name="syncPageTimeout"
label=${msg("Page timeout")}
input-hint="code"
required
value="${provider.syncPageTimeout ?? "minutes=30"}"
.bighelp=${html`<p class="pf-c-form__helper-text">
${msg("Timeout for synchronization of a single page.")}
</p>
<ak-utils-time-delta-help></ak-utils-time-delta-help>`}
>
</ak-text-input>
</div>
</ak-form-group>
`;
}