mirror of
https://github.com/suitenumerique/docs.git
synced 2026-05-07 07:32:33 +02:00
Compare commits
2 Commits
backend/uv
...
experiment
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
294b1a0dc5 | ||
|
|
cbd2705c9f |
35
.github/workflows/impress.yml
vendored
35
.github/workflows/impress.yml
vendored
@@ -96,20 +96,21 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Python
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version-file: "src/backend/pyproject.toml"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
- name: Install the project
|
||||
run: uv sync --locked --all-extras
|
||||
python-version: "3.13.3"
|
||||
cache: "pip"
|
||||
- name: Upgrade pip and setuptools
|
||||
run: pip install --upgrade pip setuptools
|
||||
- name: Install development dependencies
|
||||
run: pip install --user .[dev]
|
||||
- name: Check code formatting with ruff
|
||||
run: uv run ruff format . --diff
|
||||
run: ~/.local/bin/ruff format . --diff
|
||||
- name: Lint code with ruff
|
||||
run: uv run ruff check .
|
||||
run: ~/.local/bin/ruff check .
|
||||
- name: Lint code with pylint
|
||||
run: uv run pylint .
|
||||
run: ~/.local/bin/pylint .
|
||||
|
||||
test-back:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -191,14 +192,14 @@ jobs:
|
||||
mc mb impress/impress-media-storage && \
|
||||
mc version enable impress/impress-media-storage"
|
||||
|
||||
- name: Set up Python
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version-file: "src/backend/pyproject.toml"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
- name: Install the project
|
||||
run: uv sync --locked --all-extras
|
||||
python-version: "3.13.3"
|
||||
cache: "pip"
|
||||
|
||||
- name: Install development dependencies
|
||||
run: pip install --user .[dev]
|
||||
|
||||
- name: Install gettext (required to compile messages) and MIME support
|
||||
run: |
|
||||
@@ -207,7 +208,7 @@ jobs:
|
||||
sudo wget https://raw.githubusercontent.com/suitenumerique/django-lasuite/refs/heads/main/assets/conf/mime.types -O /etc/mime.types
|
||||
|
||||
- name: Generate a MO file from strings extracted from the project
|
||||
run: uv run python manage.py compilemessages
|
||||
run: python manage.py compilemessages
|
||||
|
||||
- name: Run tests
|
||||
run: uv run pytest -n 2
|
||||
run: ~/.local/bin/pytest -n 2
|
||||
|
||||
20
CHANGELOG.md
20
CHANGELOG.md
@@ -6,28 +6,10 @@ and this project adheres to
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- ⚡️(frontend) add skeleton on content loading #2254
|
||||
|
||||
### Changed
|
||||
|
||||
- 🏗️(core) migrate from pip to uv
|
||||
|
||||
### Fixed
|
||||
|
||||
- 🐛(frontend) fix patch and comments #2273
|
||||
- 🐛(frontend) interlinking are exported correctly in print mode #2269
|
||||
- 💬(frontend) add missing link in onboarding description #2233
|
||||
- 🐛(frontend) sanitize pasted and dropped content in document title #2210
|
||||
- 🐛(frontend) Emoji menu doesn't display above comment box #2229
|
||||
- 🐛(frontend) Block menu doesn't stay open on 1st line #2229
|
||||
- 🐛(frontend) The "+" on the first line of a new doc doesn't work #2229
|
||||
|
||||
### Security
|
||||
|
||||
- 🔒️(frontend) sanitize color during collaboration #2270
|
||||
|
||||
- 🐛(backend) replace document creation table locks with retry strategy
|
||||
|
||||
## [v5.0.0] - 2026-04-08
|
||||
|
||||
|
||||
61
Dockerfile
61
Dockerfile
@@ -1,38 +1,24 @@
|
||||
# Django impress
|
||||
|
||||
# ---- base image to inherit from ----
|
||||
FROM python:3.13.13-alpine AS base
|
||||
FROM python:3.13.3-alpine AS base
|
||||
|
||||
# Upgrade pip to its latest release to speed up dependencies installation
|
||||
RUN python -m pip install --upgrade pip
|
||||
|
||||
# Upgrade system packages to install security updates
|
||||
RUN apk update && apk upgrade --no-cache
|
||||
|
||||
# Upgrade pip to its latest release to speed up dependencies installation
|
||||
# We must do that to avoid having an outdated pip version with security issues
|
||||
RUN python -m pip install --upgrade pip
|
||||
|
||||
# ---- Back-end builder image ----
|
||||
FROM base AS back-builder
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
WORKDIR /builder
|
||||
|
||||
# Disable Python downloads, because we want to use the system interpreter
|
||||
# across both images. If using a managed Python version, it needs to be
|
||||
# copied from the build image into the final image;
|
||||
ENV UV_PYTHON_DOWNLOADS=0
|
||||
# Copy required python dependencies
|
||||
COPY ./src/backend /builder
|
||||
|
||||
# install uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.11.10 /uv /uvx /bin/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=src/backend/uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=src/backend/pyproject.toml,target=pyproject.toml \
|
||||
uv sync --locked --no-install-project --no-dev
|
||||
COPY src/backend /app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --locked --no-dev
|
||||
RUN mkdir /install && \
|
||||
pip install --prefix=/install .
|
||||
|
||||
|
||||
# ---- mails ----
|
||||
@@ -55,13 +41,14 @@ RUN apk add --no-cache \
|
||||
pango \
|
||||
rdfind
|
||||
|
||||
# Copy the application from the builder
|
||||
COPY --from=back-builder /app /app
|
||||
# Copy installed python dependencies
|
||||
COPY --from=back-builder /install /usr/local
|
||||
|
||||
# Copy impress application (see .dockerignore)
|
||||
COPY ./src/backend /app/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# collectstatic
|
||||
RUN DJANGO_CONFIGURATION=Build \
|
||||
python manage.py collectstatic --noinput
|
||||
@@ -97,12 +84,8 @@ COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint
|
||||
# docker user (see entrypoint).
|
||||
RUN chmod g=u /etc/passwd
|
||||
|
||||
# Copy the application from the builder
|
||||
COPY --from=back-builder /app /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
# Copy installed python dependencies
|
||||
COPY --from=back-builder /install /usr/local
|
||||
|
||||
# Link certifi certificate from a static path /cert/cacert.pem to avoid issues
|
||||
# when python is upgraded and the path to the certificate changes.
|
||||
@@ -112,6 +95,11 @@ RUN mkdir /cert && \
|
||||
mv $path /cert/ && \
|
||||
ln -s /cert/cacert.pem $path
|
||||
|
||||
# Copy impress application (see .dockerignore)
|
||||
COPY ./src/backend /app/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Generate compiled translation messages
|
||||
RUN DJANGO_CONFIGURATION=Build \
|
||||
python manage.py compilemessages
|
||||
@@ -131,9 +119,10 @@ USER root:root
|
||||
# Install psql
|
||||
RUN apk add --no-cache postgresql-client
|
||||
|
||||
# Install development dependencies
|
||||
RUN --mount=from=ghcr.io/astral-sh/uv:0.11.10,source=/uv,target=/bin/uv \
|
||||
uv sync --all-extras --locked
|
||||
# Uninstall impress and re-install it in editable mode along with development
|
||||
# dependencies
|
||||
RUN pip uninstall -y impress
|
||||
RUN pip install -e .[dev]
|
||||
|
||||
# Restore the un-privileged user running the application
|
||||
ARG DOCKER_USER
|
||||
|
||||
8
Makefile
8
Makefile
@@ -72,7 +72,7 @@ data/static:
|
||||
# -- Project
|
||||
|
||||
create-env-local-files: ## create env.local files in env.d/development
|
||||
create-env-local-files:
|
||||
create-env-local-files:
|
||||
@touch env.d/development/crowdin.local
|
||||
@touch env.d/development/common.local
|
||||
@touch env.d/development/postgresql.local
|
||||
@@ -141,7 +141,7 @@ else
|
||||
@echo "$(RESET)"
|
||||
@echo "$(GREEN)Starting bootstrap process...$(RESET)"
|
||||
endif
|
||||
@echo ""
|
||||
@echo ""
|
||||
.PHONY: pre-beautiful-bootstrap
|
||||
|
||||
post-beautiful-bootstrap: ## Display a success message after bootstrap
|
||||
@@ -235,7 +235,7 @@ run-backend: ## Start only the backend application and all needed services
|
||||
.PHONY: run-backend
|
||||
|
||||
run: ## start the wsgi (production) and development server
|
||||
run:
|
||||
run:
|
||||
@$(MAKE) run-backend
|
||||
@$(COMPOSE) up --force-recreate -d frontend-development
|
||||
.PHONY: run
|
||||
@@ -322,7 +322,7 @@ superuser: ## Create an admin superuser with password "admin"
|
||||
.PHONY: superuser
|
||||
|
||||
back-i18n-compile: ## compile the gettext files
|
||||
@$(MANAGE) compilemessages --ignore=".venv/**/*"
|
||||
@$(MANAGE) compilemessages --ignore="venv/**/*"
|
||||
.PHONY: back-i18n-compile
|
||||
|
||||
back-i18n-generate: ## create the .pot files used for i18n
|
||||
|
||||
@@ -80,7 +80,6 @@ services:
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
- /app/.venv
|
||||
depends_on:
|
||||
postgresql:
|
||||
condition: service_healthy
|
||||
@@ -109,7 +108,6 @@ services:
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
- /app/.venv
|
||||
depends_on:
|
||||
- app-dev
|
||||
|
||||
|
||||
@@ -134,6 +134,7 @@ These are the environment variables you can set for the `impress-backend` contai
|
||||
| THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 |
|
||||
| THEME_CUSTOMIZATION_FILE_PATH | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json |
|
||||
| TRASHBIN_CUTOFF_DAYS | Trashbin cutoff | 30 |
|
||||
| TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS | Number of attempts to create a document before failing. | 10 |
|
||||
| USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] |
|
||||
| USER_ONBOARDING_DOCUMENTS | A list of documents IDs for which a read-only access will be created for new s | [] |
|
||||
| USER_ONBOARDING_SANDBOX_DOCUMENT | ID of a template sandbox document that will be duplicated for new users | |
|
||||
|
||||
@@ -7,7 +7,6 @@ from base64 import b64decode
|
||||
from os.path import splitext
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection, transaction
|
||||
from django.db.models import Q
|
||||
from django.utils.functional import lazy
|
||||
from django.utils.text import slugify
|
||||
@@ -24,6 +23,7 @@ from core.services.converter_services import (
|
||||
ConversionError,
|
||||
Converter,
|
||||
)
|
||||
from core.utils.treebeard import create_tree_node_with_retry
|
||||
|
||||
|
||||
class UserSerializer(serializers.ModelSerializer):
|
||||
@@ -467,18 +467,12 @@ class ServerCreateDocumentSerializer(serializers.Serializer):
|
||||
{"content": ["Could not convert content"]}
|
||||
) from err
|
||||
|
||||
with transaction.atomic():
|
||||
# locks the table to ensure safe concurrent access
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
f'LOCK TABLE "{models.Document._meta.db_table}" ' # noqa: SLF001
|
||||
"IN SHARE ROW EXCLUSIVE MODE;"
|
||||
)
|
||||
|
||||
document = models.Document.add_root(
|
||||
document = create_tree_node_with_retry(
|
||||
lambda: models.Document.add_root(
|
||||
title=validated_data["title"],
|
||||
creator=user,
|
||||
)
|
||||
)
|
||||
|
||||
if user:
|
||||
# Associate the document with the pre-existing user
|
||||
|
||||
@@ -67,11 +67,10 @@ from core.services.search_indexers import (
|
||||
get_visited_document_ids_of,
|
||||
)
|
||||
from core.tasks.mail import send_ask_for_access_mail
|
||||
from core.utils import (
|
||||
extract_attachments,
|
||||
filter_descendants,
|
||||
users_sharing_documents_with,
|
||||
)
|
||||
from core.utils.paths import filter_descendants
|
||||
from core.utils.treebeard import create_tree_node_with_retry
|
||||
from core.utils.users import users_sharing_documents_with
|
||||
from core.utils.yjs import extract_attachments
|
||||
|
||||
from ..enums import FeatureFlag, SearchType
|
||||
from . import permissions, serializers, utils
|
||||
@@ -708,18 +707,12 @@ class DocumentViewSet(
|
||||
{"file": ["Could not convert file content"]}
|
||||
) from err
|
||||
|
||||
with transaction.atomic():
|
||||
# locks the table to ensure safe concurrent access
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
f'LOCK TABLE "{models.Document._meta.db_table}" ' # noqa: SLF001
|
||||
"IN SHARE ROW EXCLUSIVE MODE;"
|
||||
)
|
||||
|
||||
obj = models.Document.add_root(
|
||||
obj = create_tree_node_with_retry(
|
||||
lambda: models.Document.add_root(
|
||||
creator=self.request.user,
|
||||
**serializer.validated_data,
|
||||
)
|
||||
)
|
||||
serializer.instance = obj
|
||||
models.DocumentAccess.objects.create(
|
||||
document=obj,
|
||||
@@ -1023,16 +1016,12 @@ class DocumentViewSet(
|
||||
)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
with transaction.atomic():
|
||||
# "select_for_update" locks the table to ensure safe concurrent access
|
||||
locked_parent = models.Document.objects.select_for_update().get(
|
||||
pk=document.pk
|
||||
)
|
||||
|
||||
child_document = locked_parent.add_child(
|
||||
child_document = create_tree_node_with_retry(
|
||||
lambda: document.add_child(
|
||||
creator=request.user,
|
||||
**serializer.validated_data,
|
||||
)
|
||||
)
|
||||
|
||||
# Set the created instance to the serializer
|
||||
serializer.instance = child_document
|
||||
|
||||
@@ -9,7 +9,7 @@ from django.db import migrations, models
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
import core.models
|
||||
from core.utils import extract_attachments
|
||||
from core.utils.yjs import extract_attachments
|
||||
|
||||
|
||||
def populate_attachments_on_all_documents(apps, schema_editor):
|
||||
|
||||
@@ -19,7 +19,7 @@ from django.core.cache import cache
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import default_storage
|
||||
from django.core.mail import send_mail
|
||||
from django.db import connection, models, transaction
|
||||
from django.db import models, transaction
|
||||
from django.db.models.functions import Left, Length
|
||||
from django.template.loader import render_to_string
|
||||
from django.utils import timezone
|
||||
@@ -39,6 +39,7 @@ from core.choices import (
|
||||
RoleChoices,
|
||||
get_equivalent_link_definition,
|
||||
)
|
||||
from core.utils.treebeard import create_tree_node_with_retry
|
||||
from core.validators import sub_validator
|
||||
|
||||
logger = getLogger(__name__)
|
||||
@@ -265,8 +266,6 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
|
||||
duplicate the sandbox document for the user
|
||||
"""
|
||||
if settings.USER_ONBOARDING_SANDBOX_DOCUMENT:
|
||||
# transaction.atomic is used in a context manager to avoid a transaction if
|
||||
# the settings USER_ONBOARDING_SANDBOX_DOCUMENT is unused
|
||||
sandbox_id = settings.USER_ONBOARDING_SANDBOX_DOCUMENT
|
||||
try:
|
||||
template_document = Document.objects.get(id=sandbox_id)
|
||||
@@ -276,20 +275,15 @@ class User(AbstractBaseUser, BaseModel, auth_models.PermissionsMixin):
|
||||
sandbox_id,
|
||||
)
|
||||
return
|
||||
|
||||
with transaction.atomic():
|
||||
# locks the table to ensure safe concurrent access
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
f'LOCK TABLE "{Document._meta.db_table}" ' # noqa: SLF001
|
||||
"IN SHARE ROW EXCLUSIVE MODE;"
|
||||
sandbox_document = create_tree_node_with_retry(
|
||||
lambda: Document.add_root(
|
||||
title=template_document.title,
|
||||
content=template_document.content,
|
||||
attachments=template_document.attachments,
|
||||
duplicated_from=template_document,
|
||||
creator=self,
|
||||
)
|
||||
sandbox_document = Document.add_root(
|
||||
title=template_document.title,
|
||||
content=template_document.content,
|
||||
attachments=template_document.attachments,
|
||||
duplicated_from=template_document,
|
||||
creator=self,
|
||||
)
|
||||
|
||||
DocumentAccess.objects.create(
|
||||
|
||||
@@ -12,8 +12,11 @@ from django.utils.module_loading import import_string
|
||||
|
||||
import requests
|
||||
|
||||
from core import models, utils
|
||||
from core import models
|
||||
from core.enums import SearchType
|
||||
from core.utils.dicts import get_value_by_pattern
|
||||
from core.utils.paths import get_ancestor_to_descendants_map
|
||||
from core.utils.yjs import base64_yjs_to_text
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,7 +47,7 @@ def get_batch_accesses_by_users_and_teams(paths):
|
||||
Get accesses related to a list of document paths,
|
||||
grouped by users and teams, including all ancestor paths.
|
||||
"""
|
||||
ancestor_map = utils.get_ancestor_to_descendants_map(
|
||||
ancestor_map = get_ancestor_to_descendants_map(
|
||||
paths, steplen=models.Document.steplen
|
||||
)
|
||||
ancestor_paths = list(ancestor_map.keys())
|
||||
@@ -297,7 +300,7 @@ class FindDocumentIndexer(BaseDocumentIndexer):
|
||||
>>> get_title({"id": 1})
|
||||
""
|
||||
"""
|
||||
titles = utils.get_value_by_pattern(source, r"^title\.")
|
||||
titles = get_value_by_pattern(source, r"^title\.")
|
||||
for title in titles:
|
||||
if title:
|
||||
return title
|
||||
@@ -318,7 +321,7 @@ class FindDocumentIndexer(BaseDocumentIndexer):
|
||||
"""
|
||||
doc_path = document.path
|
||||
doc_content = document.content
|
||||
text_content = utils.base64_yjs_to_text(doc_content) if doc_content else ""
|
||||
text_content = base64_yjs_to_text(doc_content) if doc_content else ""
|
||||
|
||||
return {
|
||||
"id": str(document.id),
|
||||
|
||||
@@ -11,7 +11,7 @@ from django.dispatch import receiver
|
||||
|
||||
from core import models
|
||||
from core.tasks.search import trigger_batch_document_indexer
|
||||
from core.utils import get_users_sharing_documents_with_cache_key
|
||||
from core.utils.users import get_users_sharing_documents_with_cache_key
|
||||
|
||||
|
||||
@receiver(signals.post_save, sender=models.Document)
|
||||
|
||||
@@ -12,13 +12,14 @@ import pytest
|
||||
import responses
|
||||
from requests import HTTPError
|
||||
|
||||
from core import factories, models, utils
|
||||
from core import factories, models
|
||||
from core.services.search_indexers import (
|
||||
BaseDocumentIndexer,
|
||||
FindDocumentIndexer,
|
||||
get_document_indexer,
|
||||
get_visited_document_ids_of,
|
||||
)
|
||||
from core.utils.yjs import base64_yjs_to_text
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -199,7 +200,7 @@ def test_services_search_indexers_serialize_document_returns_expected_json():
|
||||
"depth": 1,
|
||||
"path": document.path,
|
||||
"numchild": 1,
|
||||
"content": utils.base64_yjs_to_text(document.content),
|
||||
"content": base64_yjs_to_text(document.content),
|
||||
"created_at": document.created_at.isoformat(),
|
||||
"updated_at": document.updated_at.isoformat(),
|
||||
"reach": document.link_reach,
|
||||
|
||||
@@ -8,7 +8,18 @@ from django.core.cache import cache
|
||||
import pycrdt
|
||||
import pytest
|
||||
|
||||
from core import factories, utils
|
||||
from core import factories
|
||||
from core.utils.dicts import get_value_by_pattern
|
||||
from core.utils.paths import get_ancestor_to_descendants_map
|
||||
from core.utils.users import (
|
||||
get_users_sharing_documents_with_cache_key,
|
||||
users_sharing_documents_with,
|
||||
)
|
||||
from core.utils.yjs import (
|
||||
base64_yjs_to_text,
|
||||
base64_yjs_to_xml,
|
||||
extract_attachments,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -34,12 +45,12 @@ TEST_BASE64_STRING = (
|
||||
|
||||
def test_utils_base64_yjs_to_text():
|
||||
"""Test extract text from saved yjs document"""
|
||||
assert utils.base64_yjs_to_text(TEST_BASE64_STRING) == "Hello w or ld"
|
||||
assert base64_yjs_to_text(TEST_BASE64_STRING) == "Hello w or ld"
|
||||
|
||||
|
||||
def test_utils_base64_yjs_to_xml():
|
||||
"""Test extract xml from saved yjs document"""
|
||||
content = utils.base64_yjs_to_xml(TEST_BASE64_STRING)
|
||||
content = base64_yjs_to_xml(TEST_BASE64_STRING)
|
||||
assert (
|
||||
'<heading textAlignment="left" level="1"><italic>Hello</italic></heading>'
|
||||
in content
|
||||
@@ -79,13 +90,13 @@ def test_utils_extract_attachments():
|
||||
update = ydoc.get_update()
|
||||
base64_string = base64.b64encode(update).decode("utf-8")
|
||||
# image_key2 is missing the "/media/" part and shouldn't get extracted
|
||||
assert utils.extract_attachments(base64_string) == [image_key1, image_key3]
|
||||
assert extract_attachments(base64_string) == [image_key1, image_key3]
|
||||
|
||||
|
||||
def test_utils_get_ancestor_to_descendants_map_single_path():
|
||||
"""Test ancestor mapping of a single path."""
|
||||
paths = ["000100020005"]
|
||||
result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
result = get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
|
||||
assert result == {
|
||||
"0001": {"000100020005"},
|
||||
@@ -97,7 +108,7 @@ def test_utils_get_ancestor_to_descendants_map_single_path():
|
||||
def test_utils_get_ancestor_to_descendants_map_multiple_paths():
|
||||
"""Test ancestor mapping of multiple paths with shared prefixes."""
|
||||
paths = ["000100020005", "00010003"]
|
||||
result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
result = get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
|
||||
assert result == {
|
||||
"0001": {"000100020005", "00010003"},
|
||||
@@ -119,10 +130,10 @@ def test_utils_users_sharing_documents_with_cache_miss():
|
||||
factories.UserDocumentAccessFactory(user=user2, document=doc1)
|
||||
factories.UserDocumentAccessFactory(user=user3, document=doc2)
|
||||
|
||||
cache_key = utils.get_users_sharing_documents_with_cache_key(user1)
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user1)
|
||||
cache.delete(cache_key)
|
||||
|
||||
result = utils.users_sharing_documents_with(user1)
|
||||
result = users_sharing_documents_with(user1)
|
||||
|
||||
assert user2.id in result
|
||||
|
||||
@@ -139,12 +150,12 @@ def test_utils_users_sharing_documents_with_cache_hit():
|
||||
factories.UserDocumentAccessFactory(user=user1, document=doc1)
|
||||
factories.UserDocumentAccessFactory(user=user2, document=doc1)
|
||||
|
||||
cache_key = utils.get_users_sharing_documents_with_cache_key(user1)
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user1)
|
||||
|
||||
test_cached_data = {user2.id: "2025-02-10"}
|
||||
cache.set(cache_key, test_cached_data, 86400)
|
||||
|
||||
result = utils.users_sharing_documents_with(user1)
|
||||
result = users_sharing_documents_with(user1)
|
||||
assert result == test_cached_data
|
||||
|
||||
|
||||
@@ -156,7 +167,7 @@ def test_utils_users_sharing_documents_with_cache_invalidation_on_create():
|
||||
doc1 = factories.DocumentFactory()
|
||||
|
||||
# Pre-populate cache
|
||||
cache_key = utils.get_users_sharing_documents_with_cache_key(user1)
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user1)
|
||||
cache.set(cache_key, {}, 86400)
|
||||
|
||||
# Verify cache exists
|
||||
@@ -182,7 +193,7 @@ def test_utils_users_sharing_documents_with_cache_invalidation_on_delete():
|
||||
|
||||
doc_access = factories.UserDocumentAccessFactory(user=user1, document=doc1)
|
||||
|
||||
cache_key = utils.get_users_sharing_documents_with_cache_key(user1)
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user1)
|
||||
cache.set(cache_key, {user2.id: "2025-02-10"}, 86400)
|
||||
|
||||
assert cache.get(cache_key) is not None
|
||||
@@ -196,10 +207,10 @@ def test_utils_users_sharing_documents_with_empty_result():
|
||||
"""Test when user is not sharing any documents."""
|
||||
user1 = factories.UserFactory()
|
||||
|
||||
cache_key = utils.get_users_sharing_documents_with_cache_key(user1)
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user1)
|
||||
cache.delete(cache_key)
|
||||
|
||||
result = utils.users_sharing_documents_with(user1)
|
||||
result = users_sharing_documents_with(user1)
|
||||
|
||||
assert result == {}
|
||||
|
||||
@@ -210,7 +221,7 @@ def test_utils_users_sharing_documents_with_empty_result():
|
||||
def test_utils_get_value_by_pattern_matching_key():
|
||||
"""Test extracting value from a dictionary with a matching key pattern."""
|
||||
data = {"title.extension": "Bonjour", "id": 1, "content": "test"}
|
||||
result = utils.get_value_by_pattern(data, r"^title\.")
|
||||
result = get_value_by_pattern(data, r"^title\.")
|
||||
|
||||
assert set(result) == {"Bonjour"}
|
||||
|
||||
@@ -218,7 +229,7 @@ def test_utils_get_value_by_pattern_matching_key():
|
||||
def test_utils_get_value_by_pattern_multiple_matches():
|
||||
"""Test that all matching keys are returned."""
|
||||
data = {"title.extension_1": "Bonjour", "title.extension_2": "Hello", "id": 1}
|
||||
result = utils.get_value_by_pattern(data, r"^title\.")
|
||||
result = get_value_by_pattern(data, r"^title\.")
|
||||
|
||||
assert set(result) == {
|
||||
"Bonjour",
|
||||
@@ -229,7 +240,7 @@ def test_utils_get_value_by_pattern_multiple_matches():
|
||||
def test_utils_get_value_by_pattern_multiple_extensions():
|
||||
"""Test that all matching keys are returned."""
|
||||
data = {"title.extension_1.extension_2": "Bonjour", "id": 1}
|
||||
result = utils.get_value_by_pattern(data, r"^title\.")
|
||||
result = get_value_by_pattern(data, r"^title\.")
|
||||
|
||||
assert set(result) == {"Bonjour"}
|
||||
|
||||
@@ -237,6 +248,6 @@ def test_utils_get_value_by_pattern_multiple_extensions():
|
||||
def test_utils_get_value_by_pattern_no_match():
|
||||
"""Test that empty list is returned when no key matches the pattern."""
|
||||
data = {"name": "Test", "id": 1}
|
||||
result = utils.get_value_by_pattern(data, r"^title\.")
|
||||
result = get_value_by_pattern(data, r"^title\.")
|
||||
|
||||
assert result == []
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Tests for the create_tree_node_with_retry utils."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import IntegrityError
|
||||
|
||||
import pytest
|
||||
|
||||
from core.factories import UserFactory
|
||||
from core.models import Document
|
||||
from core.utils.treebeard import _is_tree_path_collision, create_tree_node_with_retry
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
DjangoValidationError({"path": "not unique"}),
|
||||
IntegrityError("impress_document_path_key"),
|
||||
],
|
||||
)
|
||||
def test_utils_create_tree_node_with_retry_exceed_max_attempts(settings, exc):
|
||||
"""Test exceeding the max attempts should reraise the exception."""
|
||||
|
||||
settings.TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS = 2
|
||||
|
||||
create_fn = mock.MagicMock()
|
||||
create_fn.side_effect = exc
|
||||
|
||||
with (
|
||||
pytest.raises(exc.__class__),
|
||||
mock.patch(
|
||||
"core.utils.treebeard._is_tree_path_collision"
|
||||
) as mock__is_tree_path_collision,
|
||||
):
|
||||
mock__is_tree_path_collision.side_effect = _is_tree_path_collision
|
||||
create_tree_node_with_retry(create_fn)
|
||||
|
||||
mock__is_tree_path_collision.assert_called()
|
||||
assert mock__is_tree_path_collision.call_count == 2
|
||||
assert create_fn.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"exc",
|
||||
[
|
||||
DjangoValidationError({"foo": "bar"}),
|
||||
IntegrityError("not handled"),
|
||||
],
|
||||
)
|
||||
def test_utils_create_tree_node_with_retry_exceed_exception_not_handled(settings, exc):
|
||||
"""Test with an exception not handled should return reraise it immediatly."""
|
||||
|
||||
settings.TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS = 2
|
||||
|
||||
create_fn = mock.MagicMock()
|
||||
create_fn.side_effect = exc
|
||||
|
||||
with (
|
||||
pytest.raises(exc.__class__),
|
||||
mock.patch(
|
||||
"core.utils.treebeard._is_tree_path_collision"
|
||||
) as mock__is_tree_path_collision,
|
||||
):
|
||||
mock__is_tree_path_collision.side_effect = _is_tree_path_collision
|
||||
create_tree_node_with_retry(create_fn)
|
||||
|
||||
mock__is_tree_path_collision.assert_called()
|
||||
assert mock__is_tree_path_collision.call_count == 1
|
||||
assert create_fn.call_count == 1
|
||||
|
||||
|
||||
def test_utils_create_tree_node_with_retry_success():
|
||||
"""Test executing successfully the create_fn callback."""
|
||||
|
||||
user = UserFactory()
|
||||
|
||||
document = create_tree_node_with_retry(
|
||||
lambda: Document.add_root(
|
||||
creator=user,
|
||||
title="success",
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(document, Document)
|
||||
assert document.title == "success"
|
||||
assert document.path is not None
|
||||
@@ -2,7 +2,7 @@
|
||||
Unit tests for the filter_root_paths utility function.
|
||||
"""
|
||||
|
||||
from core.utils import filter_descendants
|
||||
from core.utils.paths import filter_descendants
|
||||
|
||||
|
||||
def test_utils_filter_descendants_success():
|
||||
|
||||
@@ -4,7 +4,8 @@ from django.utils import timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from core import factories, utils
|
||||
from core import factories
|
||||
from core.utils.users import users_sharing_documents_with
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
@@ -54,7 +55,7 @@ def test_utils_users_sharing_documents_with():
|
||||
doc_3_pierre_2.created_at = yesterday
|
||||
doc_3_pierre_2.save()
|
||||
|
||||
shared_map = utils.users_sharing_documents_with(user)
|
||||
shared_map = users_sharing_documents_with(user)
|
||||
|
||||
assert shared_map == {
|
||||
pierre_1.id: last_week,
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
"""Utils for the core app."""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.db import models as db
|
||||
from django.db.models import Subquery
|
||||
|
||||
import pycrdt
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core import enums, models
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_value_by_pattern(data, pattern):
|
||||
"""
|
||||
Get all values from keys matching a regex pattern in a dictionary.
|
||||
|
||||
Args:
|
||||
data (dict): Source dictionary to search
|
||||
pattern (str): Regex pattern to match against keys
|
||||
|
||||
Returns:
|
||||
list: List of values for all matching keys, empty list if no matches
|
||||
|
||||
Example:
|
||||
>>> get_value_by_pattern({"title.fr": "Bonjour", "id": 1}, r"^title\\.")
|
||||
["Bonjour"]
|
||||
>>> get_value_by_pattern({"title.fr": "Bonjour", "title.en": "Hello"}, r"^title\\.")
|
||||
["Bonjour", "Hello"]
|
||||
"""
|
||||
regex = re.compile(pattern)
|
||||
return [value for key, value in data.items() if regex.match(key)]
|
||||
|
||||
|
||||
def get_ancestor_to_descendants_map(paths, steplen):
|
||||
"""
|
||||
Given a list of document paths, return a mapping of ancestor_path -> set of descendant_paths.
|
||||
|
||||
Each path is assumed to use materialized path format with fixed-length segments.
|
||||
|
||||
Args:
|
||||
paths (list of str): List of full document paths.
|
||||
steplen (int): Length of each path segment.
|
||||
|
||||
Returns:
|
||||
dict[str, set[str]]: Mapping from ancestor path to its descendant paths (including itself).
|
||||
"""
|
||||
ancestor_map = defaultdict(set)
|
||||
for path in paths:
|
||||
for i in range(steplen, len(path) + 1, steplen):
|
||||
ancestor = path[:i]
|
||||
ancestor_map[ancestor].add(path)
|
||||
return ancestor_map
|
||||
|
||||
|
||||
def filter_descendants(paths, root_paths, skip_sorting=False):
|
||||
"""
|
||||
Filters paths to keep only those that are descendants of any path in root_paths.
|
||||
|
||||
A path is considered a descendant of a root path if it starts with the root path.
|
||||
If `skip_sorting` is not set to True, the function will sort both lists before
|
||||
processing because both `paths` and `root_paths` need to be in lexicographic order
|
||||
before going through the algorithm.
|
||||
|
||||
Args:
|
||||
paths (iterable of str): List of paths to be filtered.
|
||||
root_paths (iterable of str): List of paths to check as potential prefixes.
|
||||
skip_sorting (bool): If True, assumes both `paths` and `root_paths` are already sorted.
|
||||
|
||||
Returns:
|
||||
list of str: A list of sorted paths that are descendants of any path in `root_paths`.
|
||||
"""
|
||||
results = []
|
||||
i = 0
|
||||
n = len(root_paths)
|
||||
|
||||
if not skip_sorting:
|
||||
paths.sort()
|
||||
root_paths.sort()
|
||||
|
||||
for path in paths:
|
||||
# Try to find a matching prefix in the sorted accessible paths
|
||||
while i < n:
|
||||
if path.startswith(root_paths[i]):
|
||||
results.append(path)
|
||||
break
|
||||
if root_paths[i] < path:
|
||||
i += 1
|
||||
else:
|
||||
# If paths[i] > path, no need to keep searching
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def base64_yjs_to_xml(base64_string):
|
||||
"""Extract xml from base64 yjs document."""
|
||||
|
||||
decoded_bytes = base64.b64decode(base64_string)
|
||||
# uint8_array = bytearray(decoded_bytes)
|
||||
|
||||
doc = pycrdt.Doc()
|
||||
doc.apply_update(decoded_bytes)
|
||||
return str(doc.get("document-store", type=pycrdt.XmlFragment))
|
||||
|
||||
|
||||
def base64_yjs_to_text(base64_string):
|
||||
"""Extract text from base64 yjs document."""
|
||||
|
||||
blocknote_structure = base64_yjs_to_xml(base64_string)
|
||||
soup = BeautifulSoup(blocknote_structure, "lxml-xml")
|
||||
return soup.get_text(separator=" ", strip=True)
|
||||
|
||||
|
||||
def extract_attachments(content):
|
||||
"""Helper method to extract media paths from a document's content."""
|
||||
if not content:
|
||||
return []
|
||||
|
||||
xml_content = base64_yjs_to_xml(content)
|
||||
return re.findall(enums.MEDIA_STORAGE_URL_EXTRACT, xml_content)
|
||||
|
||||
|
||||
def get_users_sharing_documents_with_cache_key(user):
|
||||
"""Generate a unique cache key for each user."""
|
||||
return f"users_sharing_documents_with_{user.id}"
|
||||
|
||||
|
||||
def users_sharing_documents_with(user):
|
||||
"""
|
||||
Returns a map of users sharing documents with the given user,
|
||||
sorted by last shared date.
|
||||
"""
|
||||
start_time = time.time()
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user)
|
||||
cached_result = cache.get(cache_key)
|
||||
|
||||
if cached_result is not None:
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
"users_sharing_documents_with cache hit for user %s (took %.3fs)",
|
||||
user.id,
|
||||
elapsed,
|
||||
)
|
||||
return cached_result
|
||||
|
||||
user_docs_qs = models.DocumentAccess.objects.filter(user=user).values_list(
|
||||
"document_id", flat=True
|
||||
)
|
||||
shared_qs = (
|
||||
models.DocumentAccess.objects.filter(document_id__in=Subquery(user_docs_qs))
|
||||
.exclude(user=user)
|
||||
.values("user")
|
||||
.annotate(last_shared=db.Max("created_at"))
|
||||
)
|
||||
result = {item["user"]: item["last_shared"] for item in shared_qs}
|
||||
cache.set(cache_key, result, 86400) # Cache for 1 day
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
"users_sharing_documents_with cache miss for user %s (took %.3fs)",
|
||||
user.id,
|
||||
elapsed,
|
||||
)
|
||||
return result
|
||||
1
src/backend/core/utils/__init__.py
Normal file
1
src/backend/core/utils/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Core utilities package."""
|
||||
24
src/backend/core/utils/dicts.py
Normal file
24
src/backend/core/utils/dicts.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""Dictionary utility functions."""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def get_value_by_pattern(data, pattern):
|
||||
"""
|
||||
Get all values from keys matching a regex pattern in a dictionary.
|
||||
|
||||
Args:
|
||||
data (dict): Source dictionary to search
|
||||
pattern (str): Regex pattern to match against keys
|
||||
|
||||
Returns:
|
||||
list: List of values for all matching keys, empty list if no matches
|
||||
|
||||
Example:
|
||||
>>> get_value_by_pattern({"title.fr": "Bonjour", "id": 1}, r"^title\\.")
|
||||
["Bonjour"]
|
||||
>>> get_value_by_pattern({"title.fr": "Bonjour", "title.en": "Hello"}, r"^title\\.")
|
||||
["Bonjour", "Hello"]
|
||||
"""
|
||||
regex = re.compile(pattern)
|
||||
return [value for key, value in data.items() if regex.match(key)]
|
||||
63
src/backend/core/utils/paths.py
Normal file
63
src/backend/core/utils/paths.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""Path and tree structure utilities."""
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def get_ancestor_to_descendants_map(paths, steplen):
|
||||
"""
|
||||
Given a list of document paths, return a mapping of ancestor_path -> set of descendant_paths.
|
||||
|
||||
Each path is assumed to use materialized path format with fixed-length segments.
|
||||
|
||||
Args:
|
||||
paths (list of str): List of full document paths.
|
||||
steplen (int): Length of each path segment.
|
||||
|
||||
Returns:
|
||||
dict[str, set[str]]: Mapping from ancestor path to its descendant paths (including itself).
|
||||
"""
|
||||
ancestor_map = defaultdict(set)
|
||||
for path in paths:
|
||||
for i in range(steplen, len(path) + 1, steplen):
|
||||
ancestor = path[:i]
|
||||
ancestor_map[ancestor].add(path)
|
||||
return ancestor_map
|
||||
|
||||
|
||||
def filter_descendants(paths, root_paths, skip_sorting=False):
|
||||
"""
|
||||
Filters paths to keep only those that are descendants of any path in root_paths.
|
||||
|
||||
A path is considered a descendant of a root path if it starts with the root path.
|
||||
If `skip_sorting` is not set to True, the function will sort both lists before
|
||||
processing because both `paths` and `root_paths` need to be in lexicographic order
|
||||
before going through the algorithm.
|
||||
|
||||
Args:
|
||||
paths (iterable of str): List of paths to be filtered.
|
||||
root_paths (iterable of str): List of paths to check as potential prefixes.
|
||||
skip_sorting (bool): If True, assumes both `paths` and `root_paths` are already sorted.
|
||||
|
||||
Returns:
|
||||
list of str: A list of sorted paths that are descendants of any path in `root_paths`.
|
||||
"""
|
||||
results = []
|
||||
i = 0
|
||||
n = len(root_paths)
|
||||
|
||||
if not skip_sorting:
|
||||
paths.sort()
|
||||
root_paths.sort()
|
||||
|
||||
for path in paths:
|
||||
# Try to find a matching prefix in the sorted accessible paths
|
||||
while i < n:
|
||||
if path.startswith(root_paths[i]):
|
||||
results.append(path)
|
||||
break
|
||||
if root_paths[i] < path:
|
||||
i += 1
|
||||
else:
|
||||
# If paths[i] > path, no need to keep searching
|
||||
break
|
||||
return results
|
||||
56
src/backend/core/utils/treebeard.py
Normal file
56
src/backend/core/utils/treebeard.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Treebeard path collision handling utilities."""
|
||||
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||
from django.db import IntegrityError, transaction
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_tree_path_collision(exc):
|
||||
"""Return True when `exc` is caused by a Document.path uniqueness conflict.
|
||||
|
||||
Treebeard computes the materialized path by reading the current siblings;
|
||||
under concurrency two callers may compute the same value. Depending on
|
||||
timing this surfaces either as:
|
||||
|
||||
- `django.core.exceptions.ValidationError` raised by `full_clean()` /
|
||||
`validate_unique()` before the INSERT (BaseModel.save calls full_clean),
|
||||
- or `IntegrityError` from the database unique index when the validate
|
||||
step misses the conflict.
|
||||
"""
|
||||
if isinstance(exc, DjangoValidationError):
|
||||
message_dict = getattr(exc, "message_dict", None)
|
||||
if message_dict is not None:
|
||||
return "path" in message_dict
|
||||
return "path" in str(exc).lower()
|
||||
|
||||
# search in the IntegrityError exception
|
||||
return "impress_document_path_key" in str(exc).lower()
|
||||
|
||||
|
||||
def create_tree_node_with_retry(create_fn):
|
||||
"""Run `create_fn` in a fresh atomic block, retrying on path collisions.
|
||||
|
||||
The Document.path field carries a unique constraint, which is the source of
|
||||
truth that prevents duplicate paths. On collision we let the failed
|
||||
transaction roll back, and call `create_fn` again so treebeard recomputes
|
||||
the path from the latest state.
|
||||
"""
|
||||
max_attempts = settings.TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
with transaction.atomic():
|
||||
return create_fn()
|
||||
except (IntegrityError, DjangoValidationError) as exc:
|
||||
if not _is_tree_path_collision(exc) or attempt == max_attempts - 1:
|
||||
raise
|
||||
logger.info(
|
||||
"tree path collision on attempt %d/%d, retrying",
|
||||
attempt + 1,
|
||||
max_attempts,
|
||||
)
|
||||
|
||||
raise RuntimeError("create_tree_node_with_retry exited without result")
|
||||
55
src/backend/core/utils/users.py
Normal file
55
src/backend/core/utils/users.py
Normal file
@@ -0,0 +1,55 @@
|
||||
"""User sharing cache utilities."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.db import models as db
|
||||
from django.db.models import Subquery
|
||||
|
||||
from core import models
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_users_sharing_documents_with_cache_key(user):
|
||||
"""Generate a unique cache key for each user."""
|
||||
return f"users_sharing_documents_with_{user.id}"
|
||||
|
||||
|
||||
def users_sharing_documents_with(user):
|
||||
"""
|
||||
Returns a map of users sharing documents with the given user,
|
||||
sorted by last shared date.
|
||||
"""
|
||||
start_time = time.time()
|
||||
cache_key = get_users_sharing_documents_with_cache_key(user)
|
||||
cached_result = cache.get(cache_key)
|
||||
|
||||
if cached_result is not None:
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
"users_sharing_documents_with cache hit for user %s (took %.3fs)",
|
||||
user.id,
|
||||
elapsed,
|
||||
)
|
||||
return cached_result
|
||||
|
||||
user_docs_qs = models.DocumentAccess.objects.filter(user=user).values_list(
|
||||
"document_id", flat=True
|
||||
)
|
||||
shared_qs = (
|
||||
models.DocumentAccess.objects.filter(document_id__in=Subquery(user_docs_qs))
|
||||
.exclude(user=user)
|
||||
.values("user")
|
||||
.annotate(last_shared=db.Max("created_at"))
|
||||
)
|
||||
result = {item["user"]: item["last_shared"] for item in shared_qs}
|
||||
cache.set(cache_key, result, 86400) # Cache for 1 day
|
||||
elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
"users_sharing_documents_with cache miss for user %s (took %.3fs)",
|
||||
user.id,
|
||||
elapsed,
|
||||
)
|
||||
return result
|
||||
36
src/backend/core/utils/yjs.py
Normal file
36
src/backend/core/utils/yjs.py
Normal file
@@ -0,0 +1,36 @@
|
||||
"""Yjs document conversion utilities."""
|
||||
|
||||
import base64
|
||||
import re
|
||||
|
||||
import pycrdt
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from core import enums
|
||||
|
||||
|
||||
def base64_yjs_to_xml(base64_string):
|
||||
"""Extract xml from base64 yjs document."""
|
||||
|
||||
decoded_bytes = base64.b64decode(base64_string)
|
||||
|
||||
doc = pycrdt.Doc()
|
||||
doc.apply_update(decoded_bytes)
|
||||
return str(doc.get("document-store", type=pycrdt.XmlFragment))
|
||||
|
||||
|
||||
def base64_yjs_to_text(base64_string):
|
||||
"""Extract text from base64 yjs document."""
|
||||
|
||||
blocknote_structure = base64_yjs_to_xml(base64_string)
|
||||
soup = BeautifulSoup(blocknote_structure, "lxml-xml")
|
||||
return soup.get_text(separator=" ", strip=True)
|
||||
|
||||
|
||||
def extract_attachments(content):
|
||||
"""Helper method to extract media paths from a document's content."""
|
||||
if not content:
|
||||
return []
|
||||
|
||||
xml_content = base64_yjs_to_xml(content)
|
||||
return re.findall(enums.MEDIA_STORAGE_URL_EXTRACT, xml_content)
|
||||
@@ -161,8 +161,7 @@
|
||||
},
|
||||
"onboarding": {
|
||||
"enabled": true,
|
||||
"learn_more_url": "",
|
||||
"ready_template_url": ""
|
||||
"learn_more_url": ""
|
||||
},
|
||||
"help": {
|
||||
"documentation_url": ""
|
||||
|
||||
@@ -1081,6 +1081,12 @@ class Base(Configuration):
|
||||
60 * 60 * 24, environ_name="CONTENT_METADATA_CACHE_TIMEOUT", environ_prefix=None
|
||||
)
|
||||
|
||||
TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS = values.IntegerValue(
|
||||
10,
|
||||
environ_name="TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS",
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
@property
|
||||
def ENVIRONMENT(self):
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
# impress package
|
||||
#
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.11.9,<0.12"]
|
||||
build-backend = "uv_build"
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "impress"
|
||||
@@ -21,8 +21,9 @@ classifiers = [
|
||||
]
|
||||
description = "Docs is a collaborative text editor designed to address common challenges in knowledge building and sharing."
|
||||
keywords = ["Django", "Contacts", "Templates", "RBAC"]
|
||||
license = "MIT"
|
||||
requires-python = "~=3.13.0"
|
||||
license = { file = "LICENSE" }
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"beautifulsoup4==4.14.3",
|
||||
"boto3==1.42.93",
|
||||
@@ -96,13 +97,12 @@ dev = [
|
||||
"types-requests==2.33.0.20260408",
|
||||
]
|
||||
|
||||
[tool.uv.build-backend]
|
||||
module-root = ""
|
||||
source-exclude = [
|
||||
"**/tests/**",
|
||||
"**/test_*.py",
|
||||
"**/tests.py",
|
||||
]
|
||||
[tool.setuptools]
|
||||
packages = { find = { where = ["."], exclude = ["tests"] } }
|
||||
zip-safe = true
|
||||
|
||||
[tool.distutils.bdist_wheel]
|
||||
universal = true
|
||||
|
||||
[tool.ruff]
|
||||
exclude = [
|
||||
|
||||
7
src/backend/setup.py
Normal file
7
src/backend/setup.py
Normal file
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
"""Setup file for the impress module. All configuration stands in the setup.cfg file."""
|
||||
# coding: utf-8
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
setup()
|
||||
2440
src/backend/uv.lock
generated
2440
src/backend/uv.lock
generated
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -192,10 +192,10 @@ endobj
|
||||
(react-pdf)
|
||||
endobj
|
||||
55 0 obj
|
||||
(D:20260505110445Z)
|
||||
(D:20260403132357Z)
|
||||
endobj
|
||||
56 0 obj
|
||||
(chromium-4903-0-doc-export-override-content)
|
||||
(chromium-8651-0-doc-export-override-content)
|
||||
endobj
|
||||
52 0 obj
|
||||
<<
|
||||
@@ -216,7 +216,7 @@ endobj
|
||||
58 0 obj
|
||||
<<
|
||||
/Type /FontDescriptor
|
||||
/FontName /HRJUFI+Inter18pt-Regular
|
||||
/FontName /VIBRRZ+Inter18pt-Regular
|
||||
/Flags 4
|
||||
/FontBBox [-742.1875 -323.242187 2579.589844 1109.375]
|
||||
/ItalicAngle 0
|
||||
@@ -232,7 +232,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /CIDFontType2
|
||||
/BaseFont /HRJUFI+Inter18pt-Regular
|
||||
/BaseFont /VIBRRZ+Inter18pt-Regular
|
||||
/CIDSystemInfo <<
|
||||
/Registry (Adobe)
|
||||
/Ordering (Identity)
|
||||
@@ -247,7 +247,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /Type0
|
||||
/BaseFont /HRJUFI+Inter18pt-Regular
|
||||
/BaseFont /VIBRRZ+Inter18pt-Regular
|
||||
/Encoding /Identity-H
|
||||
/DescendantFonts [59 0 R]
|
||||
/ToUnicode 60 0 R
|
||||
@@ -256,7 +256,7 @@ endobj
|
||||
62 0 obj
|
||||
<<
|
||||
/Type /FontDescriptor
|
||||
/FontName /XKLDZR+Inter18pt-Bold
|
||||
/FontName /TDKMKH+Inter18pt-Bold
|
||||
/Flags 4
|
||||
/FontBBox [-790.527344 -334.472656 2580.566406 1114.746094]
|
||||
/ItalicAngle 0
|
||||
@@ -272,7 +272,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /CIDFontType2
|
||||
/BaseFont /XKLDZR+Inter18pt-Bold
|
||||
/BaseFont /TDKMKH+Inter18pt-Bold
|
||||
/CIDSystemInfo <<
|
||||
/Registry (Adobe)
|
||||
/Ordering (Identity)
|
||||
@@ -287,7 +287,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /Type0
|
||||
/BaseFont /XKLDZR+Inter18pt-Bold
|
||||
/BaseFont /TDKMKH+Inter18pt-Bold
|
||||
/Encoding /Identity-H
|
||||
/DescendantFonts [63 0 R]
|
||||
/ToUnicode 64 0 R
|
||||
@@ -296,7 +296,7 @@ endobj
|
||||
66 0 obj
|
||||
<<
|
||||
/Type /FontDescriptor
|
||||
/FontName /QHBJWW+Inter18pt-Italic
|
||||
/FontName /JYBWBW+Inter18pt-Italic
|
||||
/Flags 68
|
||||
/FontBBox [-747.558594 -323.242187 2595.703125 1109.375]
|
||||
/ItalicAngle -9.398804
|
||||
@@ -312,7 +312,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /CIDFontType2
|
||||
/BaseFont /QHBJWW+Inter18pt-Italic
|
||||
/BaseFont /JYBWBW+Inter18pt-Italic
|
||||
/CIDSystemInfo <<
|
||||
/Registry (Adobe)
|
||||
/Ordering (Identity)
|
||||
@@ -327,7 +327,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /Type0
|
||||
/BaseFont /QHBJWW+Inter18pt-Italic
|
||||
/BaseFont /JYBWBW+Inter18pt-Italic
|
||||
/Encoding /Identity-H
|
||||
/DescendantFonts [67 0 R]
|
||||
/ToUnicode 68 0 R
|
||||
@@ -336,7 +336,7 @@ endobj
|
||||
70 0 obj
|
||||
<<
|
||||
/Type /FontDescriptor
|
||||
/FontName /NBHLIK+GeistMono-Regular
|
||||
/FontName /DLRHPN+GeistMono-Regular
|
||||
/Flags 5
|
||||
/FontBBox [-1738 -247 654 1012]
|
||||
/ItalicAngle 0
|
||||
@@ -352,7 +352,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /CIDFontType2
|
||||
/BaseFont /NBHLIK+GeistMono-Regular
|
||||
/BaseFont /DLRHPN+GeistMono-Regular
|
||||
/CIDSystemInfo <<
|
||||
/Registry (Adobe)
|
||||
/Ordering (Identity)
|
||||
@@ -367,7 +367,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /Type0
|
||||
/BaseFont /NBHLIK+GeistMono-Regular
|
||||
/BaseFont /DLRHPN+GeistMono-Regular
|
||||
/Encoding /Identity-H
|
||||
/DescendantFonts [71 0 R]
|
||||
/ToUnicode 72 0 R
|
||||
@@ -376,7 +376,7 @@ endobj
|
||||
74 0 obj
|
||||
<<
|
||||
/Type /FontDescriptor
|
||||
/FontName /VMRKYJ+Inter18pt-BoldItalic
|
||||
/FontName /LHWXUO+Inter18pt-BoldItalic
|
||||
/Flags 68
|
||||
/FontBBox [-795.898437 -334.472656 2596.191406 1114.746094]
|
||||
/ItalicAngle -9.398804
|
||||
@@ -392,7 +392,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /CIDFontType2
|
||||
/BaseFont /VMRKYJ+Inter18pt-BoldItalic
|
||||
/BaseFont /LHWXUO+Inter18pt-BoldItalic
|
||||
/CIDSystemInfo <<
|
||||
/Registry (Adobe)
|
||||
/Ordering (Identity)
|
||||
@@ -407,7 +407,7 @@ endobj
|
||||
<<
|
||||
/Type /Font
|
||||
/Subtype /Type0
|
||||
/BaseFont /VMRKYJ+Inter18pt-BoldItalic
|
||||
/BaseFont /LHWXUO+Inter18pt-BoldItalic
|
||||
/Encoding /Identity-H
|
||||
/DescendantFonts [75 0 R]
|
||||
/ToUnicode 76 0 R
|
||||
@@ -713,21 +713,30 @@ endobj
|
||||
/Filter /FlateDecode
|
||||
>>
|
||||
stream
|
||||
xœí][<5B>㸱~ï_á?Ð
|
||||
oâXôC²'Áž‡ | ||||