wip

Import of (markdown) documents. See #1567 #1569.

Makefile

@@ -213,6 +213,7 @@ logs: ## display app-dev logs (follow mode)
 .PHONY: logs
 
 run-backend: ## Start only the backend application and all needed services
+	@$(COMPOSE) up --force-recreate -d docspec
 	@$(COMPOSE) up --force-recreate -d celery-dev
 	@$(COMPOSE) up --force-recreate -d y-provider-development
 	@$(COMPOSE) up --force-recreate -d nginx

compose.yml

@@ -217,3 +217,8 @@ services:
       kc_postgresql:
         condition: service_healthy
         restart: true
+
+  docspec:
+    image: ghcr.io/docspecio/api:2.0.2
+    ports:
+      - "4000:4000"

docs/env.md

@@ -103,6 +103,7 @@ These are the environment variables you can set for the `impress-backend` contai
 | USER_OIDC_ESSENTIAL_CLAIMS                      | Essential claims in OIDC token                                                                                              | []                                                                      |
 | Y_PROVIDER_API_BASE_URL                         | Y Provider url                                                                                                              |                                                                         |
 | Y_PROVIDER_API_KEY                              | Y provider API key                                                                                                          |                                                                         |
+| DOCSPEC_API_URL                                 | URL to endpoint of DocSpec conversion API                                                                                   |                                                                         |
 
 
 ## impress-frontend image

env.d/development/common

@@ -67,5 +67,7 @@ DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token
 Y_PROVIDER_API_BASE_URL=http://y-provider-development:4444/api/
 Y_PROVIDER_API_KEY=yprovider-api-key
 
+DOCSPEC_API_URL=http://docspec:4000/conversion
+
 # Theme customization
 THEME_CUSTOMIZATION_CACHE_TIMEOUT=15

env.d/development/common.e2e

@@ -3,6 +3,7 @@ BURST_THROTTLE_RATES="200/minute"
 COLLABORATION_API_URL=http://y-provider:4444/collaboration/api/
 SUSTAINED_THROTTLE_RATES="200/hour"
 Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/
+DOCSPEC_API_URL=http://docspec:4000/conversion
 
 # Throttle
 API_DOCUMENT_THROTTLE_RATE=1000/min

src/backend/core/api/serializers.py

@@ -10,6 +10,7 @@ from django.utils.functional import lazy
 from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 
+from core.services import mime_types
 import magic
 from rest_framework import serializers
 
@@ -17,7 +18,7 @@ from core import choices, enums, models, utils, validators
 from core.services.ai_services import AI_ACTIONS
 from core.services.converter_services import (
     ConversionError,
-    YdocConverter,
+    Converter,
 )
 
 
@@ -187,6 +188,7 @@ class DocumentSerializer(ListDocumentSerializer):
 
     content = serializers.CharField(required=False)
     websocket = serializers.BooleanField(required=False, write_only=True)
+    file = serializers.FileField(required=False, write_only=True, allow_null=True)
 
     class Meta:
         model = models.Document
@@ -203,6 +205,7 @@ class DocumentSerializer(ListDocumentSerializer):
             "deleted_at",
             "depth",
             "excerpt",
+            "file",
             "is_favorite",
             "link_role",
             "link_reach",
@@ -460,7 +463,11 @@ class ServerCreateDocumentSerializer(serializers.Serializer):
             language = user.language or language
 
         try:
-            document_content = YdocConverter().convert(validated_data["content"])
+            document_content = Converter().convert(
+                validated_data["content"],
+                mime_types.MARKDOWN,
+                mime_types.YJS
+            )
         except ConversionError as err:
             raise serializers.ValidationError(
                 {"content": ["Could not convert content"]}

src/backend/core/api/viewsets.py

@@ -39,14 +39,12 @@ from core import authentication, choices, enums, models
 from core.services.ai_services import AIService
 from core.services.collaboration_services import CollaborationService
 from core.services.converter_services import (
+    ConversionError,
     ServiceUnavailableError as YProviderServiceUnavailableError,
-)
-from core.services.converter_services import (
     ValidationError as YProviderValidationError,
+    Converter,
 )
-from core.services.converter_services import (
-    YdocConverter,
-)
+from core.services import mime_types
 from core.tasks.mail import send_ask_for_access_mail
 from core.utils import extract_attachments, filter_descendants
 
@@ -503,6 +501,27 @@ class DocumentViewSet(
                 "IN SHARE ROW EXCLUSIVE MODE;"
             )
 
+        # Remove file from validated_data as it's not a model field
+        # Process it if present
+        uploaded_file = serializer.validated_data.pop("file", None)
+
+        # If a file is uploaded, convert it to Yjs format and set as content
+        if uploaded_file:
+            try:
+                file_content = uploaded_file.read()
+
+                converter = Converter()
+                converted_content = converter.convert(
+                    file_content,
+                    content_type=uploaded_file.content_type,
+                    accept=mime_types.YJS
+                )
+                serializer.validated_data["content"] = converted_content
+            except ConversionError as err:
+                raise drf.exceptions.ValidationError(
+                    {"file": ["Could not convert file content"]}
+                ) from err
+
         obj = models.Document.add_root(
             creator=self.request.user,
             **serializer.validated_data,
@@ -1602,14 +1621,14 @@ class DocumentViewSet(
         if base64_content is not None:
             # Convert using the y-provider service
             try:
-                yprovider = YdocConverter()
+                yprovider = Converter()
                 result = yprovider.convert(
                     base64.b64decode(base64_content),
-                    "application/vnd.yjs.doc",
+                    mime_types.YJS,
                     {
-                        "markdown": "text/markdown",
-                        "html": "text/html",
-                        "json": "application/json",
+                        "markdown": mime_types.MARKDOWN,
+                        "html": mime_types.HTML,
+                        "json": mime_types.JSON,
                     }[content_format],
                 )
                 content = result

src/backend/core/services/converter_services.py

@@ -5,7 +5,9 @@ from base64 import b64encode
 from django.conf import settings
 
 import requests
+import typing
 
+from core.services import mime_types
 
 class ConversionError(Exception):
     """Base exception for conversion-related errors."""
@@ -19,8 +21,65 @@ class ServiceUnavailableError(ConversionError):
     """Raised when the conversion service is unavailable."""
 
 
+class ConverterProtocol(typing.Protocol):
+    def convert(self, text, content_type, accept): ...
+
+
+class Converter:
+    docspec: ConverterProtocol
+    ydoc: ConverterProtocol
+
+    def __init__(self):
+        self.docspec = DocSpecConverter()
+        self.ydoc = YdocConverter()
+
+    def convert(self, input, content_type, accept):
+        """Convert input into other formats using external microservices."""
+        
+        if content_type == mime_types.DOCX and accept == mime_types.YJS:
+            return self.convert(
+                self.docspec.convert(input, mime_types.DOCX, mime_types.BLOCKNOTE),
+                mime_types.BLOCKNOTE,
+                mime_types.YJS
+            )
+        
+        return self.ydoc.convert(input, content_type, accept)
+
+
+class DocSpecConverter:
+    """Service class for DocSpec conversion-related operations."""
+
+    def _request(self, url, data, content_type):
+        """Make a request to the DocSpec API."""
+
+        response = requests.post(
+            url,
+            headers={"Accept": mime_types.BLOCKNOTE},
+            files={"file": ("document.docx", data, content_type)},
+            timeout=settings.CONVERSION_API_TIMEOUT,
+            verify=settings.CONVERSION_API_SECURE,
+        )
+        response.raise_for_status()
+        return response
+    
+    def convert(self, data, content_type, accept):
+        """Convert a Document to BlockNote."""
+        if not data:
+            raise ValidationError("Input data cannot be empty")
+        
+        if content_type != mime_types.DOCX or accept != mime_types.BLOCKNOTE:
+            raise ValidationError(f"Conversion from {content_type} to {accept} is not supported.")
+        
+        try:
+            return self._request(settings.DOCSPEC_API_URL, data, content_type).content
+        except requests.RequestException as err:
+            raise ServiceUnavailableError(
+                "Failed to connect to DocSpec conversion service",
+            ) from err
+
+
 class YdocConverter:
-    """Service class for conversion-related operations."""
+    """Service class for YDoc conversion-related operations."""
 
     @property
     def auth_header(self):
@@ -45,7 +104,7 @@ class YdocConverter:
         return response
 
     def convert(
-        self, text, content_type="text/markdown", accept="application/vnd.yjs.doc"
+        self, text, content_type=mime_types.MARKDOWN, accept=mime_types.YJS
     ):
         """Convert a Markdown text into our internal format using an external microservice."""
 
@@ -59,14 +118,14 @@ class YdocConverter:
                 content_type,
                 accept,
             )
-            if accept == "application/vnd.yjs.doc":
+            if accept == mime_types.YJS:
                 return b64encode(response.content).decode("utf-8")
-            if accept in {"text/markdown", "text/html"}:
+            if accept in {mime_types.MARKDOWN, "text/html"}:
                 return response.text
-            if accept == "application/json":
+            if accept == mime_types.JSON:
                 return response.json()
             raise ValidationError("Unsupported format")
         except requests.RequestException as err:
             raise ServiceUnavailableError(
-                "Failed to connect to conversion service",
+                f"Failed to connect to YDoc conversion service {content_type}, {accept}",
             ) from err

src/backend/core/services/mime_types.py

@@ -0,0 +1,6 @@
+BLOCKNOTE = "application/vnd.blocknote+json"
+YJS = "application/vnd.yjs.doc"
+MARKDOWN = "text/markdown"
+JSON = "application/json"
+DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+HTML = "text/html"

src/backend/impress/settings.py

@@ -680,6 +680,12 @@ class Base(Configuration):
         environ_prefix=None,
     )
 
+    # DocSpec API microservice
+    DOCSPEC_API_URL = values.Value(
+        environ_name="DOCSPEC_API_URL",
+        environ_prefix=None
+    )
+
     # Conversion endpoint
     CONVERSION_API_ENDPOINT = values.Value(
         default="convert",

src/frontend/apps/impress/src/features/docs/docs-grid/api/useImportDoc.tsx

@@ -0,0 +1,43 @@
+import { useMutation, useQueryClient } from '@tanstack/react-query';
+
+import { APIError, errorCauses, fetchAPI } from '@/api';
+
+import { Doc, KEY_LIST_DOC } from '../../doc-management';
+
+export const importDoc = async (file: File): Promise<Doc> => {
+  const form = new FormData();
+  form.append('file', file);
+
+  const response = await fetchAPI(`documents/`, {
+    method: 'POST',
+    body: form,
+    withoutContentType: true,
+  });
+
+  if (!response.ok) {
+    throw new APIError('Failed to import the doc', await errorCauses(response));
+  }
+
+  return response.json() as Promise<Doc>;
+};
+
+interface ImportDocProps {
+  onSuccess?: (data: Doc) => void;
+  onError?: (error: APIError) => void;
+}
+
+export function useImportDoc({ onSuccess, onError }: ImportDocProps) {
+  const queryClient = useQueryClient();
+  return useMutation<Doc, APIError, File>({
+    mutationFn: importDoc,
+    onSuccess: (data) => {
+      void queryClient.resetQueries({
+        queryKey: [KEY_LIST_DOC],
+      });
+      onSuccess?.(data);
+    },
+    onError: (error) => {
+      onError?.(error);
+    },
+  });
+}

src/frontend/apps/impress/src/features/docs/docs-grid/components/DocsGrid.tsx

@@ -1,5 +1,5 @@
 import { Button } from '@openfun/cunningham-react';
-import { useMemo } from 'react';
+import { useMemo, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 import { InView } from 'react-intersection-observer';
 import { css } from 'styled-components';
@@ -9,6 +9,7 @@ import { DocDefaultFilter, useInfiniteDocs } from '@/docs/doc-management';
 import { useResponsiveStore } from '@/stores';
 
 import { useInfiniteDocsTrashbin } from '../api';
+import { useImportDoc } from '../api/useImportDoc';
 import { useResponsiveDocGrid } from '../hooks/useResponsiveDocGrid';
 
 import {
@@ -27,6 +28,7 @@ export const DocsGrid = ({
 
   const { isDesktop } = useResponsiveStore();
   const { flexLeft, flexRight } = useResponsiveDocGrid();
+  const importInputRef = useRef<HTMLInputElement>(null);
 
   const {
     data,
@@ -75,6 +77,33 @@ export const DocsGrid = ({
       title = t('All docs');
   }
 
+  const resetImportInput = () => {
+    if (!importInputRef.current) {
+      return;
+    }
+
+    importInputRef.current.value = '';
+  };
+
+  const { mutate: importDoc } = useImportDoc({
+    onSuccess: resetImportInput,
+    onError: resetImportInput,
+  });
+
+  const handleImport = (event: React.ChangeEvent<HTMLInputElement>) => {
+    console.log(event);
+
+    if (!event.target.files || event.target.files.length === 0) {
+      return;
+    }
+
+    const file = event.target.files[0];
+
+    console.log(file);
+
+    importDoc(file);
+  };
+
   return (
     <Box
       $position="relative"
@@ -107,6 +136,14 @@ export const DocsGrid = ({
           {title}
         </Text>
 
+        <input
+          type="file"
+          name="doc"
+          accept=".md,.docx"
+          onChange={handleImport}
+          ref={importInputRef}
+        ></input>
+
         {!hasDocs && !loading && (
           <Box $padding={{ vertical: 'sm' }} $align="center" $justify="center">
             <Text $size="sm" $variation="600" $weight="700">

src/frontend/servers/y-provider/__tests__/convert.test.ts

@@ -69,7 +69,7 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', 'wrong-api-key')
+      .set('authorization', `Bearer wrong-api-key`)
       .set('content-type', 'application/json');
 
     expect(response.status).toBe(401);
@@ -99,7 +99,7 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/json');
 
     expect(response.status).toBe(400);
@@ -114,7 +114,7 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/json')
       .send('');
 
@@ -129,9 +129,10 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'image/png')
       .send('randomdata');
+
     expect(response.status).toBe(415);
     expect(response.body).toStrictEqual({ error: 'Unsupported Content-Type' });
   });
@@ -141,38 +142,73 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'text/markdown')
       .set('accept', 'image/png')
       .send('# Header');
+
     expect(response.status).toBe(406);
     expect(response.body).toStrictEqual({ error: 'Unsupported format' });
   });
 
-  test.each([[apiKey], [`Bearer ${apiKey}`]])(
-    'POST /api/convert with correct content with Authorization: %s',
-    async (authHeader) => {
-      const app = initApp();
+  test('POST /api/convert BlockNote to Markdown', async () => {
+    const app = initApp();
+    const response = await request(app)
+      .post('/api/convert')
+      .set('origin', origin)
+      .set('authorization', `Bearer ${apiKey}`)
+      .set('content-type', 'application/vnd.blocknote+json')
+      .set('accept', 'text/markdown')
+      .send(expectedBlocks);
 
-      const response = await request(app)
-        .post('/api/convert')
-        .set('Origin', origin)
-        .set('Authorization', authHeader)
-        .set('content-type', 'text/markdown')
-        .set('accept', 'application/vnd.yjs.doc')
-        .send(expectedMarkdown);
+    expect(response.status).toBe(200);
+    expect(response.header['content-type']).toBe(
+      'text/markdown; charset=utf-8',
+    );
+    expect(typeof response.text).toBe('string');
+    expect(response.text.trim()).toBe(expectedMarkdown);
+  });
 
-      expect(response.status).toBe(200);
-      expect(response.body).toBeInstanceOf(Buffer);
+  test('POST /api/convert BlockNote to Yjs', async () => {
+    const app = initApp();
+    const editor = ServerBlockNoteEditor.create();
+    const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown);
+    const response = await request(app)
+      .post('/api/convert')
+      .set('origin', origin)
+      .set('authorization', `Bearer ${apiKey}`)
+      .set('content-type', 'application/vnd.blocknote+json')
+      .set('accept', 'application/vnd.yjs.doc')
+      .send(blocks)
+      .responseType('blob');
 
-      const editor = ServerBlockNoteEditor.create();
-      const doc = new Y.Doc();
-      Y.applyUpdate(doc, response.body);
-      const blocks = editor.yDocToBlocks(doc, 'document-store');
+    expect(response.status).toBe(200);
+    expect(response.header['content-type']).toBe('application/vnd.yjs.doc');
 
-      expect(blocks).toStrictEqual(expectedBlocks);
-    },
-  );
+    // Decode the Yjs response and verify it contains the correct blocks
+    const responseBuffer = Buffer.from(response.body as Buffer);
+    const ydoc = new Y.Doc();
+    Y.applyUpdate(ydoc, responseBuffer);
+    const decodedBlocks = editor.yDocToBlocks(ydoc, 'document-store');
+
+    expect(decodedBlocks).toStrictEqual(expectedBlocks);
+  });
+
+  test('POST /api/convert BlockNote to HTML', async () => {
+    const app = initApp();
+    const response = await request(app)
+      .post('/api/convert')
+      .set('origin', origin)
+      .set('authorization', `Bearer ${apiKey}`)
+      .set('content-type', 'application/vnd.blocknote+json')
+      .set('accept', 'text/html')
+      .send(expectedBlocks);
+
+    expect(response.status).toBe(200);
+    expect(response.header['content-type']).toBe('text/html; charset=utf-8');
+    expect(typeof response.text).toBe('string');
+    expect(response.text).toBe(expectedHTML);
+  });
 
   test('POST /api/convert Yjs to HTML', async () => {
     const app = initApp();
@@ -183,10 +219,11 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/vnd.yjs.doc')
       .set('accept', 'text/html')
       .send(Buffer.from(yjsUpdate));
+
     expect(response.status).toBe(200);
     expect(response.header['content-type']).toBe('text/html; charset=utf-8');
     expect(typeof response.text).toBe('string');
@@ -202,10 +239,11 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/vnd.yjs.doc')
       .set('accept', 'text/markdown')
       .send(Buffer.from(yjsUpdate));
+
     expect(response.status).toBe(200);
     expect(response.header['content-type']).toBe(
       'text/markdown; charset=utf-8',
@@ -223,15 +261,16 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/vnd.yjs.doc')
       .set('accept', 'application/json')
       .send(Buffer.from(yjsUpdate));
+
     expect(response.status).toBe(200);
     expect(response.header['content-type']).toBe(
       'application/json; charset=utf-8',
     );
-    expect(Array.isArray(response.body)).toBe(true);
+    expect(response.body).toBeInstanceOf(Array);
     expect(response.body).toStrictEqual(expectedBlocks);
   });
 
@@ -240,15 +279,16 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'text/markdown')
       .set('accept', 'application/json')
       .send(expectedMarkdown);
+
     expect(response.status).toBe(200);
     expect(response.header['content-type']).toBe(
       'application/json; charset=utf-8',
     );
-    expect(Array.isArray(response.body)).toBe(true);
+    expect(response.body).toBeInstanceOf(Array);
     expect(response.body).toStrictEqual(expectedBlocks);
   });
 
@@ -257,11 +297,12 @@ describe('Server Tests', () => {
     const response = await request(app)
       .post('/api/convert')
       .set('origin', origin)
-      .set('authorization', apiKey)
+      .set('authorization', `Bearer ${apiKey}`)
       .set('content-type', 'application/vnd.yjs.doc')
       .set('accept', 'application/json')
       .send(Buffer.from('notvalidyjs'));
+
     expect(response.status).toBe(400);
-    expect(response.body).toStrictEqual({ error: 'Invalid Yjs content' });
+    expect(response.body).toStrictEqual({ error: 'Invalid content' });
   });
 });

src/frontend/servers/y-provider/src/handlers/convertHandler.ts

@@ -14,27 +14,115 @@ interface ErrorResponse {
   error: string;
 }
 
+type ConversionResponseBody = Uint8Array | string | object | ErrorResponse;
+
+interface InputReader {
+  supportedContentTypes: string[];
+  read(data: Buffer): Promise<PartialBlock[]>;
+}
+
+interface OutputWriter {
+  supportedContentTypes: string[];
+  write(blocks: PartialBlock[]): Promise<ConversionResponseBody>;
+}
+
 const editor = ServerBlockNoteEditor.create<
   DefaultBlockSchema,
   DefaultInlineContentSchema,
   DefaultStyleSchema
 >();
 
+const ContentTypes = {
+  XMarkdown: 'text/x-markdown',
+  Markdown: 'text/markdown',
+  YJS: 'application/vnd.yjs.doc',
+  FormUrlEncoded: 'application/x-www-form-urlencoded',
+  OctetStream: 'application/octet-stream',
+  HTML: 'text/html',
+  BlockNote: 'application/vnd.blocknote+json',
+  JSON: 'application/json',
+} as const;
+
+const createYDocument = (blocks: PartialBlock[]) =>
+  editor.blocksToYDoc(blocks, 'document-store');
+
+const readers: InputReader[] = [
+  {
+    // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility
+    supportedContentTypes: [
+      ContentTypes.Markdown,
+      ContentTypes.XMarkdown,
+      ContentTypes.FormUrlEncoded,
+    ],
+    read: (data) => editor.tryParseMarkdownToBlocks(data.toString()),
+  },
+  {
+    supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream],
+    read: async (data) => {
+      const ydoc = new Y.Doc();
+      Y.applyUpdate(ydoc, data);
+      return editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[];
+    },
+  },
+  {
+    supportedContentTypes: [ContentTypes.BlockNote],
+    read: async (data) => JSON.parse(data.toString()),
+  },
+];
+
+const writers: OutputWriter[] = [
+  {
+    supportedContentTypes: [ContentTypes.BlockNote, ContentTypes.JSON],
+    write: async (blocks) => blocks,
+  },
+  {
+    supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream],
+    write: async (blocks) => Y.encodeStateAsUpdate(createYDocument(blocks)),
+  },
+  {
+    supportedContentTypes: [ContentTypes.Markdown, ContentTypes.XMarkdown],
+    write: (blocks) => editor.blocksToMarkdownLossy(blocks),
+  },
+  {
+    supportedContentTypes: [ContentTypes.HTML],
+    write: (blocks) => editor.blocksToHTMLLossy(blocks),
+  },
+];
+
+const normalizeContentType = (value: string) => value.split(';')[0];
+
 export const convertHandler = async (
   req: Request<object, Uint8Array | ErrorResponse, Buffer, object>,
-  res: Response<Uint8Array | string | object | ErrorResponse>,
+  res: Response<ConversionResponseBody>,
 ) => {
   if (!req.body || req.body.length === 0) {
     res.status(400).json({ error: 'Invalid request: missing content' });
     return;
   }
 
-  const contentType = (req.header('content-type') || 'text/markdown').split(
-    ';',
-  )[0];
-  const accept = (req.header('accept') || 'application/vnd.yjs.doc').split(
-    ';',
-  )[0];
+  const contentType = normalizeContentType(
+    req.header('content-type') || ContentTypes.Markdown,
+  );
+
+  const reader = readers.find((reader) =>
+    reader.supportedContentTypes.includes(contentType),
+  );
+
+  if (!reader) {
+    res.status(415).json({ error: 'Unsupported Content-Type' });
+    return;
+  }
+
+  const accept = normalizeContentType(req.header('accept') || ContentTypes.YJS);
+
+  const writer = writers.find((writer) =>
+    writer.supportedContentTypes.includes(accept),
+  );
+
+  if (!writer) {
+    res.status(406).json({ error: 'Unsupported format' });
+    return;
+  }
 
   let blocks:
     | PartialBlock<
@@ -44,63 +132,23 @@ export const convertHandler = async (
       >[]
     | null = null;
   try {
-    // First, convert from the input format to blocks
-    // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility
-    if (
-      contentType === 'text/markdown' ||
-      contentType === 'application/x-www-form-urlencoded'
-    ) {
-      blocks = await editor.tryParseMarkdownToBlocks(req.body.toString());
-    } else if (
-      contentType === 'application/vnd.yjs.doc' ||
-      contentType === 'application/octet-stream'
-    ) {
-      try {
-        const ydoc = new Y.Doc();
-        Y.applyUpdate(ydoc, req.body);
-        blocks = editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[];
-      } catch (e) {
-        logger('Invalid Yjs content:', e);
-        res.status(400).json({ error: 'Invalid Yjs content' });
-        return;
-      }
-    } else {
-      res.status(415).json({ error: 'Unsupported Content-Type' });
+    try {
+      blocks = await reader.read(req.body);
+    } catch (e) {
+      logger('Invalid content:', e);
+      res.status(400).json({ error: 'Invalid content' });
       return;
     }
+
     if (!blocks || blocks.length === 0) {
       res.status(500).json({ error: 'No valid blocks were generated' });
       return;
     }
 
-    // Then, convert from blocks to the output format
-    if (accept === 'application/json') {
-      res.status(200).json(blocks);
-    } else {
-      const yDocument = editor.blocksToYDoc(blocks, 'document-store');
-
-      if (
-        accept === 'application/vnd.yjs.doc' ||
-        accept === 'application/octet-stream'
-      ) {
-        res
-          .status(200)
-          .setHeader('content-type', 'application/octet-stream')
-          .send(Y.encodeStateAsUpdate(yDocument));
-      } else if (accept === 'text/markdown') {
-        res
-          .status(200)
-          .setHeader('content-type', 'text/markdown')
-          .send(await editor.blocksToMarkdownLossy(blocks));
-      } else if (accept === 'text/html') {
-        res
-          .status(200)
-          .setHeader('content-type', 'text/html')
-          .send(await editor.blocksToHTMLLossy(blocks));
-      } else {
-        res.status(406).json({ error: 'Unsupported format' });
-      }
-    }
+    res
+      .status(200)
+      .setHeader('content-type', accept)
+      .send(await writer.write(blocks));
   } catch (e) {
     logger('conversion failed:', e);
     res.status(500).json({ error: 'An error occurred' });

src/helm/impress/templates/_helpers.tpl

@@ -166,6 +166,14 @@ Requires top level scope
 {{ include "impress.fullname" . }}-y-provider
 {{- end }}
 
+{{/*
+Full name for the docSpecApi
+
+Requires top level scope
+*/}}
+{{- define "impress.docSpecApi.fullname" -}}
+{{ include "impress.fullname" . }}-docspec-api
+{{- end }}
 
 {{/*
 Full name for the Celery Worker

src/helm/impress/templates/docspec_deployment.yaml

@@ -0,0 +1,156 @@
+{{- $envVars := include "impress.common.env" (list . .Values.docSpecApi) -}}
+{{- $fullName := include "impress.docSpecApi.fullname" . -}}
+{{- $component := "docspec" -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
+  annotations:
+    {{- with .Values.backend.dpAnnotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "impress.common.labels" (list . $component) | nindent 4 }}
+spec:
+  replicas: {{ .Values.docSpecApi.replicas }}
+  selector:
+    matchLabels:
+      {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.docSpecApi.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }}
+    spec:
+      {{- if $.Values.image.credentials }}
+      imagePullSecrets:
+        - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }}
+      {{- end}}
+      {{- if .Values.docSpecApi.serviceAccountName }}
+      serviceAccountName: {{ .Values.docSpecApi.serviceAccountName }}
+      {{- end }}
+      shareProcessNamespace: {{ .Values.docSpecApi.shareProcessNamespace }}
+      containers:
+        {{- with .Values.docSpecApi.sidecars }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        - name: {{ .Chart.Name }}
+          image: "{{ (.Values.docSpecApi.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.docSpecApi.image | default dict).tag | default .Values.image.tag }}"
+          imagePullPolicy: {{ (.Values.docSpecApi.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
+          {{- with .Values.yPrdocspecovider.command }}
+          command:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.docSpecApi.args }}
+          args:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          env:
+            {{- if $envVars}}
+            {{- $envVars | indent 12 }}
+            {{- end }}
+          {{- with .Values.docSpecApi.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.docSpecApi.service.targetPort }}
+              protocol: TCP
+          {{- if .Values.docSpecApi.probes.liveness }}
+          livenessProbe:
+            {{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.liveness (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.docSpecApi.probes.readiness }}
+          readinessProbe:
+            {{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.readiness (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.docSpecApi.probes.startup }}
+          startupProbe:
+            {{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.startup (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
+          {{- end }}
+          {{- with .Values.docSpecApi.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            {{- range $index, $value := .Values.mountFiles }}
+            - name: "files-{{ $index }}"
+              mountPath: {{ $value.path }}
+              subPath: content
+            {{- end }}
+            {{- range $name, $volume := .Values.docSpecApi.persistence }}
+            - name: "{{ $name }}"
+              mountPath: "{{ $volume.mountPath }}"
+            {{- end }}
+            {{- range .Values.docSpecApi.extraVolumeMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              subPath: {{ .subPath | default "" }}
+              readOnly: {{ .readOnly }}
+            {{- end }}
+      {{- with .Values.docSpecApi.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.docSpecApi.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.docSpecApi.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        {{- range $index, $value := .Values.mountFiles }}
+        - name: "files-{{ $index }}"
+          configMap:
+            name: "{{ include "impress.fullname" $ }}-files-{{ $index }}"
+        {{- end }}
+        {{- range $name, $volume := .Values.docSpecApi.persistence }}
+        - name: "{{ $name }}"
+          {{- if eq $volume.type "emptyDir" }}
+          emptyDir: {}
+          {{- else }}
+          persistentVolumeClaim:
+            claimName: "{{ $fullName }}-{{ $name }}"
+          {{- end }}
+        {{- end }}
+        {{- range .Values.docSpecApi.extraVolumes }}
+        - name: {{ .name }}
+          {{- if .existingClaim }}
+          persistentVolumeClaim:
+            claimName: {{ .existingClaim }}
+          {{- else if .hostPath }}
+          hostPath:
+            {{ toYaml .hostPath | nindent 12 }}
+          {{- else if .csi }}
+          csi:
+            {{- toYaml .csi | nindent 12 }}
+          {{- else if .configMap }}
+          configMap:
+            {{- toYaml .configMap | nindent 12 }}
+          {{- else if .emptyDir }}
+          emptyDir:
+            {{- toYaml .emptyDir | nindent 12 }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        {{- end }}
+---
+{{ if .Values.docSpecApi.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
+{{ end }}

src/helm/impress/templates/docspec_svc.yaml

@@ -0,0 +1,21 @@
+{{- $envVars := include "impress.common.env" (list . .Values.yProvider) -}}
+{{- $fullName := include "impress.docspec.fullname" . -}}
+{{- $component := "docspec" -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "impress.common.labels" (list . $component) | nindent 4 }}
+  annotations:
+    {{- toYaml $.Values.docspec.service.annotations | nindent 4 }}
+spec:
+  type: {{ .Values.docspec.service.type }}
+  ports:
+    - port: {{ .Values.docspec.service.port }}
+      targetPort: {{ .Values.docspec.service.targetPort }}
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "impress.common.selectorLabels" (list . $component) | nindent 4 }}

src/helm/impress/templates/ingress_docspec_api.yaml

@@ -0,0 +1,67 @@
+{{- if .Values.ingressDocSpecApi.enabled -}}
+{{- $fullName := include "impress.fullname" . -}}
+{{- if and .Values.ingressDocSpecApi.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingressDocSpecApi.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingressDocSpecApi.annotations "kubernetes.io/ingress.class" .Values.ingressCollaborationApi.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}-collaboration-api
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "impress.labels" . | nindent 4 }}
+  {{- with .Values.ingressDocSpecApi.annotations }}
+  annotations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and .Values.ingressDocSpecApi.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingressDocSpecApi.className }}
+  {{- end }}
+  {{- if .Values.ingressDocSpecApi.tls.enabled }}
+  tls:
+    {{- if .Values.ingressDocSpecApi.host }}
+    - secretName: {{ .Values.ingressDocSpecApi.tls.secretName | default (printf "%s-tls" $fullName) | quote }}
+      hosts:
+        - {{ .Values.ingressDocSpecApi.host | quote }}
+    {{- end }}
+    {{- range .Values.ingressDocSpecApi.tls.additional }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- if .Values.ingressDocSpecApi.host }}
+    - host: {{ .Values.ingressDocSpecApi.host | quote }}
+      http:
+        paths:
+          - path: {{ .Values.ingressDocSpecApi.path | quote }}
+            {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
+            pathType: ImplementationSpecific
+            {{- end }}
+            backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ include "impress.docspec.fullname" . }}
+                port:
+                  number: {{ .Values.docspec.service.port }}
+              {{- else }}
+              serviceName: {{ include "impress.docspec.fullname" . }}
+              servicePort: {{ .Values.docspec.service.port }}
+            {{- end }}
+          {{- with .Values.ingressDocSpecApi.customBackends }}
+            {{- toYaml . | nindent 10 }}
+          {{- end }}
+    {{- end }}
+{{- end }}

src/helm/impress/values.yaml

@@ -661,3 +661,109 @@ yProvider:
 
   ## @param yProvider.serviceAccountName Optional service account name to use for yProvider pods
   serviceAccountName: null
+
+
+## @section docSpecApi
+
+docSpecApi:
+  ## @param docSpecApi.image.repository Repository to use to pull impress's docSpecApi container image
+  ## @param docSpecApi.image.tag impress's docSpecApi container tag
+  ## @param docSpecApi.image.pullPolicy docSpecApi container image pull policy
+  image:
+    repository: ghcr.io/docspecio/api
+    pullPolicy: IfNotPresent
+    tag: "latest"
+
+  ## @param docSpecApi.command Override the docSpecApi container command
+  command: []
+
+  ## @param docSpecApi.args Override the docSpecApi container args
+  args: []
+
+  ## @param docSpecApi.replicas Amount of docSpecApi replicas
+  replicas: 3
+
+  ## @param docSpecApi.shareProcessNamespace Enable share process namedocSpecApi between containers
+  shareProcessNamespace: false
+
+  ## @param docSpecApi.sidecars Add sidecars containers to docSpecApi deployment
+  sidecars: []
+
+  ## @param docSpecApi.securityContext Configure docSpecApi Pod security context
+  securityContext: null
+
+  ## @param docSpecApi.envVars Configure docSpecApi container environment variables
+  ## @extra docSpecApi.envVars.BY_VALUE Example environment variable by setting value directly
+  ## @extra docSpecApi.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
+  ## @extra docSpecApi.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
+  ## @extra docSpecApi.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret
+  ## @extra docSpecApi.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret
+  ## @skip docSpecApi.envVars
+  envVars:
+    <<: *commonEnvVars
+
+  ## @param docSpecApi.podAnnotations Annotations to add to the docSpecApi Pod
+  podAnnotations: {}
+
+  ## @param docSpecApi.dpAnnotations Annotations to add to the docSpecApi Deployment
+  dpAnnotations: {}
+
+  ## @param docSpecApi.service.type docSpecApi Service type
+  ## @param docSpecApi.service.port docSpecApi Service listening port
+  ## @param docSpecApi.service.targetPort docSpecApi container listening port
+  ## @param docSpecApi.service.annotations Annotations to add to the docSpecApi Service
+  service:
+    type: ClusterIP
+    port: 443
+    targetPort: 4000
+    annotations: {}
+
+  ## @extra docSpecApi.probes.liveness.path Configure path for docSpecApi HTTP liveness probe
+  ## @extra docSpecApi.probes.liveness.targetPort Configure port for docSpecApi HTTP liveness probe
+  ## @extra docSpecApi.probes.liveness.initialDelaySeconds Configure initial delay for docSpecApi liveness probe
+  ## @extra docSpecApi.probes.liveness.initialDelaySeconds Configure timeout for docSpecApi liveness probe
+  ## @extra docSpecApi.probes.startup.path Configure path for docSpecApi HTTP startup probe
+  ## @extra docSpecApi.probes.startup.targetPort Configure port for docSpecApi HTTP startup probe
+  ## @extra docSpecApi.probes.startup.initialDelaySeconds Configure initial delay for docSpecApi startup probe
+  ## @extra docSpecApi.probes.startup.initialDelaySeconds Configure timeout for docSpecApi startup probe
+  ## @extra docSpecApi.probes.readiness.path Configure path for docSpecApi HTTP readiness probe
+  ## @extra docSpecApi.probes.readiness.targetPort Configure port for docSpecApi HTTP readiness probe
+  ## @extra docSpecApi.probes.readiness.initialDelaySeconds Configure initial delay for docSpecApi readiness probe
+  ## @extra docSpecApi.probes.readiness.initialDelaySeconds Configure timeout for docSpecApi readiness probe
+  probes:
+    liveness:
+      ## @param docSpecApi.probes.liveness.path
+      ## @param docSpecApi.probes.liveness.initialDelaySeconds
+      path: /health
+      initialDelaySeconds: 10
+
+  ## @param docSpecApi.resources Resource requirements for the docSpecApi container
+  resources: {}
+
+  ## @param docSpecApi.nodeSelector Node selector for the docSpecApi Pod
+  nodeSelector: {}
+
+  ## @param docSpecApi.tolerations Tolerations for the docSpecApi Pod
+  tolerations: []
+
+  ## @param docSpecApi.affinity Affinity for the docSpecApi Pod
+  affinity: {}
+
+  ## @param docSpecApi.persistence Additional volumes to create and mount on the docSpecApi. Used for debugging purposes
+  ## @extra docSpecApi.persistence.volume-name.size Size of the additional volume
+  ## @extra docSpecApi.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir
+  ## @extra docSpecApi.persistence.volume-name.mountPath Path where the volume should be mounted to
+  persistence: {}
+
+  ## @param docSpecApi.extraVolumeMounts Additional volumes to mount on the docSpecApi.
+  extraVolumeMounts: []
+
+  ## @param docSpecApi.extraVolumes Additional volumes to mount on the docSpecApi.
+  extraVolumes: []
+
+  ## @param docSpecApi.pdb.enabled Enable pdb on docSpecApi
+  pdb:
+    enabled: true
+
+  ## @param docSpecApi.serviceAccountName Optional service account name to use for docSpecApi pods
+  serviceAccountName: null