Compare commits

...

7 Commits

Author SHA1 Message Date
Arnaud Robin
9ee9f9152c Add real time api calling 2025-02-12 23:36:33 +01:00
Arnaud Robin
b7914dfeef WIP 2025-02-12 15:42:10 +01:00
Nathan Vasse
1b2f74660b wip 2025-02-12 14:35:23 +01:00
Nathan Vasse
25491aeb06 wip 2025-02-12 14:09:48 +01:00
Arnaud Robin
6bb56e6b72 WIP 2025-02-12 13:14:24 +01:00
Nathan Vasse
cdf157c72b wip 2025-02-11 16:51:42 +01:00
Nathan Vasse
e26fcdf985 wip 2025-02-11 15:53:29 +01:00
16 changed files with 41129 additions and 14687 deletions

View File

@@ -21,8 +21,8 @@ services:
- MINIO_ROOT_USER=impress
- MINIO_ROOT_PASSWORD=password
ports:
- '9000:9000'
- '9001:9001'
- "9000:9000"
- "9001:9001"
entrypoint: ""
command: minio server --console-address :9001 /data
volumes:
@@ -59,11 +59,11 @@ services:
- ./src/backend:/app
- ./data/static:/data/static
depends_on:
- postgresql
- mailcatcher
- redis
- createbuckets
- postgresql
- mailcatcher
- redis
- createbuckets
celery-dev:
user: ${DOCKER_USER:-1000}
image: impress:backend-development
@@ -122,7 +122,7 @@ services:
frontend-dev:
user: "${DOCKER_USER:-1000}"
build:
build:
context: .
dockerfile: ./src/frontend/Dockerfile
target: frontend-production
@@ -151,13 +151,13 @@ services:
image: node:18
user: "${DOCKER_USER:-1000}"
environment:
HOME: /tmp
HOME: /tmp
volumes:
- ".:/app"
y-provider:
user: ${DOCKER_USER:-1000}
build:
build:
context: .
dockerfile: ./src/frontend/servers/y-provider/Dockerfile
target: y-provider
@@ -169,6 +169,7 @@ services:
kc_postgresql:
image: postgres:14.3
platform: linux/amd64
ports:
- "5433:5432"
env_file:
@@ -196,7 +197,7 @@ services:
KC_DB_PASSWORD: pass
KC_DB_USERNAME: impress
KC_DB_SCHEMA: public
PROXY_ADDRESS_FORWARDING: 'true'
PROXY_ADDRESS_FORWARDING: "true"
ports:
- "8080:8080"
depends_on:

View File

@@ -10,6 +10,10 @@ LOGGING_LEVEL_HANDLERS_CONSOLE=INFO
LOGGING_LEVEL_LOGGERS_ROOT=INFO
LOGGING_LEVEL_LOGGERS_APP=INFO
# Y-Provider
Y_PROVIDER_API_KEY="yprovider-api-key"
Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/
# Python
PYTHONPATH=/app
@@ -54,6 +58,9 @@ AI_BASE_URL=https://openaiendpoint.com
AI_API_KEY=password
AI_MODEL=llama
# Accessibility API
ACCESSIBILITY_API_BASE_URL=https://localhost:8000
# Collaboration
COLLABORATION_API_URL=http://nginx:8083/collaboration/api/
COLLABORATION_SERVER_ORIGIN=http://localhost:3000

View File

@@ -16,6 +16,7 @@ from core.services.converter_services import (
ConversionError,
YdocConverter,
)
from core.services.ai_services import AIService
class UserSerializer(serializers.ModelSerializer):
@@ -306,7 +307,7 @@ class ServerCreateDocumentSerializer(serializers.Serializer):
if user:
email = user.email
language = user.language or language
try:
document_content = YdocConverter().convert_markdown(
validated_data["content"]
@@ -568,6 +569,56 @@ class AITranslateSerializer(serializers.Serializer):
if len(value.strip()) == 0:
raise serializers.ValidationError("Text field cannot be empty.")
return value
class AIPdfTranscribeSerializer(serializers.Serializer):
"""Serializer for AI PDF transcribe requests."""
pdfUrl = serializers.CharField(required=True)
def __init__(self, *args, **kwargs):
"""Initialize with user."""
self.user = kwargs.pop('user', None)
super().__init__(*args, **kwargs)
def validate_pdfUrl(self, value):
"""Ensure the pdfUrl field is a valid URL."""
if not value.startswith(settings.MEDIA_BASE_URL):
raise serializers.ValidationError("Invalid PDF URL format.")
return value
def create(self, validated_data):
"""Create a new document for the transcribed content."""
if not self.user:
raise serializers.ValidationError("User is required")
# Get the transcribed content from AI service
pdf_url = validated_data["pdfUrl"]
response = AIService().transcribe_pdf(pdf_url)
try:
# Convert the markdown content to YDoc format
document_content = YdocConverter().convert_markdown(response)
except ConversionError as err:
raise serializers.ValidationError(
{"content": [f"Could not convert transcribed content: {str(err)}"]}
) from err
# Create the document as root node with converted content
document = models.Document.add_root(
title="PDF Transcription",
content=document_content,
creator=self.user,
)
# Create owner access for the user
models.DocumentAccess.objects.create(
document=document,
role=models.RoleChoices.OWNER,
user=self.user,
)
return document
class MoveDocumentSerializer(serializers.Serializer):

View File

@@ -1079,6 +1079,41 @@ class DocumentViewSet(
return drf.response.Response(response, status=drf.status.HTTP_200_OK)
@drf.decorators.action(
detail=True,
methods=["post"],
name="Just proxy ai call",
url_path="ai-proxy"
)
def ai_proxy(self, request, *args, **kwargs):
"""
POST /api/v1.0/documents/<resource_id>/ai-transform
with expected data:
- text: str
- action: str [prompt, correct, rephrase, summarize]
Return JSON response with the processed text.
"""
print('PROXY 1')
# Check permissions first
# self.get_object()
print('PROXY 2')
print(request.data)
# serializer = serializers.AITransformSerializer(data=request.data)
# serializer.is_valid(raise_exception=True)
print('PROXY 3')
system_content = request.data["system"]
text = request.data["text"]
print('PROXY 4')
response = AIService().call_proxy(system_content, text)
print('PROXY 5')
print(response)
return drf.response.Response(response, status=drf.status.HTTP_200_OK)
@drf.decorators.action(
detail=True,
methods=["post"],
@@ -1107,6 +1142,32 @@ class DocumentViewSet(
return drf.response.Response(response, status=drf.status.HTTP_200_OK)
@drf.decorators.action(
detail=True,
methods=["post"],
name="Transcribe PDF with AI",
url_path="ai-pdf-transcribe",
throttle_classes=[utils.AIDocumentRateThrottle, utils.AIUserRateThrottle],
)
def ai_pdf_transcribe(self, request, *args, **kwargs):
"""
POST /api/v1.0/documents/<resource_id>/ai-pdf-transcribe
with expected data:
- pdfUrl: str
Return JSON response with the new document ID containing the transcription.
"""
serializer = serializers.AIPdfTranscribeSerializer(
data=request.data,
user=request.user
)
serializer.is_valid(raise_exception=True)
document = serializer.save()
return drf.response.Response(
{"document_id": str(document.id)},
status=drf.status.HTTP_201_CREATED
)
class DocumentAccessViewSet(
ResourceAccessViewsetMixin,

View File

@@ -2,13 +2,18 @@
import json
import re
import os
import requests
import botocore
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import default_storage
from openai import OpenAI
from core import enums
from core.models import Document
AI_ACTIONS = {
"prompt": (
@@ -55,6 +60,22 @@ class AIService:
raise ImproperlyConfigured("AI configuration not set")
self.client = OpenAI(base_url=settings.AI_BASE_URL, api_key=settings.AI_API_KEY)
def call_proxy(self, system_content, text):
messages = [
{"role": "system", "content": system_content},
{"role": "user", "content": text},
]
print('REQUEST', messages)
response = self.client.chat.completions.create(
model=settings.AI_MODEL,
messages=messages,
)
print('RESPONSE', response)
content = response.choices[0].message.content
print('CONTENT', content)
return content
def call_ai_api(self, system_content, text):
"""Helper method to call the OpenAI API and process the response."""
response = self.client.chat.completions.create(
@@ -96,3 +117,26 @@ class AIService:
language_display = enums.ALL_LANGUAGES.get(language, language)
system_content = AI_TRANSLATE.format(language=language_display)
return self.call_ai_api(system_content, text)
def transcribe_pdf(self, pdf_url):
"""Transcribe PDF using the accessibility hackathon API and create a new document."""
try:
media_prefix = os.path.join(settings.MEDIA_BASE_URL, "media")
key = pdf_url[len(media_prefix):]
pdf_response = default_storage.connection.meta.client.get_object(
Bucket=default_storage.bucket_name,
Key=key
)
pdf_content = pdf_response['Body'].read()
api_url = f"{settings.ACCESSIBILITY_API_BASE_URL}/transcribe/pdf"
files = {'file': ('document.pdf', pdf_content, 'application/pdf')}
headers = {'Accept': 'application/json'}
response = requests.post(api_url, files=files, headers=headers)
response.raise_for_status()
transcribed_text = response.json()['markdown_content']
return transcribed_text
except Exception as e:
raise RuntimeError(f"Failed to transcribe PDF: {str(e)}")

View File

@@ -517,6 +517,12 @@ class Base(Configuration):
AI_BASE_URL = values.Value(None, environ_name="AI_BASE_URL", environ_prefix=None)
AI_MODEL = values.Value(None, environ_name="AI_MODEL", environ_prefix=None)
ACCESSIBILITY_API_BASE_URL = values.Value(
None,
environ_name="ACCESSIBILITY_API_BASE_URL",
environ_prefix=None,
)
AI_DOCUMENT_RATE_THROTTLE_RATES = {
"minute": 5,
"hour": 100,

View File

@@ -34,6 +34,7 @@
"idb": "8.0.2",
"lodash": "4.17.21",
"luxon": "3.5.0",
"marked": "^15.0.7",
"next": "15.1.6",
"posthog-js": "1.215.6",
"react": "*",

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.9 KiB

View File

@@ -0,0 +1,39 @@
import { useMutation } from '@tanstack/react-query';
import { APIError, errorCauses, fetchAPI } from '@/api';
export type DocAIPdfTranscribe = {
docId: string;
pdfUrl: string;
};
export type DocAIPdfTranscribeResponse = {
document_id: string;
};
export const docAIPdfTranscribe = async ({
docId,
...params
}: DocAIPdfTranscribe): Promise<DocAIPdfTranscribeResponse> => {
const response = await fetchAPI(`documents/${docId}/ai-pdf-transcribe/`, {
method: 'POST',
body: JSON.stringify({
...params,
}),
});
if (!response.ok) {
throw new APIError(
'Failed to request pdf transcription',
await errorCauses(response),
);
}
return response.json() as Promise<DocAIPdfTranscribeResponse>;
};
export function useDocAIPdfTranscribe() {
return useMutation<DocAIPdfTranscribeResponse, APIError, DocAIPdfTranscribe>({
mutationFn: docAIPdfTranscribe,
});
}

View File

@@ -0,0 +1,88 @@
import {
useBlockNoteEditor,
useComponentsContext,
useSelectedBlocks,
} from '@blocknote/react';
import {
Loader,
VariantType,
useToastProvider,
} from '@openfun/cunningham-react';
import { useRouter } from 'next/router';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { Text } from '@/components';
import { useDocStore } from '@/features/docs/doc-management/';
import { useDocAIPdfTranscribe } from '../api/useDocAIPdfTranscribe';
export const AIPdfButton = () => {
const editor = useBlockNoteEditor();
const Components = useComponentsContext();
const selectedBlocks = useSelectedBlocks(editor);
const { t } = useTranslation();
const { currentDoc } = useDocStore();
const { toast } = useToastProvider();
const router = useRouter();
const { mutateAsync: requestAIPdf, isPending } = useDocAIPdfTranscribe();
const [isLoading, setIsLoading] = useState(false);
if (!Components || !currentDoc) {
return null;
}
const show = selectedBlocks.length === 1 && selectedBlocks[0].type === 'file';
if (!show) {
return null;
}
const handlePdfTranscription = async () => {
console.log('selectedBlocks', selectedBlocks);
const pdfBlock = selectedBlocks[0];
const props = pdfBlock.props as { url?: string };
const pdfUrl = props?.url;
console.log('pdfUrl', pdfUrl);
if (!props || !pdfUrl) {
toast(t('No PDF file found'), VariantType.ERROR);
return;
}
setIsLoading(true);
try {
const response = await requestAIPdf({
docId: currentDoc.id,
pdfUrl,
});
setTimeout(() => {
// router.push causes the following error:
// TypeError: Cannot read properties of undefined (reading 'isDestroyed')
// void router.push(`/docs/${response.document_id}`);
window.location.href = `/docs/${response.document_id}?albert=true`;
}, 1000);
} catch (error) {
console.error('error', error);
toast(t('Failed to transcribe PDF'), VariantType.ERROR);
}
};
return (
<Components.FormattingToolbar.Button
className="bn-button bn-menu-item"
data-test="ai-pdf-transcribe"
label="AI"
mainTooltip={t('Transcribe PDF')}
icon={
isLoading ? (
<Loader size="small" />
) : (
<Text $isMaterialIcon $size="l">
auto_awesome
</Text>
)
}
onClick={() => void handlePdfTranscription()}
/>
);
};

View File

@@ -0,0 +1,303 @@
import { Button, Input, Loader } from '@openfun/cunningham-react';
import { marked } from 'marked';
import { useSearchParams } from 'next/navigation';
import { useState } from 'react';
import styled from 'styled-components';
import { fetchAPI } from '@/api';
import { Box, Text } from '@/components';
import { Doc } from '@/features/docs';
import { useEditorStore } from '../../stores/useEditorStore';
export const AIButtonEl = styled.button`
background-image: url('/assets/ia_baguette.png');
background-size: cover;
width: 48px;
height: 48px;
border-radius: 50%;
border: none;
cursor: pointer;
background-color: transparent;
`;
export const SuggestionButton = styled.button`
display: flex;
align-items: center;
border: none;
cursor: pointer;
background-color: white;
color: var(--c--theme--colors--greyscale-900);
height: 32px;
padding: 0 0.5rem;
border-radius: 4px;
font-weight: 600;
&:hover,
&:focus {
background-color: var(--c--theme--colors--greyscale-100);
}
span.material-icons {
margin-right: 4px;
}
span.sub {
color: var(--c--theme--colors--greyscale-600);
margin-left: 4px;
font-weight: 500;
}
`;
export const AiButton = ({ doc }: { doc: Doc }) => {
const searchParams = useSearchParams();
const [isOpen, setIsOpen] = useState(searchParams.get('albert') === 'true');
return (
<>
<Box
$position="absolute"
$css={`
right: 0;
bottom: 0;
padding: 1rem;
margin: 1rem;
z-index: 1;
`}
>
<AIButtonEl
aria-label="Posez une question à Albert à propos de ce document"
onClick={() => setIsOpen(true)}
/>
</Box>
<AiChat doc={doc} isOpen={isOpen} onClose={() => setIsOpen(false)} />
</>
);
};
type Message = {
role: 'user' | 'assistant';
content: string;
};
const AiChat = (props: { isOpen: boolean; onClose: () => void; doc: Doc }) => {
const [prompt, setPrompt] = useState('');
const { editor } = useEditorStore();
const [isLoading, setIsLoading] = useState(false);
const [messages, setMessages] = useState<Message[]>([]);
if (!props.isOpen) {
return null;
}
const newPrompt = async (prompt: string) => {
if (!editor) {
return;
}
setIsLoading(true);
setMessages([...messages, { role: 'user', content: prompt }]);
const editorContentFormatted = await editor.blocksToMarkdownLossy();
const response = await fetchAPI(`documents/${props.doc.id}/ai-proxy/`, {
method: 'POST',
body: JSON.stringify({
system:
'You are a helpful assistant. You are given a text in markdown format and you need to answer the question. Here is the text: ' +
editorContentFormatted,
text: prompt,
}),
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const data = (await response.json()) as string;
console.log('response', data);
setMessages((messages) => [
...messages,
{ role: 'assistant', content: data },
]);
setIsLoading(false);
};
const submit = async (e: React.FormEvent<HTMLFormElement>) => {
e.preventDefault();
setPrompt(''); // Clear the prompt after submitting the form
await newPrompt(prompt);
};
return (
<Box
$position="absolute"
$css={`
right: 0;
bottom: 0;
padding: 1rem;
width: 450px;
min-height: min(61vh, 365px);
box-shadow: rgba(15, 15, 15, 0.04) 0px 0px 0px 1px, rgba(15, 15, 15, 0.03) 0px 3px 6px, rgba(15, 15, 15, 0.06) 0px 9px 24px;
max-height: max(-180px + 100vh, 365px);
overflow-y: auto;
border-radius: 16px;
background-color: white;
margin: 1rem;
z-index: 2;
`}
$direction="column"
>
<Box $direction="row" $align="center" $justify="space-between">
<Text $theme="greyscale" $variation="1000" $weight="bold" $size="s">
{messages.length == 0 ? '' : 'Demander à Albert'}
</Text>
<Button
size="small"
onClick={props.onClose}
color="tertiary-text"
icon={<span className="material-icons">close</span>}
/>
</Box>
{messages.length == 0 && (
<Box $gap="1rem" $position="relative" $css="top: -24px;">
<Box $gap="0.5rem">
<Box
$css={`
background-image: url('/assets/ia_baguette_question_mark.png');
background-size: cover;
width: 48px;
height: 48px;
border-radius: 50%;
border: none;
cursor: pointer;
`}
></Box>
<Text $theme="primary" $variation="800">
Bonjour, comment puis-je vous aider ?
</Text>
</Box>
<Box $gap="0.5rem">
<Text $theme="greyscale" $variation="1000" $weight="bold" $size="s">
Suggestions
</Text>
<Box>
<SuggestionButton
onClick={() =>
void newPrompt(
'Resume ce document sous forme textuelle uniquement',
)
}
>
<span className="material-icons">description</span>
Résumer <span className="sub">cette page</span>
</SuggestionButton>
<SuggestionButton
onClick={() =>
void newPrompt('Quel est le sujet principal de ce document ?')
}
>
<span className="material-icons">help_center</span>
Poser des questions <span className="sub">sur cette page</span>
</SuggestionButton>
</Box>
</Box>
</Box>
)}
<Box
$flex={1}
$direction="column"
$gap="1rem"
$css={`
overflow-y: auto;
font-size: 14px;
mask-image: linear-gradient(black calc(100% - 32px), transparent calc(100% - 4px));
padding-bottom: 32px;
`}
aria-live="polite"
>
{messages.map((message, index) => (
<Message key={index} message={message} />
))}
{(isLoading || false) && (
<Box $display="flex" $direction="row" $align="center" $gap="0.5rem">
<Loader size="small" />
Albert réfléchit ...
</Box>
)}
</Box>
<Box>
<form onSubmit={(e) => void submit(e)} style={{ width: '100%' }}>
<Input
type="text"
label="Posez votre question"
name="prompt"
fullWidth={true}
onChange={(e) => setPrompt(e.target.value)}
value={prompt} // Ensure the input value is updated with the state
rightIcon={<span className="material-icons">send</span>}
/>
</form>
</Box>
</Box>
);
};
const Message = ({ message }: { message: Message }) => {
return (
<Box>
<Box $direction="row" $align="center" $gap="0.5rem">
{message.role === 'user' ? (
<Box
aria-hidden={true}
$css={`
background-color:#417DC4;
width: 24px;
height: 24px;
border-radius: 50%;
border: none;
cursor: pointer;
color: white;
font-size: 10px;
align-items: center;
justify-content: center;
display: flex;
`}
>
VD
</Box>
) : (
<Box
aria-hidden={true}
$css={`
background-image: url('/assets/ia_baguette.png');
background-size: cover;
width: 24px;
height: 24px;
border-radius: 50%;
border: none;
cursor: pointer;
`}
></Box>
)}
<Text $weight="bold">
{message.role === 'user' ? 'Vous' : 'Albert'}
</Text>
</Box>
<Box
$css={`
font-size: 12px;
padding-left: 34px;
color: var(--c--theme--colors--greyscale-700);
p {
margin: 0;
}
`}
dangerouslySetInnerHTML={{
__html: marked.parse(message.content) as string,
}}
></Box>
</Box>
);
};

View File

@@ -8,21 +8,27 @@ import {
import React, { useCallback } from 'react';
import { AIGroupButton } from './AIButton';
import { AIPdfButton } from './AIPdfButton';
import { MarkdownButton } from './MarkdownButton';
export const BlockNoteToolbar = () => {
const formattingToolbar = useCallback(
({ blockTypeSelectItems }: FormattingToolbarProps) => (
<FormattingToolbar>
{getFormattingToolbarItems(blockTypeSelectItems)}
({ blockTypeSelectItems }: FormattingToolbarProps) => {
console.log('formattingToolbar', blockTypeSelectItems);
return (
<FormattingToolbar>
{getFormattingToolbarItems(blockTypeSelectItems)}
{/* Extra button to do some AI powered actions */}
<AIGroupButton key="AIButton" />
{/* Extra button to do some AI powered actions */}
<AIGroupButton key="AIButton" />
{/* Extra button to convert from markdown to json */}
<MarkdownButton key="customButton" />
</FormattingToolbar>
),
{/* Extra button to convert from markdown to json */}
<MarkdownButton key="customButton" />
<AIPdfButton />
</FormattingToolbar>
);
},
[],
);

View File

@@ -16,6 +16,7 @@ import { TableContent } from '@/features/docs/doc-table-content/';
import { Versions, useDocVersion } from '@/features/docs/doc-versioning/';
import { useResponsiveStore } from '@/stores';
import { AiButton } from './Ai/AiButton';
import { BlockNoteEditor, BlockNoteEditorVersion } from './BlockNoteEditor';
interface DocEditorProps {
@@ -38,6 +39,7 @@ export const DocEditor = ({ doc, versionId }: DocEditorProps) => {
return (
<>
<AiButton doc={doc} />
{isDesktop && !isVersion && (
<Box
$position="absolute"

25849
src/frontend/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff