From 27c58541bd555b8b9791ed6e2d8d7bdb252230ca Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Sat, 3 Jun 2023 19:28:07 -0700 Subject: [PATCH] =?UTF-8?q?inital=20commit=20=E2=9A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 10 + LICENSE | 21 + README.md | 59 +++ clean.sh | 2 + collector/.env.example | 1 + collector/.gitignore | 6 + collector/README.md | 45 ++ collector/hotdir/__HOTDIR__.md | 17 + collector/main.py | 81 +++ collector/requirements.txt | 221 ++++++++ collector/scripts/__init__.py | 0 collector/scripts/gitbook.py | 44 ++ collector/scripts/link.py | 139 +++++ collector/scripts/link_utils.py | 14 + collector/scripts/medium.py | 71 +++ collector/scripts/medium_utils.py | 71 +++ collector/scripts/substack.py | 78 +++ collector/scripts/substack_utils.py | 86 ++++ collector/scripts/utils.py | 10 + collector/scripts/watch/__init__.py | 0 collector/scripts/watch/convert/as_docx.py | 58 +++ .../scripts/watch/convert/as_markdown.py | 32 ++ collector/scripts/watch/convert/as_pdf.py | 36 ++ collector/scripts/watch/convert/as_text.py | 28 ++ collector/scripts/watch/filetypes.py | 12 + collector/scripts/watch/main.py | 20 + collector/scripts/watch/utils.py | 30 ++ collector/scripts/youtube.py | 55 ++ collector/scripts/yt_utils.py | 120 +++++ collector/watch.py | 21 + frontend/.env.production | 1 + frontend/.eslintrc.cjs | 15 + frontend/.gitignore | 25 + frontend/.nvmrc | 1 + frontend/index.html | 36 ++ frontend/jsconfig.json | 7 + frontend/package.json | 50 ++ frontend/postcss.config.js | 7 + frontend/public/favicon.ico | Bin 0 -> 3656 bytes frontend/public/fonts/AvenirNext.ttf | Bin 0 -> 46240 bytes frontend/src/App.jsx | 19 + frontend/src/AuthContext.jsx | 30 ++ frontend/src/components/DefaultChat/index.jsx | 254 ++++++++++ frontend/src/components/Modals/Keys.jsx | 163 ++++++ .../src/components/Modals/ManageWorkspace.jsx | 476 ++++++++++++++++++ .../src/components/Modals/NewWorkspace.jsx | 104 ++++ .../Sidebar/ActiveWorkspaces/index.jsx | 82 +++ .../src/components/Sidebar/IndexCount.jsx | 34 ++ frontend/src/components/Sidebar/LLMStatus.jsx | 49 ++ frontend/src/components/Sidebar/index.jsx | 133 +++++ frontend/src/components/UserIcon/index.jsx | 27 + .../ChatHistory/HistoricalMessage/index.jsx | 106 ++++ .../ChatHistory/PromptReply/index.jsx | 112 +++++ .../ChatContainer/ChatHistory/index.jsx | 70 +++ .../ChatContainer/PromptInput/index.jsx | 106 ++++ .../WorkspaceChat/ChatContainer/index.jsx | 87 ++++ .../WorkspaceChat/LoadingChat/index.jsx | 57 +++ .../src/components/WorkspaceChat/index.jsx | 62 +++ frontend/src/index.css | 293 +++++++++++ frontend/src/main.jsx | 15 + frontend/src/models/system.js | 38 ++ frontend/src/models/workspace.js | 77 +++ frontend/src/pages/404.jsx | 24 + frontend/src/pages/Main/index.jsx | 12 + frontend/src/pages/WorkspaceChat/index.jsx | 28 ++ frontend/src/utils/chat/index.js | 59 +++ frontend/src/utils/constants.js | 2 + frontend/src/utils/numbers.js | 16 + frontend/src/utils/paths.js | 19 + frontend/tailwind.config.js | 13 + frontend/vite.config.js | 59 +++ images/choices.png | Bin 0 -> 155547 bytes images/gcp-project-bar.png | Bin 0 -> 19182 bytes images/screenshots/SCREENSHOTS.md | 14 + images/screenshots/chat.png | Bin 0 -> 327698 bytes images/screenshots/document.png | Bin 0 -> 784059 bytes images/screenshots/home.png | Bin 0 -> 589234 bytes images/screenshots/keys.png | Bin 0 -> 527376 bytes package.json | 15 + server/.env.example | 8 + server/.gitignore | 7 + server/.nvmrc | 1 + server/documents/DOCUMENTS.md | 10 + server/endpoints/chat.js | 23 + server/endpoints/system.js | 34 ++ server/endpoints/workspaces.js | 75 +++ server/index.js | 59 +++ server/models/documents.js | 99 ++++ server/models/vectors.js | 63 +++ server/models/workspace.js | 63 +++ server/models/workspaceChats.js | 68 +++ server/package.json | 35 ++ server/utils/chats/commands/reset.js | 17 + server/utils/chats/index.js | 128 +++++ server/utils/files/index.js | 120 +++++ server/utils/http/index.js | 14 + server/utils/middleware/validatedRequest.js | 37 ++ server/utils/openAi/index.js | 64 +++ server/utils/pinecone/index.js | 279 ++++++++++ server/vector-cache/VECTOR_CACHE.md | 5 + 100 files changed, 5394 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 clean.sh create mode 100644 collector/.env.example create mode 100644 collector/.gitignore create mode 100644 collector/README.md create mode 100644 collector/hotdir/__HOTDIR__.md create mode 100644 collector/main.py create mode 100644 collector/requirements.txt create mode 100644 collector/scripts/__init__.py create mode 100644 collector/scripts/gitbook.py create mode 100644 collector/scripts/link.py create mode 100644 collector/scripts/link_utils.py create mode 100644 collector/scripts/medium.py create mode 100644 collector/scripts/medium_utils.py create mode 100644 collector/scripts/substack.py create mode 100644 collector/scripts/substack_utils.py create mode 100644 collector/scripts/utils.py create mode 100644 collector/scripts/watch/__init__.py create mode 100644 collector/scripts/watch/convert/as_docx.py create mode 100644 collector/scripts/watch/convert/as_markdown.py create mode 100644 collector/scripts/watch/convert/as_pdf.py create mode 100644 collector/scripts/watch/convert/as_text.py create mode 100644 collector/scripts/watch/filetypes.py create mode 100644 collector/scripts/watch/main.py create mode 100644 collector/scripts/watch/utils.py create mode 100644 collector/scripts/youtube.py create mode 100644 collector/scripts/yt_utils.py create mode 100644 collector/watch.py create mode 100644 frontend/.env.production create mode 100644 frontend/.eslintrc.cjs create mode 100644 frontend/.gitignore create mode 100644 frontend/.nvmrc create mode 100644 frontend/index.html create mode 100644 frontend/jsconfig.json create mode 100644 frontend/package.json create mode 100644 frontend/postcss.config.js create mode 100644 frontend/public/favicon.ico create mode 100644 frontend/public/fonts/AvenirNext.ttf create mode 100644 frontend/src/App.jsx create mode 100644 frontend/src/AuthContext.jsx create mode 100644 frontend/src/components/DefaultChat/index.jsx create mode 100644 frontend/src/components/Modals/Keys.jsx create mode 100644 frontend/src/components/Modals/ManageWorkspace.jsx create mode 100644 frontend/src/components/Modals/NewWorkspace.jsx create mode 100644 frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx create mode 100644 frontend/src/components/Sidebar/IndexCount.jsx create mode 100644 frontend/src/components/Sidebar/LLMStatus.jsx create mode 100644 frontend/src/components/Sidebar/index.jsx create mode 100644 frontend/src/components/UserIcon/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/LoadingChat/index.jsx create mode 100644 frontend/src/components/WorkspaceChat/index.jsx create mode 100644 frontend/src/index.css create mode 100644 frontend/src/main.jsx create mode 100644 frontend/src/models/system.js create mode 100644 frontend/src/models/workspace.js create mode 100644 frontend/src/pages/404.jsx create mode 100644 frontend/src/pages/Main/index.jsx create mode 100644 frontend/src/pages/WorkspaceChat/index.jsx create mode 100644 frontend/src/utils/chat/index.js create mode 100644 frontend/src/utils/constants.js create mode 100644 frontend/src/utils/numbers.js create mode 100644 frontend/src/utils/paths.js create mode 100644 frontend/tailwind.config.js create mode 100644 frontend/vite.config.js create mode 100644 images/choices.png create mode 100644 images/gcp-project-bar.png create mode 100644 images/screenshots/SCREENSHOTS.md create mode 100644 images/screenshots/chat.png create mode 100644 images/screenshots/document.png create mode 100644 images/screenshots/home.png create mode 100644 images/screenshots/keys.png create mode 100644 package.json create mode 100644 server/.env.example create mode 100644 server/.gitignore create mode 100644 server/.nvmrc create mode 100644 server/documents/DOCUMENTS.md create mode 100644 server/endpoints/chat.js create mode 100644 server/endpoints/system.js create mode 100644 server/endpoints/workspaces.js create mode 100644 server/index.js create mode 100644 server/models/documents.js create mode 100644 server/models/vectors.js create mode 100644 server/models/workspace.js create mode 100644 server/models/workspaceChats.js create mode 100644 server/package.json create mode 100644 server/utils/chats/commands/reset.js create mode 100644 server/utils/chats/index.js create mode 100644 server/utils/files/index.js create mode 100644 server/utils/http/index.js create mode 100644 server/utils/middleware/validatedRequest.js create mode 100644 server/utils/openAi/index.js create mode 100644 server/utils/pinecone/index.js create mode 100644 server/vector-cache/VECTOR_CACHE.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..9b3385212 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +v-env +.env +!.env.example + +node_modules +__pycache__ +v-env +*.lock +.DS_Store + diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..cc42d1d08 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) Mintplex Labs Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..6bbcfde5d --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# 🤖 AnythingLLM: A full-stack personalized AI assistant + +[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/tim.svg?style=social&label=Follow%20%40Timothy%20Carambat)](https://twitter.com/tcarambat) [![](https://dcbadge.vercel.app/api/server/6UyHPeGZAC?compact=true&style=flat)](https://discord.gg/6UyHPeGZAC) + +A full-stack application and tool suite that enables you to turn any document, resource, or piece of content into a piece of data that any LLM can use as reference during chatting. This application runs with very minimal overhead as by default the LLM and vectorDB are hosted remotely, but can be swapped for local instances. Currently this project supports Pinecone and OpenAI. + +![Chatting](/images/screenshots/chat.png) +[view more screenshots](/images/screenshots/SCREENSHOTS.md) + +### Watch the demo! + +_tbd_ + +### Product Overview +AnythingLLM aims to be a full-stack application where you can use commercial off-the-shelf LLMs with Long-term-memory solutions or use popular open source LLM and vectorDB solutions. + +Anything LLM is a full-stack product that you can run locally as well as host remotely and be able to chat intelligently with any documents you provide it. + +AnythingLLM divides your documents into objects called `workspaces`. A Workspace functions a lot like a thread, but with the addition of containerization of your documents. Workspaces can share documents, but they do not talk to each other so you can keep your context for each workspace clean. + +Some cool features of AnythingLLM +- Atomically manage documents to be used in long-term-memory from a simple UI +- Two chat modes `conversation` and `query`. Conversation retains previous questions and amendments. Query is simple QA against your documents +- Each chat response contains a citation that is linked to the original content +- Simple technology stack for fast iteration +- Fully capable of being hosted remotely +- "Bring your own LLM" model and vector solution. _still in progress_ +- Extremely efficient cost-saving measures for managing very large documents. you'll never pay to embed a massive document or transcript more than once. 90% more cost effective than other LTM chatbots + +### Technical Overview +This monorepo consists of three main sections: +- `collector`: Python tools that enable you to quickly convert online resources or local documents into LLM useable format. +- `frontend`: A viteJS + React frontend that you can run to easily create and manage all your content the LLM can use. +- `server`: A nodeJS + express server to handle all the interactions and do all the vectorDB management and LLM interactions. + +### Requirements +- `yarn` and `node` on your machine +- `python` 3.8+ for running scripts in `collector/`. +- access to an LLM like `GPT-3.5`, `GPT-4`*. +- a [Pinecone.io](https://pinecone.io) free account*. +*you can use drop in replacements for these. This is just the easiest to get up and running fast. + +### How to get started +- `yarn setup` from the project root directory. + +This will fill in the required `.env` files you'll need in each of the application sections. Go fill those out before proceeding or else things won't work right. + +Next, you will need some content to embed. This could be a Youtube Channel, Medium articles, local text files, word documents, and the list goes on. This is where you will use the `collector/` part of the repo. + +[Go set up and run collector scripts](./collector/README.md) + +[Learn about documents](./server/documents/DOCUMENTS.md) + +[Learn about vector caching](./server/documents/VECTOR_CACHE.md) + +### Contributing +- create issue +- create PR with branch name format of `-` +- yee haw let's merge \ No newline at end of file diff --git a/clean.sh b/clean.sh new file mode 100644 index 000000000..e93aba6a2 --- /dev/null +++ b/clean.sh @@ -0,0 +1,2 @@ +# Easily kill process on port because sometimes nodemon fails to reboot +kill -9 $(lsof -t -i tcp:5000) \ No newline at end of file diff --git a/collector/.env.example b/collector/.env.example new file mode 100644 index 000000000..32e969687 --- /dev/null +++ b/collector/.env.example @@ -0,0 +1 @@ +GOOGLE_APIS_KEY= \ No newline at end of file diff --git a/collector/.gitignore b/collector/.gitignore new file mode 100644 index 000000000..f62b514d7 --- /dev/null +++ b/collector/.gitignore @@ -0,0 +1,6 @@ +outputs/*/*.json +hotdir/* +hotdir/processed/* +!hotdir/__HOTDIR__.md +!hotdir/processed + diff --git a/collector/README.md b/collector/README.md new file mode 100644 index 000000000..fae3dd98c --- /dev/null +++ b/collector/README.md @@ -0,0 +1,45 @@ +# How to collect data for vectorizing +This process should be run first. This will enable you to collect a ton of data across various sources. Currently the following services are supported: +- [x] YouTube Channels +- [x] Medium +- [x] Substack +- [x] Arbitrary Link +- [x] Gitbook +- [x] Local Files (.txt, .pdf, etc) [See full list](./hotdir/__HOTDIR__.md) +_these resources are under development or require PR_ +- Twitter +![Choices](../images/choices.png) + +### Requirements +- [ ] Python 3.8+ +- [ ] Google Cloud Account (for YouTube channels) +- [ ] `brew install pandoc` [pandoc](https://pandoc.org/installing.html) (for .ODT document processing) + +### Setup +This example will be using python3.9, but will work with 3.8+. Tested on MacOs. Untested on Windows +- install virtualenv for python3.8+ first before any other steps. `python3.9 -m pip install virutalenv` +- `cd collector` from root directory +- `python3.9 -m virtualenv v-env` +- `source v-env/bin/activate` +- `pip install -r requirements.txt` +- `cp .env.example .env` +- `python main.py` for interactive collection or `python watch.py` to process local documents. +- Select the option you want and follow follow the prompts - Done! +- run `deactivate` to get back to regular shell + +### Outputs +All JSON file data is cached in the `output/` folder. This is to prevent redundant API calls to services which may have rate limits to quota caps. Clearing out the `output/` folder will execute the script as if there was no cache. + +As files are processed you will see data being written to both the `collector/outputs` folder as well as the `server/documents` folder. Later in this process, once you boot up the server you will then bulk vectorize this content from a simple UI! + +If collection fails at any point in the process it will pick up where it last bailed out so you are not reusing credits. + +### How to get a Google Cloud API Key (YouTube data collection only) +**required to fetch YouTube transcripts and data** +- Have a google account +- [Visit the GCP Cloud Console](https://console.cloud.google.com/welcome) +- Click on dropdown in top right > Create new project. Name it whatever you like + - ![GCP Project Bar](../images/gcp-project-bar.png) +- [Enable YouTube Data APIV3](https://console.cloud.google.com/apis/library/youtube.googleapis.com) +- Once enabled generate a Credential key for this API +- Paste your key after `GOOGLE_APIS_KEY=` in your `collector/.env` file. diff --git a/collector/hotdir/__HOTDIR__.md b/collector/hotdir/__HOTDIR__.md new file mode 100644 index 000000000..5fa4459f5 --- /dev/null +++ b/collector/hotdir/__HOTDIR__.md @@ -0,0 +1,17 @@ +### What is the "Hot directory" + +This is the location where you can dump all supported file types and have them automatically converted and prepared to be digested by the vectorizing service and selected from the AnythingLLM frontend. + +Files dropped in here will only be processed when you are running `python watch.py` from the `collector` directory. + +Once converted the original file will be moved to the `hotdir/processed` folder so that the original document is still able to be linked to when referenced when attached as a source document during chatting. + +**Supported File types** +- `.md` +- `.text` +- `.pdf` + +__requires more development__ +- `.png .jpg etc` +- `.mp3` +- `.mp4` \ No newline at end of file diff --git a/collector/main.py b/collector/main.py new file mode 100644 index 000000000..6ec477c50 --- /dev/null +++ b/collector/main.py @@ -0,0 +1,81 @@ +import os +from whaaaaat import prompt, Separator +from scripts.youtube import youtube +from scripts.link import link, links +from scripts.substack import substack +from scripts.medium import medium +from scripts.gitbook import gitbook + +def main(): + if os.name == 'nt': + methods = { + '1': 'YouTube Channel', + '2': 'Article or Blog Link', + '3': 'Substack', + '4': 'Medium', + '5': 'Gitbook' + } + print("There are options for data collection to make this easier for you.\nType the number of the method you wish to execute.") + print("1. YouTube Channel\n2. Article or Blog Link (Single)\n3. Substack\n4. Medium\n\n[In development]:\nTwitter\n\n") + selection = input("Your selection: ") + method = methods.get(str(selection)) + else: + questions = [ + { + "type": "list", + "name": "collector", + "message": "What kind of data would you like to add to convert into long-term memory?", + "choices": [ + "YouTube Channel", + "Substack", + "Medium", + "Article or Blog Link(s)", + "Gitbook", + Separator(), + {"name": "Twitter", "disabled": "Needs PR"}, + "Abort", + ], + }, + ] + method = prompt(questions).get('collector') + + if('Article or Blog Link' in method): + questions = [ + { + "type": "list", + "name": "collector", + "message": "Do you want to scrape a single article/blog/url or many at once?", + "choices": [ + 'Single URL', + 'Multiple URLs', + 'Abort', + ], + }, + ] + method = prompt(questions).get('collector') + if(method == 'Single URL'): + link() + exit(0) + if(method == 'Multiple URLs'): + links() + exit(0) + + if(method == 'Abort'): exit(0) + if(method == 'YouTube Channel'): + youtube() + exit(0) + if(method == 'Substack'): + substack() + exit(0) + if(method == 'Medium'): + medium() + exit(0) + if(method == 'Gitbook'): + gitbook() + exit(0) + + print("Selection was not valid.") + exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/collector/requirements.txt b/collector/requirements.txt new file mode 100644 index 000000000..f1744b68b --- /dev/null +++ b/collector/requirements.txt @@ -0,0 +1,221 @@ +about-time==4.2.1 +aiohttp==3.8.4 +aiosignal==1.3.1 +alive-progress==3.1.2 +anyio==3.7.0 +appdirs==1.4.4 +argilla==1.8.0 +async-timeout==4.0.2 +attrs==23.1.0 +backoff==2.2.1 +beautifulsoup4==4.12.2 +bs4==0.0.1 +certifi==2023.5.7 +cffi==1.15.1 +chardet==5.1.0 +charset-normalizer==3.1.0 +click==8.1.3 +commonmark==0.9.1 +cryptography==41.0.1 +cssselect==1.2.0 +dataclasses-json==0.5.7 +Deprecated==1.2.14 +et-xmlfile==1.1.0 +exceptiongroup==1.1.1 +fake-useragent==1.1.3 +frozenlist==1.3.3 +grapheme==0.6.0 +greenlet==2.0.2 +h11==0.14.0 +httpcore==0.16.3 +httpx==0.23.3 +idna==3.4 +importlib-metadata==6.6.0 +importlib-resources==5.12.0 +install==1.3.5 +joblib==1.2.0 +langchain==0.0.189 +lxml==4.9.2 +Markdown==3.4.3 +marshmallow==3.19.0 +marshmallow-enum==1.5.1 +monotonic==1.6 +msg-parser==1.2.0 +multidict==6.0.4 +mypy-extensions==1.0.0 +nltk==3.8.1 +numexpr==2.8.4 +numpy==1.23.5 +olefile==0.46 +openapi-schema-pydantic==1.2.4 +openpyxl==3.1.2 +packaging==23.1 +pandas==1.5.3 +parse==1.19.0 +pdfminer.six==20221105 +Pillow==9.5.0 +prompt-toolkit==1.0.14 +pycparser==2.21 +pydantic==1.10.8 +pyee==8.2.2 +Pygments==2.15.1 +pyobjc==9.1.1 +pyobjc-core==9.1.1 +pyobjc-framework-Accounts==9.1.1 +pyobjc-framework-AddressBook==9.1.1 +pyobjc-framework-AdSupport==9.1.1 +pyobjc-framework-AppleScriptKit==9.1.1 +pyobjc-framework-AppleScriptObjC==9.1.1 +pyobjc-framework-ApplicationServices==9.1.1 +pyobjc-framework-AudioVideoBridging==9.1.1 +pyobjc-framework-AuthenticationServices==9.1.1 +pyobjc-framework-AutomaticAssessmentConfiguration==9.1.1 +pyobjc-framework-Automator==9.1.1 +pyobjc-framework-AVFoundation==9.1.1 +pyobjc-framework-AVKit==9.1.1 +pyobjc-framework-BusinessChat==9.1.1 +pyobjc-framework-CalendarStore==9.1.1 +pyobjc-framework-CFNetwork==9.1.1 +pyobjc-framework-CloudKit==9.1.1 +pyobjc-framework-Cocoa==9.1.1 +pyobjc-framework-Collaboration==9.1.1 +pyobjc-framework-ColorSync==9.1.1 +pyobjc-framework-Contacts==9.1.1 +pyobjc-framework-ContactsUI==9.1.1 +pyobjc-framework-CoreAudio==9.1.1 +pyobjc-framework-CoreAudioKit==9.1.1 +pyobjc-framework-CoreBluetooth==9.1.1 +pyobjc-framework-CoreData==9.1.1 +pyobjc-framework-CoreHaptics==9.1.1 +pyobjc-framework-CoreLocation==9.1.1 +pyobjc-framework-CoreMedia==9.1.1 +pyobjc-framework-CoreMediaIO==9.1.1 +pyobjc-framework-CoreMIDI==9.1.1 +pyobjc-framework-CoreML==9.1.1 +pyobjc-framework-CoreMotion==9.1.1 +pyobjc-framework-CoreServices==9.1.1 +pyobjc-framework-CoreSpotlight==9.1.1 +pyobjc-framework-CoreText==9.1.1 +pyobjc-framework-CoreWLAN==9.1.1 +pyobjc-framework-CryptoTokenKit==9.1.1 +pyobjc-framework-DeviceCheck==9.1.1 +pyobjc-framework-DictionaryServices==9.1.1 +pyobjc-framework-DiscRecording==9.1.1 +pyobjc-framework-DiscRecordingUI==9.1.1 +pyobjc-framework-DiskArbitration==9.1.1 +pyobjc-framework-DVDPlayback==9.1.1 +pyobjc-framework-EventKit==9.1.1 +pyobjc-framework-ExceptionHandling==9.1.1 +pyobjc-framework-ExecutionPolicy==9.1.1 +pyobjc-framework-ExternalAccessory==9.1.1 +pyobjc-framework-FileProvider==9.1.1 +pyobjc-framework-FileProviderUI==9.1.1 +pyobjc-framework-FinderSync==9.1.1 +pyobjc-framework-FSEvents==9.1.1 +pyobjc-framework-GameCenter==9.1.1 +pyobjc-framework-GameController==9.1.1 +pyobjc-framework-GameKit==9.1.1 +pyobjc-framework-GameplayKit==9.1.1 +pyobjc-framework-ImageCaptureCore==9.1.1 +pyobjc-framework-IMServicePlugIn==9.1.1 +pyobjc-framework-InputMethodKit==9.1.1 +pyobjc-framework-InstallerPlugins==9.1.1 +pyobjc-framework-InstantMessage==9.1.1 +pyobjc-framework-Intents==9.1.1 +pyobjc-framework-IOBluetooth==9.1.1 +pyobjc-framework-IOBluetoothUI==9.1.1 +pyobjc-framework-IOSurface==9.1.1 +pyobjc-framework-iTunesLibrary==9.1.1 +pyobjc-framework-LatentSemanticMapping==9.1.1 +pyobjc-framework-LaunchServices==9.1.1 +pyobjc-framework-libdispatch==9.1.1 +pyobjc-framework-libxpc==9.1.1 +pyobjc-framework-LinkPresentation==9.1.1 +pyobjc-framework-LocalAuthentication==9.1.1 +pyobjc-framework-MapKit==9.1.1 +pyobjc-framework-MediaAccessibility==9.1.1 +pyobjc-framework-MediaLibrary==9.1.1 +pyobjc-framework-MediaPlayer==9.1.1 +pyobjc-framework-MediaToolbox==9.1.1 +pyobjc-framework-Metal==9.1.1 +pyobjc-framework-MetalKit==9.1.1 +pyobjc-framework-MetalPerformanceShaders==9.1.1 +pyobjc-framework-ModelIO==9.1.1 +pyobjc-framework-MultipeerConnectivity==9.1.1 +pyobjc-framework-NaturalLanguage==9.1.1 +pyobjc-framework-NetFS==9.1.1 +pyobjc-framework-Network==9.1.1 +pyobjc-framework-NetworkExtension==9.1.1 +pyobjc-framework-NotificationCenter==9.1.1 +pyobjc-framework-OpenDirectory==9.1.1 +pyobjc-framework-OSAKit==9.1.1 +pyobjc-framework-OSLog==9.1.1 +pyobjc-framework-PencilKit==9.1.1 +pyobjc-framework-Photos==9.1.1 +pyobjc-framework-PhotosUI==9.1.1 +pyobjc-framework-PreferencePanes==9.1.1 +pyobjc-framework-PushKit==9.1.1 +pyobjc-framework-Quartz==9.1.1 +pyobjc-framework-QuickLookThumbnailing==9.1.1 +pyobjc-framework-SafariServices==9.1.1 +pyobjc-framework-SceneKit==9.1.1 +pyobjc-framework-ScreenSaver==9.1.1 +pyobjc-framework-ScriptingBridge==9.1.1 +pyobjc-framework-SearchKit==9.1.1 +pyobjc-framework-Security==9.1.1 +pyobjc-framework-SecurityFoundation==9.1.1 +pyobjc-framework-SecurityInterface==9.1.1 +pyobjc-framework-ServiceManagement==9.1.1 +pyobjc-framework-Social==9.1.1 +pyobjc-framework-SoundAnalysis==9.1.1 +pyobjc-framework-Speech==9.1.1 +pyobjc-framework-SpriteKit==9.1.1 +pyobjc-framework-StoreKit==9.1.1 +pyobjc-framework-SyncServices==9.1.1 +pyobjc-framework-SystemConfiguration==9.1.1 +pyobjc-framework-SystemExtensions==9.1.1 +pyobjc-framework-UserNotifications==9.1.1 +pyobjc-framework-VideoSubscriberAccount==9.1.1 +pyobjc-framework-VideoToolbox==9.1.1 +pyobjc-framework-Vision==9.1.1 +pyobjc-framework-WebKit==9.1.1 +pypandoc==1.11 +pyppeteer==1.0.2 +pyquery==2.0.0 +python-dateutil==2.8.2 +python-docx==0.8.11 +python-dotenv==0.21.1 +python-magic==0.4.27 +python-pptx==0.6.21 +python-slugify==8.0.1 +pytz==2023.3 +PyYAML==6.0 +regex==2023.5.5 +requests==2.31.0 +requests-html==0.10.0 +rfc3986==1.5.0 +rich==13.0.1 +six==1.16.0 +sniffio==1.3.0 +soupsieve==2.4.1 +SQLAlchemy==2.0.15 +tenacity==8.2.2 +text-unidecode==1.3 +tiktoken==0.4.0 +tqdm==4.65.0 +typer==0.9.0 +typing-inspect==0.9.0 +typing_extensions==4.6.3 +unstructured==0.7.1 +urllib3==1.26.16 +uuid==1.30 +w3lib==2.1.1 +wcwidth==0.2.6 +websockets==10.4 +whaaaaat==0.5.2 +wrapt==1.14.1 +xlrd==2.0.1 +XlsxWriter==3.1.2 +yarl==1.9.2 +youtube-transcript-api==0.6.0 +zipp==3.15.0 diff --git a/collector/scripts/__init__.py b/collector/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/collector/scripts/gitbook.py b/collector/scripts/gitbook.py new file mode 100644 index 000000000..3a0e9b502 --- /dev/null +++ b/collector/scripts/gitbook.py @@ -0,0 +1,44 @@ +import os, json +from langchain.document_loaders import GitbookLoader +from urllib.parse import urlparse +from datetime import datetime +from alive_progress import alive_it +from .utils import tokenize +from uuid import uuid4 + +def gitbook(): + url = input("Enter the URL of the GitBook you want to collect: ") + if(url == ''): + print("Not a gitbook URL") + exit(1) + + primary_source = urlparse(url) + output_path = f"./outputs/gitbook-logs/{primary_source.netloc}" + transaction_output_dir = f"../server/documents/gitbook-{primary_source.netloc}" + + if os.path.exists(output_path) == False:os.makedirs(output_path) + if os.path.exists(transaction_output_dir) == False: os.makedirs(transaction_output_dir) + loader = GitbookLoader(url, load_all_paths= primary_source.path in ['','/']) + for doc in alive_it(loader.load()): + metadata = doc.metadata + content = doc.page_content + source = urlparse(metadata.get('source')) + name = 'home' if source.path in ['','/'] else source.path.replace('/','_') + output_filename = f"doc-{name}.json" + transaction_output_filename = f"doc-{name}.json" + data = { + 'id': str(uuid4()), + 'url': metadata.get('source'), + "title": metadata.get('title'), + "description": metadata.get('title'), + "published": datetime.today().strftime('%Y-%m-%d %H:%M:%S'), + "wordCount": len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + with open(f"{output_path}/{output_filename}", 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=True, indent=4) + + with open(f"{transaction_output_dir}/{transaction_output_filename}", 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=True, indent=4) diff --git a/collector/scripts/link.py b/collector/scripts/link.py new file mode 100644 index 000000000..8d19f35aa --- /dev/null +++ b/collector/scripts/link.py @@ -0,0 +1,139 @@ +import os, json, tempfile +from urllib.parse import urlparse +from requests_html import HTMLSession +from langchain.document_loaders import UnstructuredHTMLLoader +from .link_utils import append_meta +from .utils import tokenize, ada_v2_cost + +# Example Channel URL https://tim.blog/2022/08/09/nft-insider-trading-policy/ +def link(): + print("[NOTICE]: The first time running this process it will download supporting libraries.\n\n") + fqdn_link = input("Paste in the URL of an online article or blog: ") + if(len(fqdn_link) == 0): + print("Invalid URL!") + exit(1) + + session = HTMLSession() + req = session.get(fqdn_link) + if(req.ok == False): + print("Could not reach this url!") + exit(1) + + req.html.render() + full_text = None + with tempfile.NamedTemporaryFile(mode = "w") as tmp: + tmp.write(req.html.html) + tmp.seek(0) + loader = UnstructuredHTMLLoader(tmp.name) + data = loader.load()[0] + full_text = data.page_content + tmp.close() + + link = append_meta(req, full_text, True) + if(len(full_text) > 0): + source = urlparse(req.url) + output_filename = f"website-{source.netloc}-{source.path.replace('/','_')}.json" + output_path = f"./outputs/website-logs" + + transaction_output_filename = f"article-{source.path.replace('/','_')}.json" + transaction_output_dir = f"../server/documents/website-{source.netloc}" + + if os.path.isdir(output_path) == False: + os.makedirs(output_path) + + if os.path.isdir(transaction_output_dir) == False: + os.makedirs(transaction_output_dir) + + full_text = append_meta(req, full_text) + tokenCount = len(tokenize(full_text)) + link['pageContent'] = full_text + link['token_count_estimate'] = tokenCount + + with open(f"{output_path}/{output_filename}", 'w', encoding='utf-8') as file: + json.dump(link, file, ensure_ascii=True, indent=4) + + with open(f"{transaction_output_dir}/{transaction_output_filename}", 'w', encoding='utf-8') as file: + json.dump(link, file, ensure_ascii=True, indent=4) + else: + print("Could not parse any meaningful data from this link or url.") + exit(1) + + print(f"\n\n[Success]: article or link content fetched!") + print(f"////////////////////////////") + print(f"Your estimated cost to embed this data using OpenAI's text-embedding-ada-002 model at $0.0004 / 1K tokens will cost {ada_v2_cost(tokenCount)} using {tokenCount} tokens.") + print(f"////////////////////////////") + exit(0) + +def links(): + links = [] + prompt = "Paste in the URL of an online article or blog: " + done = False + + while(done == False): + new_link = input(prompt) + if(len(new_link) == 0): + done = True + links = [*set(links)] + continue + + links.append(new_link) + prompt = f"\n{len(links)} links in queue. Submit an empty value when done pasting in links to execute collection.\nPaste in the next URL of an online article or blog: " + + if(len(links) == 0): + print("No valid links provided!") + exit(1) + + totalTokens = 0 + for link in links: + print(f"Working on {link}...") + session = HTMLSession() + req = session.get(link) + if(req.ok == False): + print(f"Could not reach {link} - skipping!") + continue + + req.html.render() + full_text = None + with tempfile.NamedTemporaryFile(mode = "w") as tmp: + tmp.write(req.html.html) + tmp.seek(0) + loader = UnstructuredHTMLLoader(tmp.name) + data = loader.load()[0] + full_text = data.page_content + tmp.close() + + link = append_meta(req, full_text, True) + if(len(full_text) > 0): + source = urlparse(req.url) + output_filename = f"website-{source.netloc}-{source.path.replace('/','_')}.json" + output_path = f"./outputs/website-logs" + + transaction_output_filename = f"article-{source.path.replace('/','_')}.json" + transaction_output_dir = f"../server/documents/website-{source.netloc}" + + if os.path.isdir(output_path) == False: + os.makedirs(output_path) + + if os.path.isdir(transaction_output_dir) == False: + os.makedirs(transaction_output_dir) + + full_text = append_meta(req, full_text) + tokenCount = len(tokenize(full_text)) + link['pageContent'] = full_text + link['token_count_estimate'] = tokenCount + totalTokens += tokenCount + + with open(f"{output_path}/{output_filename}", 'w', encoding='utf-8') as file: + json.dump(link, file, ensure_ascii=True, indent=4) + + with open(f"{transaction_output_dir}/{transaction_output_filename}", 'w', encoding='utf-8') as file: + json.dump(link, file, ensure_ascii=True, indent=4) + else: + print(f"Could not parse any meaningful data from {link}.") + continue + + print(f"\n\n[Success]: {len(links)} article or link contents fetched!") + print(f"////////////////////////////") + print(f"Your estimated cost to embed this data using OpenAI's text-embedding-ada-002 model at $0.0004 / 1K tokens will cost {ada_v2_cost(totalTokens)} using {totalTokens} tokens.") + print(f"////////////////////////////") + exit(0) \ No newline at end of file diff --git a/collector/scripts/link_utils.py b/collector/scripts/link_utils.py new file mode 100644 index 000000000..913653cc8 --- /dev/null +++ b/collector/scripts/link_utils.py @@ -0,0 +1,14 @@ +import json +from datetime import datetime +from dotenv import load_dotenv +load_dotenv() + +def append_meta(request, text, metadata_only = False): + meta = { + 'url': request.url, + 'title': request.html.find('title', first=True).text if len(request.html.find('title')) != 0 else '', + 'description': request.html.find('meta[name="description"]', first=True).attrs.get('content') if request.html.find('meta[name="description"]', first=True) != None else '', + 'published':request.html.find('meta[property="article:published_time"]', first=True).attrs.get('content') if request.html.find('meta[property="article:published_time"]', first=True) != None else datetime.today().strftime('%Y-%m-%d %H:%M:%S'), + 'wordCount': len(text.split(' ')), + } + return "Article JSON Metadata:\n"+json.dumps(meta)+"\n\n\nText Content:\n" + text if metadata_only == False else meta diff --git a/collector/scripts/medium.py b/collector/scripts/medium.py new file mode 100644 index 000000000..cc43d9fb2 --- /dev/null +++ b/collector/scripts/medium.py @@ -0,0 +1,71 @@ +import os, json +from urllib.parse import urlparse +from .utils import tokenize, ada_v2_cost +from .medium_utils import get_username, fetch_recent_publications, append_meta +from alive_progress import alive_it + +# Example medium URL: https://medium.com/@yujiangtham or https://davidall.medium.com +def medium(): + print("[NOTICE]: This method will only get the 10 most recent publishings.") + author_url = input("Enter the medium URL of the author you want to collect: ") + if(author_url == ''): + print("Not a valid medium.com/@author URL") + exit(1) + + handle = get_username(author_url) + if(handle is None): + print("This does not appear to be a valid medium.com/@author URL") + exit(1) + + publications = fetch_recent_publications(handle) + if(len(publications)==0): + print("There are no public or free publications by this creator - nothing to collect.") + exit(1) + + totalTokenCount = 0 + transaction_output_dir = f"../server/documents/medium-{handle}" + if os.path.isdir(transaction_output_dir) == False: + os.makedirs(transaction_output_dir) + + for publication in alive_it(publications): + pub_file_path = transaction_output_dir + f"/publication-{publication.get('id')}.json" + if os.path.exists(pub_file_path) == True: continue + + full_text = publication.get('pageContent') + if full_text is None or len(full_text) == 0: continue + + full_text = append_meta(publication, full_text) + item = { + 'id': publication.get('id'), + 'url': publication.get('url'), + 'title': publication.get('title'), + 'published': publication.get('published'), + 'wordCount': len(full_text.split(' ')), + 'pageContent': full_text, + } + + tokenCount = len(tokenize(full_text)) + item['token_count_estimate'] = tokenCount + + totalTokenCount += tokenCount + with open(pub_file_path, 'w', encoding='utf-8') as file: + json.dump(item, file, ensure_ascii=True, indent=4) + + print(f"[Success]: {len(publications)} scraped and fetched!") + print(f"\n\n////////////////////////////") + print(f"Your estimated cost to embed all of this data using OpenAI's text-embedding-ada-002 model at $0.0004 / 1K tokens will cost {ada_v2_cost(totalTokenCount)} using {totalTokenCount} tokens.") + print(f"////////////////////////////\n\n") + exit(0) + + + + + + + + + + + + + \ No newline at end of file diff --git a/collector/scripts/medium_utils.py b/collector/scripts/medium_utils.py new file mode 100644 index 000000000..37e6ab86d --- /dev/null +++ b/collector/scripts/medium_utils.py @@ -0,0 +1,71 @@ +import os, json, requests, re +from bs4 import BeautifulSoup + +def get_username(author_url): + if '@' in author_url: + pattern = r"medium\.com/@([\w-]+)" + match = re.search(pattern, author_url) + return match.group(1) if match else None + else: + # Given subdomain + pattern = r"([\w-]+).medium\.com" + match = re.search(pattern, author_url) + return match.group(1) if match else None + +def get_docid(medium_docpath): + pattern = r"medium\.com/p/([\w-]+)" + match = re.search(pattern, medium_docpath) + return match.group(1) if match else None + +def fetch_recent_publications(handle): + rss_link = f"https://medium.com/feed/@{handle}" + response = requests.get(rss_link) + if(response.ok == False): + print(f"Could not fetch RSS results for author.") + return [] + + xml = response.content + soup = BeautifulSoup(xml, 'xml') + items = soup.find_all('item') + publications = [] + + if os.path.isdir("./outputs/medium-logs") == False: + os.makedirs("./outputs/medium-logs") + + file_path = f"./outputs/medium-logs/medium-{handle}.json" + + if os.path.exists(file_path): + with open(file_path, "r") as file: + print(f"Returning cached data for Author {handle}. If you do not wish to use stored data then delete the file for this author to allow refetching.") + return json.load(file) + + for item in items: + tags = [] + for tag in item.find_all('category'): tags.append(tag.text) + content = BeautifulSoup(item.find('content:encoded').text, 'html.parser') + data = { + 'id': get_docid(item.find('guid').text), + 'title': item.find('title').text, + 'url': item.find('link').text.split('?')[0], + 'tags': ','.join(tags), + 'published': item.find('pubDate').text, + 'pageContent': content.get_text() + } + publications.append(data) + + with open(file_path, 'w+', encoding='utf-8') as json_file: + json.dump(publications, json_file, ensure_ascii=True, indent=2) + print(f"{len(publications)} articles found for author medium.com/@{handle}. Saved to medium-logs/medium-{handle}.json") + + return publications + +def append_meta(publication, text): + meta = { + 'url': publication.get('url'), + 'tags': publication.get('tags'), + 'title': publication.get('title'), + 'createdAt': publication.get('published'), + 'wordCount': len(text.split(' ')) + } + return "Article Metadata:\n"+json.dumps(meta)+"\n\nArticle Content:\n" + text + diff --git a/collector/scripts/substack.py b/collector/scripts/substack.py new file mode 100644 index 000000000..39bde6725 --- /dev/null +++ b/collector/scripts/substack.py @@ -0,0 +1,78 @@ +import os, json +from urllib.parse import urlparse +from .utils import tokenize, ada_v2_cost +from .substack_utils import fetch_all_publications, only_valid_publications, get_content, append_meta +from alive_progress import alive_it + +# Example substack URL: https://swyx.substack.com/ +def substack(): + author_url = input("Enter the substack URL of the author you want to collect: ") + if(author_url == ''): + print("Not a valid author.substack.com URL") + exit(1) + + source = urlparse(author_url) + if('substack.com' not in source.netloc or len(source.netloc.split('.')) != 3): + print("This does not appear to be a valid author.substack.com URL") + exit(1) + + subdomain = source.netloc.split('.')[0] + publications = fetch_all_publications(subdomain) + valid_publications = only_valid_publications(publications) + + if(len(valid_publications)==0): + print("There are no public or free preview newsletters by this creator - nothing to collect.") + exit(1) + + print(f"{len(valid_publications)} of {len(publications)} publications are readable publically text posts - collecting those.") + + totalTokenCount = 0 + transaction_output_dir = f"../server/documents/substack-{subdomain}" + if os.path.isdir(transaction_output_dir) == False: + os.makedirs(transaction_output_dir) + + for publication in alive_it(valid_publications): + pub_file_path = transaction_output_dir + f"/publication-{publication.get('id')}.json" + if os.path.exists(pub_file_path) == True: continue + + full_text = get_content(publication.get('canonical_url')) + if full_text is None or len(full_text) == 0: continue + + full_text = append_meta(publication, full_text) + item = { + 'id': publication.get('id'), + 'url': publication.get('canonical_url'), + 'thumbnail': publication.get('cover_image'), + 'title': publication.get('title'), + 'subtitle': publication.get('subtitle'), + 'description': publication.get('description'), + 'published': publication.get('post_date'), + 'wordCount': publication.get('wordcount'), + 'pageContent': full_text, + } + + tokenCount = len(tokenize(full_text)) + item['token_count_estimate'] = tokenCount + + totalTokenCount += tokenCount + with open(pub_file_path, 'w', encoding='utf-8') as file: + json.dump(item, file, ensure_ascii=True, indent=4) + + print(f"[Success]: {len(valid_publications)} scraped and fetched!") + print(f"\n\n////////////////////////////") + print(f"Your estimated cost to embed all of this data using OpenAI's text-embedding-ada-002 model at $0.0004 / 1K tokens will cost {ada_v2_cost(totalTokenCount)} using {totalTokenCount} tokens.") + print(f"////////////////////////////\n\n") + exit(0) + + + + + + + + + + + + + \ No newline at end of file diff --git a/collector/scripts/substack_utils.py b/collector/scripts/substack_utils.py new file mode 100644 index 000000000..c95303211 --- /dev/null +++ b/collector/scripts/substack_utils.py @@ -0,0 +1,86 @@ +import os, json, requests, tempfile +from requests_html import HTMLSession +from langchain.document_loaders import UnstructuredHTMLLoader + +def fetch_all_publications(subdomain): + file_path = f"./outputs/substack-logs/substack-{subdomain}.json" + + if os.path.isdir("./outputs/substack-logs") == False: + os.makedirs("./outputs/substack-logs") + + if os.path.exists(file_path): + with open(file_path, "r") as file: + print(f"Returning cached data for substack {subdomain}.substack.com. If you do not wish to use stored data then delete the file for this newsletter to allow refetching.") + return json.load(file) + + collecting = True + offset = 0 + publications = [] + + while collecting is True: + url = f"https://{subdomain}.substack.com/api/v1/archive?sort=new&offset={offset}" + response = requests.get(url) + if(response.ok == False): + print("Bad response - exiting collection") + collecting = False + continue + + data = response.json() + + if(len(data) ==0 ): + collecting = False + continue + + for publication in data: + publications.append(publication) + offset = len(publications) + + with open(file_path, 'w+', encoding='utf-8') as json_file: + json.dump(publications, json_file, ensure_ascii=True, indent=2) + print(f"{len(publications)} publications found for author {subdomain}.substack.com. Saved to substack-logs/channel-{subdomain}.json") + + return publications + +def only_valid_publications(publications= []): + valid_publications = [] + for publication in publications: + is_paid = publication.get('audience') != 'everyone' + if (is_paid and publication.get('should_send_free_preview') != True) or publication.get('type') != 'newsletter': continue + valid_publications.append(publication) + return valid_publications + +def get_content(article_link): + print(f"Fetching {article_link}") + if(len(article_link) == 0): + print("Invalid URL!") + return None + + session = HTMLSession() + req = session.get(article_link) + if(req.ok == False): + print("Could not reach this url!") + return None + + req.html.render() + + full_text = None + with tempfile.NamedTemporaryFile(mode = "w") as tmp: + tmp.write(req.html.html) + tmp.seek(0) + loader = UnstructuredHTMLLoader(tmp.name) + data = loader.load()[0] + full_text = data.page_content + tmp.close() + return full_text + +def append_meta(publication, text): + meta = { + 'url': publication.get('canonical_url'), + 'thumbnail': publication.get('cover_image'), + 'title': publication.get('title'), + 'subtitle': publication.get('subtitle'), + 'description': publication.get('description'), + 'createdAt': publication.get('post_date'), + 'wordCount': publication.get('wordcount') + } + return "Newsletter Metadata:\n"+json.dumps(meta)+"\n\nArticle Content:\n" + text \ No newline at end of file diff --git a/collector/scripts/utils.py b/collector/scripts/utils.py new file mode 100644 index 000000000..288da0f21 --- /dev/null +++ b/collector/scripts/utils.py @@ -0,0 +1,10 @@ +import tiktoken +encoder = tiktoken.encoding_for_model("text-embedding-ada-002") + +def tokenize(fullText): + return encoder.encode(fullText) + +def ada_v2_cost(tokenCount): + rate_per = 0.0004 / 1_000 # $0.0004 / 1K tokens + total = tokenCount * rate_per + return '${:,.2f}'.format(total) if total >= 0.01 else '< $0.01' \ No newline at end of file diff --git a/collector/scripts/watch/__init__.py b/collector/scripts/watch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/collector/scripts/watch/convert/as_docx.py b/collector/scripts/watch/convert/as_docx.py new file mode 100644 index 000000000..a639a1434 --- /dev/null +++ b/collector/scripts/watch/convert/as_docx.py @@ -0,0 +1,58 @@ +import os +from langchain.document_loaders import Docx2txtLoader, UnstructuredODTLoader +from slugify import slugify +from ..utils import guid, file_creation_time, write_to_server_documents, move_source +from ...utils import tokenize + +# Process all text-related documents. +def as_docx(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.txt') + fullpath = f"{parent_dir}/{filename}{ext}" + + loader = Docx2txtLoader(fullpath) + data = loader.load()[0] + content = data.page_content + + print(f"-- Working {fullpath} --") + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), + 'title': f"{filename}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") + +def as_odt(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.txt') + fullpath = f"{parent_dir}/{filename}{ext}" + + loader = UnstructuredODTLoader(fullpath) + data = loader.load()[0] + content = data.page_content + + print(f"-- Working {fullpath} --") + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), + 'title': f"{filename}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") \ No newline at end of file diff --git a/collector/scripts/watch/convert/as_markdown.py b/collector/scripts/watch/convert/as_markdown.py new file mode 100644 index 000000000..4c68d473b --- /dev/null +++ b/collector/scripts/watch/convert/as_markdown.py @@ -0,0 +1,32 @@ +import os +from langchain.document_loaders import UnstructuredMarkdownLoader +from slugify import slugify +from ..utils import guid, file_creation_time, write_to_server_documents, move_source +from ...utils import tokenize + +# Process all text-related documents. +def as_markdown(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.txt') + fullpath = f"{parent_dir}/{filename}{ext}" + + loader = UnstructuredMarkdownLoader(fullpath) + data = loader.load()[0] + content = data.page_content + + print(f"-- Working {fullpath} --") + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), + 'title': f"{filename}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") \ No newline at end of file diff --git a/collector/scripts/watch/convert/as_pdf.py b/collector/scripts/watch/convert/as_pdf.py new file mode 100644 index 000000000..53cee00a0 --- /dev/null +++ b/collector/scripts/watch/convert/as_pdf.py @@ -0,0 +1,36 @@ +import os +from langchain.document_loaders import PyPDFLoader +from slugify import slugify +from ..utils import guid, file_creation_time, write_to_server_documents, move_source +from ...utils import tokenize + +# Process all text-related documents. +def as_pdf(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.txt') + fullpath = f"{parent_dir}/{filename}{ext}" + + loader = PyPDFLoader(fullpath) + pages = loader.load_and_split() + + print(f"-- Working {fullpath} --") + for page in pages: + pg_num = page.metadata.get('page') + print(f"-- Working page {pg_num} --") + + content = page.page_content + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), + 'title': f"{filename}_pg{pg_num}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + write_to_server_documents(data, f"{slugify(filename)}-pg{pg_num}-{data.get('id')}") + + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") \ No newline at end of file diff --git a/collector/scripts/watch/convert/as_text.py b/collector/scripts/watch/convert/as_text.py new file mode 100644 index 000000000..b7935d622 --- /dev/null +++ b/collector/scripts/watch/convert/as_text.py @@ -0,0 +1,28 @@ +import os +from slugify import slugify +from ..utils import guid, file_creation_time, write_to_server_documents, move_source +from ...utils import tokenize + +# Process all text-related documents. +def as_text(**kwargs): + parent_dir = kwargs.get('directory', 'hotdir') + filename = kwargs.get('filename') + ext = kwargs.get('ext', '.txt') + fullpath = f"{parent_dir}/{filename}{ext}" + content = open(fullpath).read() + + print(f"-- Working {fullpath} --") + data = { + 'id': guid(), + 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), + 'title': f"{filename}{ext}", + 'description': "a custom file uploaded by the user.", + 'published': file_creation_time(fullpath), + 'wordCount': len(content), + 'pageContent': content, + 'token_count_estimate': len(tokenize(content)) + } + + write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}") + move_source(parent_dir, f"{filename}{ext}") + print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n") \ No newline at end of file diff --git a/collector/scripts/watch/filetypes.py b/collector/scripts/watch/filetypes.py new file mode 100644 index 000000000..5e2d818f0 --- /dev/null +++ b/collector/scripts/watch/filetypes.py @@ -0,0 +1,12 @@ +from .convert.as_text import as_text +from .convert.as_markdown import as_markdown +from .convert.as_pdf import as_pdf +from .convert.as_docx import as_docx, as_odt + +FILETYPES = { + '.txt': as_text, + '.md': as_markdown, + '.pdf': as_pdf, + '.docx': as_docx, + '.odt': as_odt, +} \ No newline at end of file diff --git a/collector/scripts/watch/main.py b/collector/scripts/watch/main.py new file mode 100644 index 000000000..f3bd3a1c5 --- /dev/null +++ b/collector/scripts/watch/main.py @@ -0,0 +1,20 @@ +import os +from .filetypes import FILETYPES + +RESERVED = ['__HOTDIR__.md'] +def watch_for_changes(directory): + for raw_doc in os.listdir(directory): + if os.path.isdir(f"{directory}/{raw_doc}") or raw_doc in RESERVED: continue + + filename, fileext = os.path.splitext(raw_doc) + if filename in ['.DS_Store'] or fileext == '': continue + + if fileext not in FILETYPES.keys(): + print(f"{fileext} not a supported file type for conversion. Please remove from hot directory.") + continue + + FILETYPES[fileext]( + directory=directory, + filename=filename, + ext=fileext, + ) \ No newline at end of file diff --git a/collector/scripts/watch/utils.py b/collector/scripts/watch/utils.py new file mode 100644 index 000000000..6eae31cd7 --- /dev/null +++ b/collector/scripts/watch/utils.py @@ -0,0 +1,30 @@ +import os, json +from datetime import datetime +from uuid import uuid4 + +def guid(): + return str(uuid4()) + +def file_creation_time(path_to_file): + try: + if os.name == 'nt': + return datetime.fromtimestamp(os.path.getctime(path_to_file)).strftime('%Y-%m-%d %H:%M:%S') + else: + stat = os.stat(path_to_file) + return datetime.fromtimestamp(stat.st_birthtime).strftime('%Y-%m-%d %H:%M:%S') + except AttributeError: + return datetime.today().strftime('%Y-%m-%d %H:%M:%S') + +def move_source(working_dir='hotdir', new_destination_filename= ''): + destination = f"{working_dir}/processed" + if os.path.exists(destination) == False: + os.mkdir(destination) + + os.replace(f"{working_dir}/{new_destination_filename}", f"{destination}/{new_destination_filename}") + return + +def write_to_server_documents(data, filename): + destination = f"../server/documents/custom-documents" + if os.path.exists(destination) == False: os.makedirs(destination) + with open(f"{destination}/{filename}.json", 'w', encoding='utf-8') as file: + json.dump(data, file, ensure_ascii=True, indent=4) diff --git a/collector/scripts/youtube.py b/collector/scripts/youtube.py new file mode 100644 index 000000000..382236528 --- /dev/null +++ b/collector/scripts/youtube.py @@ -0,0 +1,55 @@ +import os, json +from youtube_transcript_api import YouTubeTranscriptApi +from youtube_transcript_api.formatters import TextFormatter, JSONFormatter +from .utils import tokenize, ada_v2_cost +from .yt_utils import fetch_channel_video_information, get_channel_id, clean_text, append_meta, get_duration +from alive_progress import alive_it + +# Example Channel URL https://www.youtube.com/channel/UCmWbhBB96ynOZuWG7LfKong +# Example Channel URL https://www.youtube.com/@mintplex + +def youtube(): + channel_link = input("Paste in the URL of a YouTube channel: ") + channel_id = get_channel_id(channel_link) + + if channel_id == None or len(channel_id) == 0: + print("Invalid input - must be full YouTube channel URL") + exit(1) + + channel_data = fetch_channel_video_information(channel_id) + transaction_output_dir = f"../server/documents/youtube-{channel_data.get('channelTitle')}" + + if os.path.isdir(transaction_output_dir) == False: + os.makedirs(transaction_output_dir) + + print(f"\nFetching transcripts for {len(channel_data.get('items'))} videos - please wait.\nStopping and restarting will not refetch known transcripts in case there is an error.\nSaving results to: {transaction_output_dir}.") + totalTokenCount = 0 + for video in alive_it(channel_data.get('items')): + video_file_path = transaction_output_dir + f"/video-{video.get('id')}.json" + if os.path.exists(video_file_path) == True: + continue + + formatter = TextFormatter() + json_formatter = JSONFormatter() + try: + transcript = YouTubeTranscriptApi.get_transcript(video.get('id')) + raw_text = clean_text(formatter.format_transcript(transcript)) + duration = get_duration(json_formatter.format_transcript(transcript)) + + if(len(raw_text) > 0): + fullText = append_meta(video, duration, raw_text) + tokenCount = len(tokenize(fullText)) + video['pageContent'] = fullText + video['token_count_estimate'] = tokenCount + totalTokenCount += tokenCount + with open(video_file_path, 'w', encoding='utf-8') as file: + json.dump(video, file, ensure_ascii=True, indent=4) + except: + print("There was an issue getting the transcription of a video in the list - likely because captions are disabled. Skipping") + continue + + print(f"[Success]: {len(channel_data.get('items'))} video transcripts fetched!") + print(f"\n\n////////////////////////////") + print(f"Your estimated cost to embed all of this data using OpenAI's text-embedding-ada-002 model at $0.0004 / 1K tokens will cost {ada_v2_cost(totalTokenCount)} using {totalTokenCount} tokens.") + print(f"////////////////////////////\n\n") + exit(0) diff --git a/collector/scripts/yt_utils.py b/collector/scripts/yt_utils.py new file mode 100644 index 000000000..b4c74bbf7 --- /dev/null +++ b/collector/scripts/yt_utils.py @@ -0,0 +1,120 @@ +import json, requests, os, re +from slugify import slugify +from dotenv import load_dotenv +load_dotenv() + +def is_yt_short(videoId): + url = 'https://www.youtube.com/shorts/' + videoId + ret = requests.head(url) + return ret.status_code == 200 + +def get_channel_id(channel_link): + if('@' in channel_link): + pattern = r'https?://www\.youtube\.com/(@\w+)/?' + match = re.match(pattern, channel_link) + if match is False: return None + handle = match.group(1) + print('Need to map username to channelId - this can take a while sometimes.') + response = requests.get(f"https://yt.lemnoslife.com/channels?handle={handle}", timeout=20) + + if(response.ok == False): + print("Handle => ChannelId mapping endpoint is too slow - use regular youtube.com/channel URL") + return None + + json_data = response.json() + return json_data.get('items')[0].get('id') + else: + pattern = r"youtube\.com/channel/([\w-]+)" + match = re.search(pattern, channel_link) + return match.group(1) if match else None + + +def clean_text(text): + return re.sub(r"\[.*?\]", "", text) + +def append_meta(video, duration, text): + meta = { + 'youtubeURL': f"https://youtube.com/watch?v={video.get('id')}", + 'thumbnail': video.get('thumbnail'), + 'description': video.get('description'), + 'createdAt': video.get('published'), + 'videoDurationInSeconds': duration, + } + return "Video JSON Metadata:\n"+json.dumps(meta, indent=4)+"\n\n\nAudio Transcript:\n" + text + +def get_duration(json_str): + data = json.loads(json_str) + return data[-1].get('start') + +def fetch_channel_video_information(channel_id, windowSize = 50): + if channel_id == None or len(channel_id) == 0: + print("No channel id provided!") + exit(1) + + if os.path.isdir("./outputs/channel-logs") == False: + os.makedirs("./outputs/channel-logs") + + file_path = f"./outputs/channel-logs/channel-{channel_id}.json" + if os.path.exists(file_path): + with open(file_path, "r") as file: + print(f"Returning cached data for channel {channel_id}. If you do not wish to use stored data then delete the file for this channel to allow refetching.") + return json.load(file) + + if(os.getenv('GOOGLE_APIS_KEY') == None): + print("GOOGLE_APIS_KEY env variable not set!") + exit(1) + + done = False + currentPage = None + pageTokens = [] + items = [] + data = { + 'id': channel_id, + } + + print("Fetching first page of results...") + while(done == False): + url = f"https://www.googleapis.com/youtube/v3/search?key={os.getenv('GOOGLE_APIS_KEY')}&channelId={channel_id}&part=snippet,id&order=date&type=video&maxResults={windowSize}" + if(currentPage != None): + print(f"Fetching page ${currentPage}") + url += f"&pageToken={currentPage}" + + req = requests.get(url) + if(req.ok == False): + print("Could not fetch channel_id items!") + exit(1) + + response = req.json() + currentPage = response.get('nextPageToken') + if currentPage in pageTokens: + print('All pages iterated and logged!') + done = True + break + + for item in response.get('items'): + if 'id' in item and 'videoId' in item.get('id'): + if is_yt_short(item.get('id').get('videoId')): + print(f"Filtering out YT Short {item.get('id').get('videoId')}") + continue + + if data.get('channelTitle') is None: + data['channelTitle'] = slugify(item.get('snippet').get('channelTitle')) + + newItem = { + 'id': item.get('id').get('videoId'), + 'url': f"https://youtube.com/watch?v={item.get('id').get('videoId')}", + 'title': item.get('snippet').get('title'), + 'description': item.get('snippet').get('description'), + 'thumbnail': item.get('snippet').get('thumbnails').get('high').get('url'), + 'published': item.get('snippet').get('publishTime'), + } + items.append(newItem) + + pageTokens.append(currentPage) + + data['items'] = items + with open(file_path, 'w+', encoding='utf-8') as json_file: + json.dump(data, json_file, ensure_ascii=True, indent=2) + print(f"{len(items)} videos found for channel {data.get('channelTitle')}. Saved to channel-logs/channel-{channel_id}.json") + + return data \ No newline at end of file diff --git a/collector/watch.py b/collector/watch.py new file mode 100644 index 000000000..1223ae2a7 --- /dev/null +++ b/collector/watch.py @@ -0,0 +1,21 @@ +import _thread, time +from scripts.watch.main import watch_for_changes + +a_list = [] +WATCH_DIRECTORY = "hotdir" +def input_thread(a_list): + input() + a_list.append(True) + +def main(): + _thread.start_new_thread(input_thread, (a_list,)) + print(f"Watching '{WATCH_DIRECTORY}/' for new files.\n\nUpload files into this directory while this script is running to convert them.\nPress enter or crtl+c to exit script.") + while not a_list: + watch_for_changes(WATCH_DIRECTORY) + time.sleep(1) + + print("Stopping watching of hot directory.") + exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/frontend/.env.production b/frontend/.env.production new file mode 100644 index 000000000..4f79a0f8e --- /dev/null +++ b/frontend/.env.production @@ -0,0 +1 @@ +GENERATE_SOURCEMAP=false \ No newline at end of file diff --git a/frontend/.eslintrc.cjs b/frontend/.eslintrc.cjs new file mode 100644 index 000000000..ec601b2ce --- /dev/null +++ b/frontend/.eslintrc.cjs @@ -0,0 +1,15 @@ +module.exports = { + env: { browser: true, es2020: true }, + extends: [ + 'eslint:recommended', + 'plugin:react/recommended', + 'plugin:react/jsx-runtime', + 'plugin:react-hooks/recommended', + ], + parserOptions: { ecmaVersion: 'latest', sourceType: 'module' }, + settings: { react: { version: '18.2' } }, + plugins: ['react-refresh'], + rules: { + 'react-refresh/only-export-components': 'warn', + }, +} diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 000000000..76ec03f57 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +bundleinspector.html diff --git a/frontend/.nvmrc b/frontend/.nvmrc new file mode 100644 index 000000000..95c758cad --- /dev/null +++ b/frontend/.nvmrc @@ -0,0 +1 @@ +v18.12.1 \ No newline at end of file diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 000000000..695fb15a8 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,36 @@ + + + + + + + + AnythingLLM | Your personal LLM trained on anything + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/frontend/jsconfig.json b/frontend/jsconfig.json new file mode 100644 index 000000000..cffec6289 --- /dev/null +++ b/frontend/jsconfig.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "esnext", + "jsx": "react" + } +} \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 000000000..d141c2fc8 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,50 @@ +{ + "name": "anything-llm-frontend", + "private": false, + "version": "0.1.0", + "type": "module", + "scripts": { + "start": "vite --open", + "build": "vite build", + "lint": "yarn prettier --write ./src", + "preview": "vite preview" + }, + "dependencies": { + "@esbuild-plugins/node-globals-polyfill": "^0.1.1", + "@metamask/jazzicon": "^2.0.0", + "@react-oauth/google": "^0.11.0", + "buffer": "^6.0.3", + "email-validator": "^2.0.4", + "he": "^1.2.0", + "js-file-download": "^0.4.12", + "moment-timezone": "^0.5.43", + "pluralize": "^8.0.0", + "react": "^18.2.0", + "react-confetti-explosion": "^2.1.2", + "react-device-detect": "^2.2.2", + "react-dom": "^18.2.0", + "react-drag-drop-files": "^2.3.7", + "react-feather": "^2.0.10", + "react-loading-skeleton": "^3.1.0", + "react-router-dom": "^6.3.0", + "react-type-animation": "^3.0.1", + "text-case": "^1.0.9", + "truncate": "^3.0.0", + "uuid": "^9.0.0" + }, + "devDependencies": { + "@types/react": "^18.0.28", + "@types/react-dom": "^18.0.11", + "@vitejs/plugin-react": "^4.0.0-beta.0", + "autoprefixer": "^10.4.14", + "eslint": "^8.38.0", + "eslint-plugin-react": "^7.32.2", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.3.4", + "postcss": "^8.4.23", + "prettier": "^2.4.1", + "rollup-plugin-visualizer": "^5.9.0", + "tailwindcss": "^3.3.1", + "vite": "^4.3.0" + } +} diff --git a/frontend/postcss.config.js b/frontend/postcss.config.js new file mode 100644 index 000000000..869bdc11d --- /dev/null +++ b/frontend/postcss.config.js @@ -0,0 +1,7 @@ +import tailwind from 'tailwindcss' +import autoprefixer from 'autoprefixer' +import tailwindConfig from './tailwind.config.js' + +export default { + plugins: [tailwind(tailwindConfig), autoprefixer], +} \ No newline at end of file diff --git a/frontend/public/favicon.ico b/frontend/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..659944431173f1bdaf064787801a3735864f0ee1 GIT binary patch literal 3656 zcmV-O4!7|D009620AyqU0096X05T2$02TlM0EtjeM-2)Z3IG5A4M|8uQUCw|WB>pF zWC#WT003~}l~e!#00DDSM?wMF$t-^W000SaNLh0L01FcU01FcV0GgZ_000fwNklJ&JAkxA zB!mPAj{rd-)Km&+ho^{QBNLFlkdu*%kvovzB2Oc)BI}S%GVA2OotFR24*4^a z28Qmz`_pHj@j=Mvk(Y$J(pX$X z4o3!w_ODk!gOiZ^g}TmMe2#obw4b?vhW-k9S+K(b0s7^S742_bpmEX2e8El<0UC3( zX#cB#2E`)324iN__Zn!-2a&O&b#MTrcl95{`ot0-A3P>njBdy)f*oNcR3f{H5@QHR z$-4VcgM9K#pvC9`_r>r8^39`x5~B~YM(}`Jv1Q8^IC=7JO zXEZf6L1JQ}LlEN3nKMP3qzCd@p3!2A6+J2~_U+r(Dh4q=J{}qyEuBy{Cd~p;qR?^~ z+fk!NsRSYJ+__WwpLP_c#d2jA@}YkTNsv%ujDp6i?_^MJNY(=R}5;^$rEg_Tim&G2cn~+ z?<&Z|aZf9P-1t=ba7#U2;1p_dN9-UJcpfTDaY2|{eXCVDtdkgxSizaaIsS`ss6%-|No$x}zN zB|)y_;v3f|iXeARiyjV(33_7CT8v|&t#gOw=4PEx7Szj&*LcFG={fSbYjDRuE)i{+ z9)=Db>XRVEt?OS?1gZH*5#;uFqKC!8b)pE;$|B2wf_Z3YsLz6=4}3%s;}8?v zZ;BqC6XjYl;ze7ehs?}OpT!^s2M58W<9|^^xxQTyq`^|{Wbv#ZFN(HC4=E`rehET+ z=a0h_L8{g&g4E;O#BP}k(F?2~Yeie*4%@eH_gf5NeCHUjS1gf5Sqf15g(6CmWfIzJ zonp|U%Tm$SxWnktqYV*+xM{^JOmHt^g1co}h)9Dl=Q3o=HNDl;)Ielpq+x>md`J&P zkgItp#$iR2mRq8SOCSvrNRTwJP+r=J1q&7!CI&GqEJPjLZ+ylCcT2Pokp_K8kSU@q zafh_DG%iPXwzc;yxKnVs6;i%L(U zjzXto*GUk|RBEPApU(X)-@5T_l^|3Vq{$!e=a22pWD<^Bnc)7tNEb9%MiA;qOiD`P za%N;?INYh&$O!V0!?D{6Cvyw(+Jpg2!f}lzxC=`-Hj*HgC<+u87jrvv%*g)gUq{ty zhhrNqEQWRugmYb^@!|)HD7Bw6!ChES@D0lZ^${aRa0{|`>l{XqB6Mg@t7CsX zra$*})_m|XlW^R`Bpm-M$_*8kcb%zHc_w$(yS-F)coYOn}aYFLy2H7UY>h zY&9`glW?pR;R?rV?@eu1u<3(pqdt)5;_1=0Toe{J^T}=%9f}d@|lZ*TJ z@8@=2eCIf@R~IqCe61!~D9tBM9NNr%&7?%UW+iit+5=3&G2pv~WD8Hfj~_pt+j)PO z`U}n6V;M8!UeCO`9)aW^C2(K2cx?ugaNNQK_kb5trwA=j)z{ZUTwENt<4$~;#RT(> zn#a|@RD%3D^HuKazBv2|CgHe39oz%Ae1SAb6I#N}%gf_-+QScZ(ggFBkTutJC9bcm zE_;s=cPA6v1GapjwS+6I$zkx|!Q2j;{q`75Fkfn@AoFHE&piGLnc#jrP<_BslLG++ zv;mcumwUCm;QNvsCYY~tIL=5x9`6y)JOXnx)x@emxMWQmCqO`>+_Y)axSjR%z#cA% zM56^EezpDgOmP2)rkZGam{m2(bv65LENFQ_LIVF=P=5C7E{Q}l1o_QNeVO2{S-xO? ztB`uBGz2h(vUxNAJRu}F7%rd8XC7CpOcSJiY$Vj5%hFlCaM4Vcl(x|V0=OK3w!q+Z z(#T;?Y7&V{O%sHevtW$=@&)tK@@|(S(CS;$tNaxe6^<=1cpda%{_oUl1(~@=*4$qj zF37;XDVpUAY&Fr8wTZe5cIfr@TQly+V%Dr#+|G#z4~3dD1@04x$ecw}AvPw;J!4~# zu@3UAZFjP5+ct3Wz@M0e<3E|;PRkbz5AN=lw8^4OH2Q8@q^72FJ7>bUr`%azol%|^K{FRiLu~morMW3=DrY?)#y?eJ;kB_YzGnrt% zMlA^27LHSy*iCb%0~&m&)Z-jfkb+Bl%4r3J=~<4=pnL`O2a zF%+p2iH?%%0ewN5T^nf*9z5vPm!U9sA`{#93Tr~%GMgCrKJ!R6~*n4 zDH+c(!F;{uah0zxhNF(*`kbdGisa;E?)yKR-d(+X!Lb{JzTiH9dut6-HlJCzaG_V< z(A}TRW`g+&wIFQUj+P!Oee>tf=e~bvXfRwj_O4Df(U6^T$;W)>TYCiiK;@BHkeb(kM85j^EwZTW4(L#=C&jCCpJwrk!clby6TThvb_>R7R0bE z{t5~T9J#BmUAwxW4Z3!TYh}vAHaNU@p-UoBd30>^@VL7BN+{YDLS;*r2UYi{1)@$q zlH3WdpIXSg*?r8s$x)`sr#XplTl~qQ7_tc_EitI9taQxMU3lg*s3^;CW!34`VApB! zaam-eB}W2~5}PVpe)v?me`zjMd^fL^s-l&1`9kacDdAY>i#ta7B!(=Ax!c6j>mvnd zHnpy)3aTGh1s)z($HAv>EG|Ak{wfVwRSUe!`n-BKgp}K<+^lbG90%AgLz^{Tfuxpeo+KJv^@b zNNLdL^_nMKxJK~=Om6ajYqc_zZt(QNE8@)T!md~M59h~Eh(!?OU-WXQs`;!;di&XT@uB4e-fCz zw~D(pFBT=zj;H!C=eP=4l)xP5;@m-$mqzza(c{mX`|EFpX5N7=j40Gt$Nk{+9(HMh z$LkEUmJU$YBDW20tBG_hl(|P9F`}4vC>Sl^_vYJaR{t$s*Bd+3)x^LZ3dV?H&arva za>MNi6F0M@mkzoq`OlUw9)VkhAK@!i%0NNE{4tZtbM>e3pDg0q$STA zG6*y_|4v*BG(bw-4@a8X6dPdBSZGZCrO-tJjkh5eA)AD{(m#;xSxB2`e;NiHOd?1b zV@saWr312+xaCyL)=6i^vK2${@Z6W)BW5{*O-%7w~<8 z5bGPW7cc1y?~i;1-?!p=VPJ74yJ@FlFCp|1d>>u2W%bteKfLdEgvkFL=RRF?+4g{T zZbc5yUXSm;UcYt2mNohVKPN=_93d57-LQJwR$?M*l!rQqal?gIuCMv=rl$#U;QNY$ z8`rH~tL=QE0?(#!y^Bk8Iqs{fe58BbO4wsR-epy71yPtN(Nl{|+HG zT&KU>vigdxd?l^K@vXQ&aM9{5>soewcpf3!e~ZUkw_dz$`@-KRuO;OE03izR)=SoH zz4Nh`QI~lI$_x@lD{SAjp7@VdhSt9jKmR_?kT*N(UMlY=-=0*+Uf>_XeJaASv-rU? z{6mv65c$7w{SDa*VoB09>GOnm<^)+wl%yMD!4V_L5Y$S)N#^4lM|f@%_aPe#?%%m_ zJamV&r~gH=7(uVZkjyJeHVK@&VK`&4clT|zaIM$65;mZ+5jz#y&?W_j0I@N*MSBa^bz!AG-33k?UU~^8p_6q zQr3lgX5k)2&qpUe=KhX78-qRAvvI()e1vFOSu)q;huj*X=XspN#-8E*M9Id0jg2@4 z?0JlXSRbQ5%H+pMn9-g$;`~dbAX|vCHsSaUBIh5Ox?e6^j^q8rERMlVq=xGQ9lwcw zeUVhKao|TLPtm{P_&S_p+cIp|ql_T-7h+>$!1fpiHh$QwBJF44Jfr;$xHg~E@wbrz ze*nkV5-a*%;Oej`2<10pKLWl)T^MU7gY0}<97{Itm6LC?vFGnWojY(1_lRv^W6J6U zt=Kd4Zj>=hd>D%awkm9u^q)BXCyxJ#aX62}uqD8Aeb6MGvi|}4BiE7#$fM-z9D`cx=I|J5$E8q)+0e5rΠHZ_1?GM zd-=VW-h1)A7v4Ma-lOj|zWc*>kCobts{Y^nPz6S$^Zr4`FSf=p1*2z_5a%3|GJ2*+3@Z*vSrin$VPVe#e}T8 zWaeWRurHf0eQ!)W`PSRN_|KoR@gn56zy1v&?`;41FMlO|{Z}7_W#=s)Su(VE(cr-R zdHr+y=FINx>F(;BRp@AMYi((6YRos(*X6RcHJR#kDw(LNjK`vpa3~n?`@9~v%jvM& z1gphtG8*(ct)@b)QYz#!o}V2HryRH1}3x%KANAIAZB+kYzpKu zBk9sz6Y}H)vGh+^2ZxW4LgUDUB6;>L3nuue@njGW&vyrUCS+0kh3BqbJ5jl4I2bm* z?Vh?mf?_9JUBkhkdxDGNuMd~-H+OYl?Syd<=W#(i(>Fl|huQYrm0bY>OovG*E3g9bfoH-~%HZURNuZ62K_>PHeugMsnD_@ZG`&K(Sj zt?r+Ab>Z+4O+{C@Yh4ApLBtc}NQDMxH0%V%W$O{DZKvV^r|oGz!V#4Y^fR;e_ONa9 zMB(;P9E7_;I9#zzUwLlwl{;o$B6v)?iQqQzh>CSiD7q$;QX>MJCJL)3$nAk6saMAD zc+N=1Mw8mL;kBz*3{UW@(WoPYkM?XFn(*`wF2^Y}0^8`u02}Xa@q>+Cpl4%X9N*c! zqxcMWI6U)eklh0=aHLa#@t!bhLt#CeJK2zBrp8TV$G)}V=oD^W9hkU&Y%^$vztwk? z1t2(XoY4L?h|$6L;7KMPjKFMLJGz;5Wiy_XVHX&`eVy3VJH)PmVFNvzyV=Gb1pAXE zczXG8&&F^M=vk@|1i|xs^z3JY!3k%QJvctz!jDPH&sBxhS#y>z3$Bid&XaC^TL)3v1#xo}v z3E7p!;r@jZk8EVrV;`eKl6-@vCBDF|rTbZ(ZO0w%<5K+-6jibxcW=0NbbP!w9OxY% z9bf(2j9Bgz4O;h(k&3yLdl1xH6AR|mB$Blmib+k}l?m#coMbANnxF?K_Or+72 zCBu-JHK?Lzt@r_1T8zToOse>iXwRm_C33lgs8VDurX&}Z&Z3ZDkjdNa&lSiRzD!)d za9H{tAY<+)NgsveV$`Jba_r?hoC?>LorE8dXbL#P#2A$b0+{*4=yKzd?q{c(GS>MomV;G|CmdIc-hglC_V**jO9MN7TcNh|pP;*Bn z^vobm=>H^spuKL012h#Rd!cv`=m~6OgFO)#?G{zWh?`w6pMGxgz0q!_BAF237I%rs z!Riue&t%1P>i?F-uLp~N{*IB2&1h30fgS|vQ7t2S*U)fD;KXraGR5lZV_iA(`Vs;f$ij{08{j`4w>JxUd^GKN9 zy6{MW-nw}C@bgAO0=EtgKgm(9YqWFZNCel0pAQgH5YKY#EIY|=39v7$q(%6m67P3E zUm)aq@v2NbA%0);93|p8=_Z_`WX*G&bj~Q=cO)iORDk=s<~f;krBE(GhI14gTq&kY#bjQVN7#9Ms5(I zh|14nUfe{$^}UkXpBRAzj~o9_8Wfy-_dT24^X6Ol*m1SZz2j=e_i_*l2m>Rz|;3!ig2x7n`hat5!&_NCoiav!$) zu5&cGKIwAWvS0?PhE@VY?BVJ1;mI1fgCq*9nH4pv?;E zT0v1G@EL(r3$eJqGHysD^{J#G6xK(=2A^La@EcBi_}^MY5n2{cAgs_Ly3i8JY5NTq z7;ZQ4ZtB%JlrEiMwCl_=t1i`=Xsv9GwMJSC?oex>)z|87b+iht=2keE2x02Su z+z_2G_Y?5sgoR?ixN{KW~xd4Y;w$Qd_JI3_yK=$W^%<0ER;`oTq={SzwKSJmKo6Rt46yoxUk z_@WJWN-QgOju65tCr=Q9ER?1EiIY=5^d$0N_+j@jpS*lLDV}BD{(t=Q7$WOuOYQ$M zIY_=j4vF6$KdVo>$S(3F{9cRS+whCrAAX5=bk?o^-}*|QI;TC=WG=av+UOom4{h4Y zE$8m#p5%^kZ*rgT75qGYGyfQJ49&7}+3T|RQ?or`ZD!b)vs1mRCHF{Rq=gIwPudyD$O3vG0pF^T5U?(tzDtr zuDx6Pfc8n9N;jnYvhEN1PW@H-cML{D!Elq|py3&#%-C$a+W4x;YU(sSW_r(THCLOL zo3AoYm|wRzED6gcmS^z$sdc+`!up!9NVv(y*?hLFt<(0T?GJXFeaQZG`!n`;>_vym zag*b2$2*QcI9r_$IG=RB#SyKk5 z-Vom$zbyV*{7;pQm4(WSDzB=1smfNhv}#Y)v8o?ceU#89Vu^-CZ(=yHIdOGjU*fsM zn~A?BBgwhReW^LA<*5r(KT6l6H>W4k-%5W{?WitPFMuh-V(O*xa@s^`L?YOcPnoy%o?-1(2_4ep558g>~CF6<+2!wLG+V%2@tu+yM- zg{+pa%V2PZ!8;ss%}>)WiDfH^@fjJBQ(n#rHy&%Mt)VDAh>bdLD1PHc+-WF&%r-n% zjfc~C&Q1KJwqTd3s8_4gRal6}DHu&Ai=x78x1gY0N6y@oIiAa9v!?O^3RdQXuriz% zxB6T@j4h`WHz9~U{l0$5nC~L*fcGNrk~M}UW8RCs3%nP3hKld$7U=1{Yd5}%pN-|+ zS|$cO2~R%3_48)5t)8@!`31GQwSHeHWK;2Qiwj;}F1M+KhJ6;N)9Cbggy24#Go48( z(yD!!6zDjS>&RyCna#Z3*l{8wwziRd;uKo!qjrU|n#KhmFM!}mAzY7d?KJP>bM@6c zBYwUs%z(Snu@E2eiKIoQC|u1`qA)ESGKP=s8zo<%GFPSb%0$ARMQc9VC9i`4|}pFUfvtMf)-cPTERmv zDW9zD$$7oGo=Wk-bo%rkQ`|4$5b-2Ed6L(1KP6F8OWH_l!AslPvcX7%1VW)~Akb{f z==HKHVs4gMs%#PkI?#PEIs3@5VaGAH+L~ZoPs?(Zuf~8%;w6s09qekSb)}x$+o?G4 zKFD7~o@?t0t(jZ1Vs_F|)!+?QR(1@;D;KmyZSgwq$n0nCpD5lH@q6QLSE%M%U)tmj znS*tfo=C=7KhO|uuZida^QvpQQ&u6>oeAaB5yN>~j{h6ybH}0{XSJZO{)skODFp3W zj|)yOYilg%!O|NfaV5CNJGq<&xZfe)%+=P|Sx*}Bc8O6ZX3v{9 zduvBmSI5I+AMDui!B~FT54LRi!7|7bB@KA4sr($ZDwJS%#@2ZsrA?*Bw$7eM1Jlpt z$39@PNQsBm(c@^Bo-`HQyph-9WQAIxkXOiMGDD^Y;3jkhjzM3zsh(7+(1K@O2km7ozW6(6MDx$HwH^)D1UOLIY6JNc`Ly zu7hZZ^;u5IDT!JpC(u1v$TY+-3Xz4PL0LJ0e({F4-n`+hkAD2)Yw?F?t&=xl*6smt zkm?AMv>A}m9=7DZbW8t%tEoroQ-pknhUjeZ9z4SCfyfJy;&*Pifz@9-`B(B!@&Yjs z^^-c@B92AgswneNE^GT|OO+7y7>%B&P}MS=Xf(LO0`#-7kA_PK?in_sN~keFmn*I!=Ia=ZGj1 zU_B>T8HcuW+_0~6Q-A;FE}yS!bASJ)PT%7yTU9Wau&Y$|L@-!oQ_;2kw{B=_+i+`t z|1IlUTi4yv9~|t-WO@dJp@nR}5dAHn(ci`xD~Rh!;$TYl8BCE*H^-^88jXs2_7R6_pGoRR7OJTvin;oz z$a+vTU_zc*IVOe%8Wg&z)fK3zOw`LCRAlC?tgBryCut3*9S3Rfj!&I+fmo)c#?~;@ z;&(MH%E#L3(zac@P=5gQn~nOLNL9gFq0#8;oi0~O4?4zUvGBeWYXX?E;{?->O{^u; zdXWhLBV#@EVv^QNhfEbrw@smho#19?=B=u&Ij_G_zrUir#vJupoK zEerL$(w}4`Bu;#+#TZmT(mVwnQORZdbUFp+&??v{bcptg88F8E!Fq~u5%@5laIj4T2q(vC!7@~J>7lxRvGVU8bbRxl0^GP&^`x=5L=sOYQHdHk?lA5=^sd?SJR4RP<2VdtaJlg6d9kEAR zv)T66T(<4Q*3r&zxbytxjt%u>@+34@ux&|WO-FmSy0cxPQwJI|OOye_Gt{Br_c7*& zD*?t2(_np2pxJLzfAI)_19f-b%%0&%6WTt1%0~GL24k>(cP8U#K?kaKJ0$LBWNm^E zHq#jA;c`ryA(6-U8QrnWIOgW3`c~J~kM^ak;c8cPdu8|X+S-xss@58Jb;$a3HXN%? z$D=j6d@_~Kr&0~yZ(N*ryBiia2J%V2iZjh`shyW|IkWw>ErUj0OWf{n$A*?RAqEUEKCy{;W=9(F5}tX=|1?=qW8@9@Scm0m)+^o z2{yJI@lD=D)$|`g*$Bk=w3WB=It$56HDiV`@Nqm03{^E%TElK5^VnIvqvQP#(0!G8 z=BJkmKK-_6fG8OtQLdkBguV^JQ-=|iD#0 ze4S?hx#8_wZ@lg{w6B}&;g-_xgA0|!{H&}Xr*b*O9ab{4686jjj+JBU=FF!**f@4Iba^FZm_e=C_AU_G_9AT?kws)6}|1c8*hDk1M04v{3~Ywuf$-f z-36_-E>V}Tcs(9xh&f_R+h&X`s9~d})bm~%X3X9J!G+f`L%s4dZ7MX1!RW1W83nC; ze}&a+4VoPBP$I#7MUm`YT9aPVS-GpRs!U?+4V)pEO+>9at6(>}Vy>!Cz~nas?SWWR z#MH2;!ELXdS2Rl_P>z&1_)s763hr%0*DIJ;>6VAX^}S8Trp89Kc6hkH*X<#zdG9lUk0SbCPos+r&uu-56_eao1MizqDuy~Bx%jExfXr3X&=JJ$wj5gE{py+ zukbhaCo6L@nZ7k0s*hUTx!zPPtE9>c{c@GoQDA>Y%Q#x6Hiz5Hr40HZd*8b>)NKOMOSVxcx*a*v>IWRJ(3XRVe%w|;`h*x zN?0g&LB}0XGpEcfGpksbnd@NAH`5Wpcu*OpIW;0ft}0*rue{U(MT1M;I8zi9P-m$QMXL`G1K~=iH!RM==o660zUq05*(XeKG@se@q_T2Vh zAlN*RsTphvR0g(6`iIfy9neP$&%pSsb(g|y*;OJ-NexOAVKH42@jdef-|6|>U46yB zQ_{cdlHMm-n7U+P^^#d}x_0F)%c{6z;f1%ZZ5_y1i(^8~Aant%Bab>X#8uF#4#?$H zy$eISlZ788^4HcNb0Cy|`{*x=-@{=3g7$I8c7EZudv>9*QaNsvqh^)^k5i?>OiHbl zxWdAUFojXnNop>B_$YmY%kH|fc!E2|e7H)Wq+SyBNvu#2J`joMe7ksyk@=x)1nw0{ zpJi^S=%m1w{*S~*a1b*UuXs~5rP3@wVpt|Dobr2mHzHR4OHoS4)|dX_iNBC5%vwWmq^> z@lS^jQ|n<`SwuKj{Df-ZCQCFTe?eJ1sW^%WJWv>}1)Mlj-{VXw7#$~1PoAK!;T|*d zgj8y^QG1WvVq~|neouL2V&KNq3!bL0S#syiS$>6euQKXRxF0{6STR4lvM=WeHD~Bg zipcJi#^>iKR|`&JB0JRqh${_?ZO>UCvh>s9Ke$I8;ZMI^{2}e4y~P*MgZF5l_(D;P zix|HsQI4GGp60{f+*ms9_sg(MWJ*&?>oe zM}!qi#Ug2rN0FN1Z+O|m-=-dX(jOLUQCOC~j>4E7JHaQxccs`1#CG0mki`64S3^pk zl$Ciq$0wUN+_iY|og12)H{7{+@m(95pY_%Er_*yAybyA>_tAAjx36hvSabW(lG{e> z>PBx{Qa#ud3^ol`*DP!f2Adb64UDE^=#L&5I7pA;fX${>nS}$4PA5dKUup_u9O5Io zTTI}g{2=gS)eF|=mmH?D=5_ta!`4vJ-aiL1iIx7LeMz082!6BY7PmVCX2sW3R;V}B z7lHC+KC7K10T$K@2Z%L}q4eHUA$I)qaU)aWGqR9Uk}NOqQ`Sm~J?WQ+?pTvY-xn>q zV||m^7qjgCl^5DA+1KFn*7v27AV3ZD&cd5V>+4s2VX$g;bK1c3x6)133mX0Y#s$?i z15JK^(*T-PqQe~`zj(kgT0LjC%ZbcvJHX7#2`Tx=hL&giS~dk1)~FIO=&}2oAEEl( z>iN|RdY=EG5UjGCw@^q$?9^F=;S#co+UAT2#UIl@I{hZaA*DGa@~Hv3Ck20?p41c! zl{y`98C`C7*0lZ!5P0Z%V;yDfQ}oplp>j^4MZo%v@ z8&q0v&FuQNk!DYQ?InSv%Ip)Y9=k=O@nyPdT80{&sU&1v4k8vnKP53d!SziRdrr^xHy~Xw2#rQ=f!rJ zo|SRb3?Eml!^{Q~jslZ=`pDtpC$AioJ@g`kvjTusv<*KpD)NK3VBj&8C|6Lk+^pv- z6nGx~1{@Aig@_u27t3K~@%cyZ;O}_sd(Um zzKgOoqCVu!sKQ5 z_!G)Ff-+RZf0S43Vor2HjW%;;B`+)WFw=N~u?gr;b8ecW4<0W5^@n@+zDiTYH))FA zSNv3hXtG4m5|pJTLZN~xA&v@uCoAMc7JtyM^-=^H6ykJY@mbDa{2Cp-mwsjEZN-s$ zL96=7wN%Ib5@WA>iol}>w;mT0U&s+awF>n=8yVTpU$Or5(0bN?wEi{va4A0pOe~$w zPswSX()!|SpQCkkyfS|3Z{q!ZC_h*(pUUJ@<=5xHSukNEBR|`}o_l=#savJ{=`wC3 z`lBG0X90Uggu^-mE0M`E$C;5hXQ6Liz2n}uE_;(Md+RN_rFah;^T`$TlgX!HBUMN6 zm32X2jQh#Kkr9*vta&T>J@*=}&&ZL+bM^~%J-m@?z58yEkpw3{p{Ib@*JD*cv|zoC zQd(K*GdqO=4b}K;soiEH(@Pa_waAS7F!R6wv^o$l4wI9&;!$>>kIowbJZ{A2l+_5I0Y|N3TkeQlM2E;QQ&Bsg5v zVSsHf_JtxnW9@BYJ<$r2)4*sI2dxIDX~i*0nTg=pYcN_xT0ti;SqYvoZqU5UIfm#ljv{E5|+fWGF3c{ek34-}z-E}%nrz)6)IVx4yyxVCj@RSdOaA3U?@QRbiSHh`V z$rJ<&iV-QxJyaRQEY!L5qkC&>9pGY0H`C%?4z|q}Z!ldF-ZrmsZGSQ`XRIl|qBBaf zah8a-bW~W?$T{l0_Boi5>l*85ThkMZ_Oi(}QLobLc_q9J_+@%Qt2XX788D;{BB`64 zXfmYH^irljnDxj>*(%tvOVtjo-D6Z-eDT5a&a>7oZWpRd+zuoU+#yqTNAWg#`Hq_E z#T`*(BTDo!V%)H32)Tl)f=y+FUt6c$EBBb{_E?3mA($v}lf>>I$#KjWlsR5Z&^ENu znX<^n6cco34JkL6n75^~^Y+}L$g;$o9^Sqn+l29DGLc#8;=Z)s6i9hINgsGQ>G7of zrcB?3v%{6KO9TFS9X?;As&{2w-T8BpvC6_|SAcQ0zx)Xz)2LCyjjwTvG zub3gF7{s7SY>+|#y{2STFwt8TCXX#yeefV>np>CaNm{60hEfRugfIItk*2xP z;%g$U8t{3N`=gk1vKG{4K`z%>3_8Z%U>GO@imnhtgvCJu5t zEkTFAK57m+40=cK^a^^is29g#@;`wF`HNV7n0e1A-R-;2?g>d%Kzu;`b1RYRlyc=P z=J?B*HEK<%1+At^y+&>;}c&s4?U;?e&Y=3#OIDAyXp<&mKc3WnS4B3OP1EVZ)T5 zI7?yT{#{{o+XTN*aWE8(Cl7svQ@djsPeZ@pT+o>7Nm!^+hL1fN@z4{T zV_|jEaI0uj8W-C6FP&pkKBDnJv}{$M1ti224T!@5%a85z6p#xw6Zn9o#5Qk|ryDO*#^`lO#o4EZ;I6Epaa#8skO} zJ$_^f4!&GlsLwPv%{tBr-U#veU%F~8>P{qeS>O(7qb$-{vv`iJ5xw7_T( z!pA$Ps}x7=A%cS6Ekiw1aTEYbK>?Vh&>wz9AN}~B*MD;Dj=x_g@{s~DP6RPdlK2X` zfIr|@R{#W6WwS`Z4(#u7X$s07inUQ@>+6w$!BImC6)>ma6(>6|S&Rc4PQB4FS8H)- zwGNARZpWgG*JSkuwAz5zYVv0G4|ade?lmedQW!kK;qJjrG27QL^Pxu|rgbHQ57}ay z#5Uz8732l1WtQR{m^lPQcI31g>5ZTA@1KIpMoAX!9z?rS$Qi0hG8wGU1cFm-#w1J> zt|i)}5GT0dN?|4j(~mq3kUGblw+$DO(Glq2XSQ{b)Q=AoMz8&AVpAUqCS6)IRqv|u zehp0>>^>}b42p};WVB}qd7fKH$6?zlpOQO}zF_)9W6d`=OX!mO?jx}T-d7i!$Wgey?%_tkp5m4;4xS0>Qx3)PrBRpwdLYl*<(S}m5i ze`C7AVzydS?sZZ-=74WMhdh^`Br!v!whDqrXWC~p`&kg8YS`*?TCczd?SaaJRxbw~jwzX9a z&&|Y832fi4$q)E2cO_z}Jn1h)RC2jbZ?X8?4u>yDh!0=|pQgIn>{BQ$W`iXh3PpV4 zBpzfI5i?^u5J^N&FfdbjLrZ-Cs3`%gVUboW&Jb;RIED=jWy4$+n>}xr&wz@sxhU>? zR$F@I&}#qOs@R<98llQIny^|{E?E_rmq^SDtXNWf-jfNM&B2VvlL?tE!3_O@uRpzf z*2KFp_}_+{ zno}v$jB2-PIShiCCsu>eXohFY;}|X<>mMM6te-2T0YJn7SPvpyk?kQEcjALail>n95NVyCTnKr22=W39=R>DIe8(MMGmk)21%@7UcmDj_%vzjZcR;XZQbs)#3(tY zsH7nP`GKNhBcK-%n=^qVqPc5eW6=(bj?F$bGsKJ|=E^geK4X&ln z7rL}I6t2?FGQcT4?JQQ=U-dv z>%)e=%F1MKuOW<7Jj=wFaE+K!mL_1u;BYE8DN>jL5TK1`uzFmIUL}y!l)W#H2d{12 zFuq{m_6@DA8*U$1Fphn8+4j!POPA%cOD`4oZ|ZB^o&@u8(>_)eCN0-`u?Zwgm&Xt#4^rf7<|iHn$AVZeNx~S@)YE4QVl>VlmD01^vxS^Yu$w z0?`n7%}H$ZH2ocTQ^QbvQHZP3E*C%P{s$jiy=`0ZMYh{+d;k58M<4AFKOs{&vXeWX zejCpL*T##vu_$mrK@+CrmSHK!vpb5fGuiu}v28NE2SEdn!4HwIU^4(6BvG)ab-G%c zMy0CtD)oAAytc(VWj`cDX$dw(*^D-uZ?#K7nm8Y;v`0(BP!@)KIbU2F7Fw${nl!G~ zlyUeHAr-RKv=w(o+J`ck#ch#D+u}@Ss67(#)%R9b_SE@&bv>1pz4bm$t*cN~RP8_i zd|H923uoT4`MRkG7PW=LZHqE-+=u7lmA&k_UP%X3(wpfO+*{x+KeC~6uU%v#(ae;D zwH!Uq#I4XPT89)OSA6q2$sbA67wKjE zRm4jI1-*;ol$wePC3TYzm3ZF?#9{`h&Qi~ou+IcmNSHccEg$nb1iP2JNan3>4)~hW zZk^yZEPIH)xMIJ{XSRjh`fxsGH3qX`N6hch&6jdSarzY9fx5FKR4@WGX4lIXNH*c9WRGq&vKsW}h#g39C0L{Vr=nXY&|>5qHdG zHGAZnRKZl8+m}zc{H{QlYm3wcje2i&&|~vkT?TW7Y)EjK1eeXp0#CiK+8?UbX;{(( z9y5zEx6_km1^GQrP~w=pj!VJ@nTY8bfFT9{QPaG>Y=b7gpGRFws&tp`Q56A(L#`4L(^1x@nSIQo7vg!YV z&1lpe@OWf`@&K`zWe0e(!@5&I=a@&&q6hQ?K3pS4Lp~Kflo*cGU_x%^&IeWcZ5K8U<{XT_ zSeq669mY6$pInbtD~L4L!SIzw;b%rju3!ooF(SSSCnj+#eD@NIjtz)TuQ%N*&aO^n z%*A%IJg0nWqIJqqP{Qq;N&?e|1{sEA{rKR*JJT217B)xPYJy5CFaARwsO^ZhEOl*6 zj9UQRS-g0Btf8^yN`Iw2Ugx_t*bK~icw2UsNY^vwI8CXPpEm-5=T`0{79VS&&+iZI zbWgj(ViH!GU&%QQ6SoU7s9_vFOq15r-?L)Hy`4PySLvS6z^fF%jq!~vHX^MQcT@Mh0WE#p<` zSL9U$aI78?@YP0v7!i#qyhuzAGZu}OZR{pzvq5jQd8z{@&a=D`sW+tJ^2JY&ds|m< zJLj9*HY-axfvl-KJjEHC-3&V3KL~1e^%Zz5BVN;FjW5|{b4XgW-$-dEs;t60~ zxR-K;_WB{ru>OT=ihr@!4Yjg4X0|FCl1Exp- zA>;IRvz+c^Q^JsFwse3=Gi%2T(G~H7qIolgkBcI^By;5@hjXpoS?0qfK^@!j^RKtn zR;Y@9Ey?Q7OR|a?U&!hgnckWFc=8kOIsSg=o%=++^GEU>Q9k-6H*x=l_GwtYO`}oE zc1_Pyp8#;5<)uLsMi>MpwD$eO)Oh&)Jpj94D*bfpUeI}RCO(ek)(^m3^Jb-4%_xkV z9V0M|WsZhLfLt859I&{TB!t|LhnF)D(CxQV>z{#+D~%g>5!z&g$B3C4#{rp)I1d=j zcny=pQc{5okPR$|1fvfJr$-7&oaZlU7-J)~x&g2_Wv-*Geg${z_M7c>i&~o&*E?^G zrBm^TF*KrXldyOJ*cf1)D*U^vsz~5~V6L259by1W8G2D#KE?0< zV0m%|zylBucs!u;&II9w87^-e06Z6Pcr*}?0h7mUxTteM1BN$}n}IkId(Ilcz+`>& z#o~vDfgn5mI`~{WxrsNH=*5tGM#P&=vP*Ba>@30NvRN2FjAb5)eNprz^ML7jCU{D1 zyCdSza*hRiij{jf&8y5)+U4zAxqbFir(ZArbnjkTMIUUMy>(=kz1+4b?3|K~j%t_2 z%tcLsK@2%MCJo}-f=LOxSyw?HuCqcfQWvl>hH#H1aY5K%LP1QSSx&qC^s z88tpk^~H~l|B*h%{qgnUhcxulkGaG2&u`Js)2`yr81M|+i^c2cyy8=A6M0+@JJkX^ z)j;YBCQ<(@BWbh65@A*#$gDsx5b*7coM{Cl+^cA)&Oo+It(IVs7h6swv7iXfWYnb9 zYj3Yz@0izWPX?{_Se>V-&6N(A&EAN4q+w0+>gCO=x}x2+-dt3unYY@~n7!Da(mFym zS4dFpP+0?Zi^o#YzA2e^)y_^AhV;D3n{15a0qQN&9?&#NUOBZ!Y1T}sG6))DQ&0>` zBtR@0?t4^5jZZzw7f(J5Emck5INeIWTWl)ni5EnDaA!#${2xglKmf~lIh9#EUA9!P zI1~&w*H?zhy~}WODEvJME60#?tevnA+}miU4HB$3Oj!lf46AUqJwU09erIU^!%XK- zD|J&gUB#g46~ADuw9=BuKQXUYKL}<=(;l64#k#a zPr^Pz`>1zv+}TuS>I|idN)ex7(KZ}r7NLZCWQ@qlIvZzah2ix{4iQVsPrE>p5{rxa zWICd!kIux{G0Yu<*3E$0S!M$47~pO{kaaaPk#^<&FddJ~(6dZ0%S)2VJP5R2>;bp_ zaPcFwpHcYqajET02hG6eX%D!O7Xd_Bl5=LTSOFyk%s_gUQv0D0`y^nV1k(F61Llbk zJrR}HCL#3bJuJEv5qhjXX}u1^9|uXWVD$TZCgwi6K{8Vbp(na4Q2tO3)1;g79%m#y z=%N1sf1DWy**+V8+|a?=0h`U0@noPD-Bq&D=&fj)MN{1J>S2Z@g92u4m+UKOY-e*Z zMxD;YbPMx?OuQZ{jrkd9J%i3oIa_R5fCP@x^IuCem;`MMlh#hF2IBgihxfly{F|eu zWr0>Hzf-2vIl?vteTU6>6jhmlCge}qTHzD?2GGk%0zhEY7u>;}cDqv(*hz>}6W^)R zIW?}G6%`t%MFZ6)0xCsEZ2AL97%3EFYXBu29~K2;of4bR0@wz|Elw})yK1<$dg+e& zhv!|rG*dfrRo~&EzP=&xql+dNT-4dSrN4?V6!^;iEwekfEJz`X=2Q)}&sv~XFPPOn zqyl~Z4w{y6XDU%=F-?yj5VeFk3u9L0qcY^|C%~UggU&w1iZ0kh_$sH$x(}h>8NLcE zTy0GiE%_=Cd2apb-!Nxoe%V=pB@@RAxlY1eigElnHjKl z;|a-W5mU^V`7bS0U~ARG^!x6i%$3_7ZD`d8d@eOG&kB`(YzvkDqOF#fDP^=_3a=|u ziS31bQ*f}6DjAa<%wsaRj%3`JfingB4!?`vQ&@#1`cwgCNRjU0Z(-R1aK@+;gr1VU~l;$H{zD;##a_1-h1%NgZ^;}EB#m2G&YQlFJ64xSRQQz9VSo6vS_0VZxjd=Oo{^$BA?=>-( zryK)GN>xH20BJY1LiKE`#C^B_>Xpsv3lAUOs%>3y`J6?!oDcZWEsK&}4EfNNOm<~} z6YH|n3~lcz!Urgx?6_cFvT4n&0|U3MYu+Dj8BC`KT0)_gfpmJXg-vCnoo0M27Wv#m zu*Qb4)Q1Y@+jbpNZT@y?7&A^-(VcgpURV&Lu7vKFjkjnT61|9KAUmrbL;muXW z^BPXol8rWmP3zoVzvJR#4v)*ej*T^CZJ|$~En;jV@Vi+HPKeG3gWMJ9mRAqalh+pS zV0=+Ac_Vi{?lnSxlw%t^*`pUMyOrW&q6$1Cwn3&Dsjp}~yh(+!BBQgJ)bu+&Hy8KZ zOz*haWxdXrY?+t6u6XCEYpyx@BwB|VXsa>Z>>bV0#+F6OoD*O z1l5Uf0uTvv|Kb{a1^^IYTlIJg$diw~w39!??|kX8XI|cqgZ(d4A9H?g^P2PIq*EYER_!jz97uA?jGl$m2VLq)KNLmZ=j^j{NS zWbH_YYO(TDNTy>oI;&F~XcZQOH_b`UYw*>@(yf4;SW;EX?IDK;%M4}ftzNB3EwkAd z$EwWXTrl0H<~6OkcuO2SE{ze~6NY^-Kg+3=8bQWJ33-Zgh{3#SCV_>oa!-8y0sgTk z5p};qLq$Dx7g;C)niepC9o!F*H~a(MD$H^bPfV449?I4NSE15QmCOu)z}65-BrP1` zANuaMU;Ex;Jh)r{uePt%S=og5^Ig;sF@jvHimFJSDln-!}`G1T<}UkUJ_!cL@}fg<@irxyBS3# zSw0I3XY*L?dx5|*aKXI_)_kOjut{uC4y%zKgo;IR^sLyZFol4d!o*RVh|ZhF!_ku^ z1RR5YGe{iW44k*58cVou#!qKpzVZH~K%l}4qB8i#GnlitNW`7WV4s=1MhbXFI=BR2 zrtN1SyjPur05{IWcwclD{+qSQijUuv=nL}F-3$mNQKuAxJi43yaQ7j){aR5|;`TG~ zMtUuOpG>Az-bXw}EmK{R*TXn=TDz3NMu<_aJqKy@E(0KabSBj3^tT!O2=)Z{LEZ%% zsuX*K3s$EmxYuqA>?OANUY*eenrRp~wUhwHsLKEMNMk|Imid)c^S1OH>e(`{s&YOK zIy>6B4|TV7bed?aXLW1q>h37_$tPU2n;rMW=(|8y_BJ)l<+!;GO}!kGuQPC4Du|Y- zxmiZ3jM-w5#<3D)>zjaYl|ftg-!I80Y|gf_e6l!W&lC{t#996}Qlck{iw_+<)N?aL z{Z6_KW<@Nc4uYyf8D=rY*sal!-Fl;5uP_S+5tlY2#t>I@wgGR>)G@l7ekJ!Kz~n^x$-~Y;jzI_Kkjc0KCv_Ty5({Gu03vv{_(Tk*Kf?hZzl^%qCHfnK!N&eX@vnaj zSH>f~;PZ#f@j^OPsIp$@3-~=(q8pN5gVjnWOYudtAQWtach~8+Q?*NVi#1Po_%rMMtUAxG zkM)(UUHTCG&LP^*B5q~#SW3G_tFzn4Qnh znUik>5(4k9z=u)fZ3pvoiO%6bC?S|t;Dh5+%PGsR!-8|byf+p}h_45`r#c^OoLfb0 zhuW5WgU%la*HlNJHO+2KbyQi3dN70bT7MwynerKb1OH|tc~JCk9uvojU_PF$m#7B9 z#KwDM7$1)y_&s)q!xb>@jTl^_X3VmYfwv;5^{LrL;ATbih`5}hjMD`$tYi_{^c-Db zPMCB^qa?hgB^I5X2XDT)yv#z#4t^w)U(d!O^br-&1Ei(cs5(yso2JJjl;Uh2d2c>l(63L=a-5+T3%nm zOS28U0NY3m;!-mVgXnZ1e_4ijNle*!@Zh59?05A7X%nV^(?5s5>X0uDi%&|rqe+)Ym;Y`6~t4uMvXSE_Uy}$&^O`6VV&_`3N zONx{&HsOJJD=mHz8z;7d`OEy}kcSA6@6`oYAvKudQYMqyFp#EcZ^?i5&+wn2u^4Yc zN1x?K&(NEaLp?R=c9v)Ttr>ds^Uj*t^pvQ@MOXW3$<=;gN~e}Q?n@-K4t>IMMrbda zw+&<7Rxw~*W#z2kGI;G}Px&JVP)U0-i01I#U+krn0h-O5=t8bLWXNX#^r$Ujes4p*6zD zmjx4{2U5w*l@rAiU7w1Y=sG5)ebccMlhtzc1YQ*}>7|$qi$=k%rI-Xh&s0pp=7DGU zJmNHPj!grzU(oi?M?ur0WsgUH4QR&Z3KaYt!~>>hm1d5UOEZ>OfB+f+ODs+@(A>B< zZFy7igVR4`eJ$kzxOLKV0-los3F$c&W3x!&&sVXGz_yQB7=dSe=Z*15gUjH%I{2A9d&yP25#PtMTE65|)#IWU*Wl|XrVO{PwcAj@Fd*dLhqY4LX z-u~=+zIN@Lo#lHfCvU(4EH;^P#{2dj#x2*fQa}>1t~bPT*t<#P6q#GLh`aP|QugvB zw%vaB$hF1ae^IKQy`NNkmbKx>|Le1tUpM!Q#ka4W^F`LIsb^W+ek8U{Nwi1h9G017 zj%8WKs$Az~<>%1W(qrg%4A`-H)XDGw6?pmJZn+$9Aw-Ka;Q13dh7)JObZPkr-FDZ@ zFW>bodTo)?$!os%y=%nrNuZ1@%CL|S-V3I~sz7@apA<}DWR0&^IR$Ft~e^bDk%{D1hy zA8M(xMm;9!&4|ULXMSr=;2PeV7&+^$iHR9#xtYh1h|uJ(WiN>DLG=Rz+d&qSRb&n6 zECg1Mj>^}JjSbihhPHuxCet?1vvOtIfLCs5i&~ZqAgK$KBtVLc#xp)jAUM`17FJKc zZFeRd^nbcm#yfr%jFi1LRrZ3Xwlf-?RqOH8&Wc7mYdxD2fk1+7#b0_$r%JbYHz)n< zl0R8|^(?pZ4$jp8gGg=(%@ghq!$-SbX=c zKvTluNHhh4(mv=71iW6K4-*Fi^azsn4;PoyZA)&l#T!GRd|VLX`B11aZd(>}Ib$)W ziv=N=XQ!3?Q_wkjcn!gV!DJ#bjf&GM)V#rBWhy?~fdRr@n7F!h8(zfEaY1^t*;Gq)k7W9e~EM`xT z2;tx!-ey!f_G)c--HDiV5rc9_Bcx88OT z!sd_LT~!s_!w<*9$@If!tHD4O%2ZVEb6^tiH23S`C#^MwG+<{>*sbj4ceA)tbb!A@ zv_%x&yJ*t#xfkILaHnACK_mQcK=`Tn&yH+;gZ3!aH(+_gF;S(@T+r};v$}y_G_|H- z#_9&h75*n+D{ThGUWdFf0dQM}D3#GcTzgam-^lM6qomE)(sAh>g6N_f#OSR{7r@Hz~fp9or z>ca{_Chwj2{2hGC(z$@rVA3FaPH`O3d=tO6P%ll>c|> zm?tR(dxI?dkrAunk`gSy4Jxk}s;-O2;aqQ+-f6R+F&0qjX#$n``<< z=@=H+w!L($B~jbYO2<0>hrTbDj`er}{MXSZDpSLteG4s-j5bOV;QMbTw6Mp6Nlo((y@X>6u&JUE2&TEEgh?fOLe?-tR}sxayb>Md(~@7=QX6- z)?GT*lDRgvk7MdWIdj~NNn3S9IeOEu+;=ga4WOP&Q2IqUyB>F6f?rk+ z>*pq1+lad^6X(I95**@1J|=D? z^G5V6P3E8}3VHs@BXYz1#Cc^qSS%k_Te7D7TbQlhr7wAr?v(Vl0Q_nxIRkGA*RrpA zT0Wfo3+N+j$O?CYqlK#;AP2`$%#-vW9pwxLVXV}K^#o~G5?Rl(vJcnA<>dEn%XIf- zqmIseursKyt!)TqQ`FT5%cFEA-_soiw|BKHofT}^vZW?ftg30vbkzh4QmMdn%mvvn z7iKqyZ8gE#&L|hOXVUo~mub&$NoK>KE7=o(DrgC5*PYF@6U%8|Cw1I!JbdYI8CbWZElP zg|W?HI?4txk`%hOuq&C}bU{0q>4;|`oSg>z>47S}6hE2%gku1I_3YXARASTIZwTpMavJ;xI zt#HEz6Pm!+g1YvJL(Bksko^)V#@ zC0)rPsxNtyLL$j@q4mq{QbV7GhGS_zZS-huQQk^kS1Gd%23?}iU6eP4#sU-KS?=U7 zvdArlGsz|y#iy*Juu*qy7%A&1$q;q5DCt99B}rAI_G%$Nvn@2DoyL?#h~hR%>DBs} zxR=HQ?>g$Qi+kF!D_TwtjRd7)S*&P7N%@1qQ>8RvamKLTtpHWjG#6;($S)+Dyi+ZN zr4&?4CdrUSQ{qLfMA?mDJI+Kyp>+$Ou`3TKd5i8yZ^LaNuBF#39wE6kGBuJ)Jw;n{ zlQ4^h?kIh-LpL(Ds1Rbt{*hJ+0oXAIc|Ew6$}yJjJdBIeN>FmEvlwbfa&1w7qXfT={(Nq*KBr zOUsMYUn9r%bTZ0sY<$|yea#itLoHp++FJKiXXkjewVo|svUD^bx$O??2bKCPX~@nP zqG`vz))yM=sN0{B}sa3PfwCOQ2%PZu08WoDN0d3r^dS=aAriL_}MNnWwt<|y1^vcWjjn-!1> z<&sgG%Jwj&ge}Olq@s?bvaTSXafwngAZ>-drBclFlom#&AxtgKws2bs*OEfMkWJ1C zT9kBVf@CU}A<^0t28CQG_KS%qoQCOQI+)A%q(~#9R_JXGQ&<1Ncv(`b%m0CX1hp-Go_9tS+I8+hB{GG4zfv`1Uu~nnJpn2%XAs8BAOU= zk|F0~@vSU1!9rvyq*D=6IPsj3awaeDhHk1S=mh*_%(j!L2CY%HH5IB+ zTacq0$u@42%%Pt)(k`PPl$|al(J&c27Dgp0a`l!d73F(O$Bt-o=v*xcui36lI?5Di zF05fX7qz6w*1Nm0yYR3hkquf%&VxK@q;1w2Cfnp0*3;PeNCX#)$Bh-W4bm26;U7H1 zHynni>1-@0sYfdHE$Ui~wPr*17AmSuMk?mibX5xDr3+obZJAUGTKF-eKqI6zRbYe# zX+}Y=E6Kh~YqA?f^BKd%V=tdAu)Tl{qE=%C)EbaMcc;QU3Z*Z)=rA7BnnBrV?YL|S zwe^DzihLS|OiG6JbMS*6%Z6urvMWmU%tFLuTaVEy`vKrI;u%s~wI#6;?%4*)X1dUN zW)q&13AAgGq^ZVLn{lliQvq{{c8&8)SMxZpnnSRs|Cina_B!+V?s;?O4BwCXZSl-8 zlc@rx%7sJ$%%y0|(XK>M-C3Hg@K&;yxs0HH8B5&)bG?$XODS?!&n#okl1~SqH~jsO?^S&jl6|pOR4v zw8AAVj&9i$ZwfUPNZVa)n{L`bSzIU_?J#|{3vO{Kjj$nzf&`m@JGxqmrD5HXE+t*% zDTo3j+gg^tFs-~&?a2xXK%D)oOeM@Nq?E186D=2c&lvE-ly9!fjTf+s%mA-v(>s!rP45DUJANf{k+&O&1$6slr+-~@kUs@j- zPTBgbcrPVOvXBe?6o}fhJhf2xEgRYs7$=wYcv)Y(U4F)%gQu2j?_!*4+rkFL%W1pK zNU+V&l+Vi=O4KzMS%0$W=n87{w6^zhM)R#^J&_$W?p){|FavA5!zTPz1HTLBtBfDZ zGh}<4c($?iA&Pb!$ivJ1%O(=CYL+(AsdQ&$w)gBtlWK)YlG%jVB@G(&x~FwQn>NQ3jrRscy_LruW%{v~){>J4ny-^lNQ)bZR-J+%ooI1`*{B&ca(d0x48 z*^*_qEm*s3<%(eCl3@9=+ZL}_vp86A`|8Du?^?WK?Zj~t$8E?Il$8(`wde%LglEJ* zl}A`12T_E(HUuQ(GLcQip0GK(%5)Iv(@y7BrSmV2w+O3ADZt&*6J*jR1arM-=EzS7 zesdv_s2IA)37U<^lBq({gtfU`n7<$jtqInpQ#v5)aRQNW6c4u)as|2Wu$2S1R!1&K zc4Wg)*-FqsU`XWIVfrT9Y-U8R1BQOYZ6iCB0z3BurZ!D5s(mN(^#CWN}tGu^EBezkwn&}k8On@mFb9D zqcBB!5p-w7Hrm`A79#lFrf#`HN=%U5kr3vvKDk88<#}>GZ;q77VT`iGLDHOHCiC;m z00>k3)Z=Fp%{SEEIJd5`K1ehrW+&_YiJm_*|9F27hl>~a_b=4( zu2Y_X8OgVvt9X!U48PRPyEOO?>_nbfo6L8nrtn>{X?%z3<2=Fk36F2C@O_$3@onvE zdEWWc%u%0VGnOaK`1bbo%)gRJ1k1@Nl#Q(NK2L_RhRk9e$D8ZDdx*I1_3qJ#J}BN>M!%}@bC1O`*-;({FVMH|MUK8e~rJ^U+3TLulMiq zH}GuVeg6Ia7yK{!f98M5Px>u>tKa5_-Yec7Z?Ct{>-B!>z3LtDzV03NzU95>{hfEn z`-R``cX;3SJN<~)(QNWle%I(iI;yR$U1ZAzwZ*ddd11Z%ys)8I-aIzh9z}Jvwe=0| z>gKxmsy@Cd-A%-Gao?Ng#&wPHRa1P`?5^h4##b?oxsFC%BF0x2W2uX=)WulpV=VPC zmU@S!wm#;sKE_uc^H(2Zu8%R-$Cw*p%ndQ-h8S~0tb>M_Mng=aA*Rs~(`blkG{iI# zF^xn_BN5X`#558yjgnpxF^xn_BN5X`#5Cr{T+NMp%#C}@jeE?Edo;#eHO9O&#=JDf zyfns`8)M9kG3Le?b7PFTDaPCs%i9#wXo_hx#Wb2?8ci{crkDmV2QGSSQ%s{drqLYJ zXpU(#$26K_8qG0{=9orvOrx=c#nE7d6zSI0y1R9?js}lz#&;c!x>`r0uGZ1u(eAjN zlcTOSronH_S=s7JJm2iIs%=V`BE_Fm;b&YN35`LU(^X5&|Qm+h&K^^ce_;;SQ`8*z5Txsg*x-Y{~<$e)h7 zdQ|tQ?Nwv=9rgOETdVGon6zZllamflIzH*6$-(4#lT(wQn*2fa)z!7tcTYTA-Cg|`)z3{1CLOLm#Lpso zQ?8zp82z;=EmL0L|C>{1O(tXz2c}J()->(jXrF;;0lzUH8Zl`>aw=nL5(n0P|ifJ|u{QwMy!0;x$ z_rt@9%hH}k&+~le=e&0lP!A-4#-U!XiLx2|l6pR$Hw=B`-3#0gbPc_NB*&4Y7fFsI zNw0TyNZ%c}p4Rn10%)Y=e9A>&T1vSZ*ueF@!2Q5jZ!&$S05gzhHaHrG{u!Fmaz8!$ zyw7nxf#AE5u6^zVk!aVQ=4mVlFQ^k&kFO2(^AF z^+Tl}KKsGdZ*cJo3G|j1Tu;qSKs}HE8i!s+xALer;ovx!_JBqFo(GFGBuV?w@xx`_ z@)A0J2^}A-(3O+1@*c_!l=lMn17865MsOD3 zzn}7bWF3J92B0eMIL|yQKc9>aWz{K^*V1-6Fazz$Yi|PNxr>l}DPVob`Hx2YN$?MN z?=Zqo0jGiY0QuDWfGj=;g+ow~cOEM9PKUV<%za4Gha>|?Vf}6+WdRKD5dWV7P6O`& z4)ZvJS$sR4Nnh6cJ^}}25&qAmhEVIyLUUV z6tLc~-CN7`I<$H>n!aaf7uwcbA`K6~S054&us+GZ>$$SLIKR+5VMn-TK+SL?@QGhD zTMXdE8fTM@h6cIPXuEIdL(O2=Xr#WEg5OLe%ejt>=2otkA?td2eTnOLs5=Fm2HpeC zk^%Y5x)aE!q#t3kW9clMe+1`t){rmACWD3+KL-I%)kFVD&2jqtD;hYD1~f~Z_wE2R z7S{sn>G2MHodQk+?*XF>H^XZslby?S)d%K2q>(2qLAs^DYG4fZ(F~yZN3;BS#>)V+ zdoQzlFEhK|8mkt=4jSg?BNai~0um+ip9JFv`d?;+q3 zW~j$#^#pjH0-h#&e4chcqWwlhLqD@xakbiH6Y2c~mKcNN(Bk?^3mgEl|n-V4A^IN3$r zi(J1%`D4nT0K2Jwner825A{E#d==Qs^*+i2lm{t)2E4}g&nXW9hk-YMUh3Y3r#|3} z_%qhqh9w_BK3P#KzLoVn9G*AY_%8B1i`Dm0?#C}*qkNlE>!;Sv31B(rJpeoiJOVrh zJPm9Io&~;7kLM`&18)N(Ol#M&=8_(<73*(9KM$ar2f4nKhdcw<{}*0z6t8#-xRkey zVdrByTAXh5=XCcjpINcIGr%;(=0y}%LRC{XT&@PhreG21*y`2^)tfE^oyc%A&_JYFZ?InQ2EUq}QA2IjrqWm%CPk`NAzf8G@@>R;cl>31L zz(GF$95@900(c$xC2*Kg`37Y#ZH@wO0Vnu;5_p$Ys1G;;ybq7{Wtz0}f|dQ8_f`Dt z0a{20*3bXF#3KP{XPT8b0!r%wq%7} zwA#a}zn5}92Ar=~#bZn3s(eh1itm)yR++iXtwiNB_@#V(&}1aTeSIvx zJIKs?7GH4QcMjX1!{+DE!l0SKreVi}$aai5{ve(-XnO6U+)b-Jz+Uwx$~ZuI1UL%3 zMg0l-oHQfgAf9m$&p2qlwQT2+oy-|8(rY&|AEbOuy@ts@!f>~bI|o^n6~~+-OB;tD zpM~;SD4&Pwd8nR+>RBkBgW@@2Z?!D7I(yMpRTs!FDw3NYoZ+wHkl`9q3#XlTx!(tz z0mgYd7-u`_`4X_3&-?hiACPQ^k@YRgxA`o69)rJr@EV!k%2f~upa2=qeVaGt}r>d}#+!b|D1T$fXRD;_mAF2?5=#`ER( z4X%NXGP51z2m8qjp5)wMD(41MIXRfhUdB|S<9;I8lSIchE4qVnHLw=YT>B9Hw1O!n zd!DvCS-ExtZvkh(%laNuP^{exwqD}lUa<63##x_<#fNjodrpwu?)g;ix0LVmxahx+xL&(S7e@cKh4U1f z0k}NDjWT(e^D1Q_E-&#pEgxa#!D`~E8D(cg_I&8lXG1y>vZp~SI00J6*^fK<`J(y$ z>sFq8dBFP`>+08u3?Aa_=UbffJi__Uw|TeB6W(8PBJ?zGmHDo>-TN!=dqf6*?fuaE z8$A65X0Bb##y{aib|M0mVnx78()hn;x$&RG=t0Uig^nU;a3rYn5 literal 0 HcmV?d00001 diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx new file mode 100644 index 000000000..c29c66936 --- /dev/null +++ b/frontend/src/App.jsx @@ -0,0 +1,19 @@ +import React, { lazy, Suspense } from "react"; +import { Routes, Route } from "react-router-dom"; +import { ContextWrapper } from "./AuthContext"; + +const Main = lazy(() => import("./pages/Main")); +const WorkspaceChat = lazy(() => import("./pages/WorkspaceChat")); + +export default function App() { + return ( + }> + + + } /> + } /> + + + + ); +} diff --git a/frontend/src/AuthContext.jsx b/frontend/src/AuthContext.jsx new file mode 100644 index 000000000..4b7f820ac --- /dev/null +++ b/frontend/src/AuthContext.jsx @@ -0,0 +1,30 @@ +import React, { useState, createContext } from "react"; + +export const AuthContext = createContext(null); +export function ContextWrapper(props) { + const localUser = localStorage.getItem("anythingllm_user"); + const localAuthToken = localStorage.getItem("anythingllm_authToken"); + const [store, setStore] = useState({ + user: localUser ? JSON.parse(localUser) : null, + authToken: localAuthToken ? localAuthToken : null, + }); + + const [actions] = useState({ + updateUser: (user, authToken = "") => { + localStorage.setItem("anythingllm_user", JSON.stringify(user)); + localStorage.setItem("anythingllm_authToken", authToken); + setStore({ user, authToken }); + }, + unsetUser: () => { + localStorage.removeItem("anythingllm_user"); + localStorage.removeItem("anythingllm_authToken"); + setStore({ user: null, authToken: null }); + }, + }); + + return ( + + {props.children} + + ); +} diff --git a/frontend/src/components/DefaultChat/index.jsx b/frontend/src/components/DefaultChat/index.jsx new file mode 100644 index 000000000..4342299d1 --- /dev/null +++ b/frontend/src/components/DefaultChat/index.jsx @@ -0,0 +1,254 @@ +import React, { useEffect, useState } from "react"; +import { GitHub, GitMerge, Mail, Plus } from "react-feather"; +import NewWorkspaceModal, { + useNewWorkspaceModal, +} from "../Modals/NewWorkspace"; + +export default function DefaultChatContainer() { + const [mockMsgs, setMockMessages] = useState([]); + const { + showing: showingNewWsModal, + showModal: showNewWsModal, + hideModal: hideNewWsModal, + } = useNewWorkspaceModal(); + const popMsg = !window.localStorage.getItem("anythingllm_intro"); + + const MESSAGES = [ + +
+
+

+ Welcome to AnythingLLM, AnythingLLM is an open-source AI tool by + Mintplex Labs that turns anything into a trained chatbot you + can query and chat with. AnythingLLM is a BYOK (bring-your-own-keys) + software so there is no subscription, fee, or charges for this + software outside of the services you want to use with it. +

+
+
+ , + + +
+
+

+ AnythingLLM is the easiest way to put powerful AI products like + OpenAi, GPT-4, LangChain, PineconeDB, ChromaDB, and other services + together in a neat package with no fuss to increase your + productivity by 100x. +

+
+
+ , + + +
+
+

+ AnythingLLM can run totally locally on your machine with little + overhead you wont even notice it's there! No GPU needed. Cloud and + on-premises installtion is available as well. +
+ The AI tooling ecosytem gets more powerful everyday. AnythingLLM + makes it easy to use. +

+ + +

+ Create an issue on Github +

+ +
+
+ , + + +
+
+

+ How do I get started?! +

+
+
+ , + + +
+
+

+ It's simple. All collections are organized into buckets we call{" "} + "Workspaces". Workspaces are buckets of files, documents, + images, PDFs, and other files which will be transformed into + something LLM's can understand and use in conversation. +
+
+ You can add and remove files at anytime. +

+ +
+
+ , + + +
+
+

+ Is this like an AI dropbox or something? What about chatting? It is + a chatbot isnt it? +

+
+
+ , + + +
+
+

+ AnythingLLM is more than a smarter Dropbox. +
+
+ AnythingLLM offers two ways of talking with your data: +
+
+ Query: Your chats will return data or inferences found with + the documents in your workspace it has access to. Adding more + documents to the Workspace make it smarter! +
+
+ Conversational: Your documents + your on-going chat history + both contribute to the LLM knowledge at the same time. Great for + appending real-time text-based info or corrections and + misunderstandings the LLM might have. +
+
+ You can toggle between either mode in the middle of chatting! +

+
+
+ , + + +
+
+

+ Wow, this sounds amazing, let me try it out already! +

+
+
+ , + + + + , + ]; + + useEffect(() => { + function processMsgs() { + if (!!window.localStorage.getItem("anythingllm_intro")) { + setMockMessages([...MESSAGES]); + return false; + } else { + setMockMessages([MESSAGES[0]]); + } + + var timer = 500; + var messages = []; + + MESSAGES.map((child) => { + setTimeout(() => { + setMockMessages([...messages, child]); + messages.push(child); + }, timer); + timer += 2_500; + }); + window.localStorage.setItem("anythingllm_intro", 1); + } + + processMsgs(); + }, []); + + return ( +
+ {mockMsgs.map((content, i) => { + return {content}; + })} + {showingNewWsModal && } +
+ ); +} diff --git a/frontend/src/components/Modals/Keys.jsx b/frontend/src/components/Modals/Keys.jsx new file mode 100644 index 000000000..24cb2efb5 --- /dev/null +++ b/frontend/src/components/Modals/Keys.jsx @@ -0,0 +1,163 @@ +import React, { useState, useEffect } from "react"; +import { X } from "react-feather"; +import System from "../../models/system"; + +const noop = () => false; +export default function KeysModal({ hideModal = noop }) { + const [loading, setLoading] = useState(true); + const [settings, setSettings] = useState({}); + + useEffect(() => { + async function fetchKeys() { + const settings = await System.keys(); + setSettings(settings); + setLoading(false); + } + fetchKeys(); + }, []); + + const allSettingsValid = + !!settings && Object.values(settings).every((val) => !!val); + return ( +
+
+
+
+
+

+ Your System Settings +

+ +
+
+ {loading ? ( +
+

+ loading system settings +

+
+ ) : ( +
+ {allSettingsValid ? ( +
+

All system settings are defined. You are good to go!

+
+ ) : ( +
+

+ ENV setttings are missing - this software will not + function fully. +
+ After updating restart the server. +

+
+ )} + + +
+ + + +
+ )} +
+
+ +
+
+
+
+ ); +} + +function ShowKey({ name, value, valid }) { + if (!valid) { + return ( +
+ + +

+ Need setup in .env file. +

+
+ ); + } + + return ( +
+ + +
+ ); +} + +export function useKeysModal() { + const [showing, setShowing] = useState(false); + const showModal = () => { + setShowing(true); + }; + const hideModal = () => { + setShowing(false); + }; + + return { showing, showModal, hideModal }; +} diff --git a/frontend/src/components/Modals/ManageWorkspace.jsx b/frontend/src/components/Modals/ManageWorkspace.jsx new file mode 100644 index 000000000..4cc52d1f0 --- /dev/null +++ b/frontend/src/components/Modals/ManageWorkspace.jsx @@ -0,0 +1,476 @@ +import React, { useState, useEffect } from "react"; +import { + FileMinus, + FilePlus, + Folder, + FolderMinus, + FolderPlus, + X, + Zap, +} from "react-feather"; +import System from "../../models/system"; +import Workspace from "../../models/workspace"; +import { nFormatter } from "../../utils/numbers"; +import { dollarFormat } from "../../utils/numbers"; +import paths from "../../utils/paths"; +import { useParams } from "react-router-dom"; + +const noop = () => false; +export default function ManageWorkspace({ hideModal = noop, workspace }) { + const { slug } = useParams(); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [showConfirmation, setShowConfirmation] = useState(false); + const [directories, setDirectories] = useState(null); + const [originalDocuments, setOriginalDocuments] = useState([]); + const [selectedFiles, setSelectFiles] = useState([]); + + useEffect(() => { + async function fetchKeys() { + const _workspace = await Workspace.bySlug(workspace.slug); + const localFiles = await System.localFiles(); + const originalDocs = _workspace.documents.map((doc) => doc.docpath) || []; + setDirectories(localFiles); + setOriginalDocuments([...originalDocs]); + setSelectFiles([...originalDocs]); + setLoading(false); + } + fetchKeys(); + }, []); + + const deleteWorkspace = async () => { + if ( + !window.confirm( + `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remiain untouched. This action is irreversible.` + ) + ) + return false; + await Workspace.delete(workspace.slug); + workspace.slug === slug + ? (window.location = paths.home()) + : window.location.reload(); + }; + + const docChanges = () => { + const changes = { + adds: [], + deletes: [], + }; + + selectedFiles.map((doc) => { + const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc); + if (!inOriginal) { + changes.adds.push(doc); + } + }); + + originalDocuments.map((doc) => { + const selected = !!selectedFiles.find((oDoc) => oDoc === doc); + if (!selected) { + changes.deletes.push(doc); + } + }); + + return changes; + }; + + const confirmChanges = (e) => { + e.preventDefault(); + const changes = docChanges(); + changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e); + }; + + const updateWorkspace = async (e) => { + e.preventDefault(); + setSaving(true); + setShowConfirmation(false); + const changes = docChanges(); + await Workspace.modifyEmbeddings(workspace.slug, changes); + setSaving(false); + window.location.reload(); + }; + + const isSelected = (filepath) => { + const isFolder = !filepath.includes("/"); + return isFolder + ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0])) + : selectedFiles.some((doc) => doc.includes(filepath)); + }; + + const toggleSelection = (filepath) => { + const isFolder = !filepath.includes("/"); + const parent = isFolder ? filepath : filepath.split("/")[0]; + + if (isSelected(filepath)) { + const updatedDocs = isFolder + ? selectedFiles.filter((doc) => !doc.includes(parent)) + : selectedFiles.filter((doc) => !doc.includes(filepath)); + setSelectFiles([...new Set(updatedDocs)]); + } else { + var newDocs = []; + if (isFolder) { + const folderItems = directories.items.find( + (item) => item.name === parent + ).items; + newDocs = folderItems.map((item) => parent + "/" + item.name); + } else { + newDocs = [filepath]; + } + + const combined = [...selectedFiles, ...newDocs]; + setSelectFiles([...new Set(combined)]); + } + }; + + if (loading) { + return ( +
+
+
+
+
+

+ {workspace.name} Settings +

+ +
+
+
+

+ loading workspace files +

+
+
+
+
+
+
+ ); + } + + return ( + <> + {showConfirmation && ( + setShowConfirmation(false)} + additions={docChanges().adds} + updateWorkspace={updateWorkspace} + /> + )} +
+
+
+
+
+

+ "{workspace.name}" workspace settings +

+ +
+
+
+
+

+ Select folders to add or remove from workspace. +

+

+ {selectedFiles.length} documents in workspace selected. +

+
+
+ {!!directories && ( + + )} +
+
+
+ +
+ +
+ +
+
+
+
+
+ + ); +} + +function Directory({ + files, + parent = null, + nested = 0, + toggleSelection, + isSelected, +}) { + const [isExpanded, toggleExpanded] = useState(false); + const [showDetails, toggleDetails] = useState(false); + const [showZap, setShowZap] = useState(false); + + if (files.type === "folder") { + return ( +
+
+ {files.items.some((files) => files.type === "folder") ? ( + + ) : ( + + )} + +
toggleExpanded(!isExpanded)} + > +

{files.name}

+ {files.items.some((files) => files.type === "folder") ? ( +

{files.items.length} folders

+ ) : ( +

+ {files.items.length} documents |{" "} + {nFormatter( + files.items.reduce((a, b) => a + b.token_count_estimate, 0) + )}{" "} + tokens +

+ )} +
+
+ {isExpanded && + files.items.map((item) => ( + + ))} +
+ ); + } + + const { name, type: _type, ...meta } = files; + return ( +
+
+ {meta?.cached && ( + + )} + {showZap && ( + +
+
+

+ What does{" "} + {" "} + mean? +

+

+ This symbol indicates that you have embed this document before + and will not have to pay to re-embed this document. +

+
+ +
+
+
+ + )} + +
+ +
toggleDetails(!showDetails)} + > +

{name}

+
+
+
+
+ {showDetails && ( +
+ {Object.entries(meta).map(([key, value]) => { + if (key === "cached") return null; + return ( +

+ {key}: {value} +

+ ); + })} +
+ )} +
+ ); +} + +function ConfirmationModal({ + directories, + hideConfirm, + additions, + updateWorkspace, +}) { + function estimateCosts() { + const cachedTokens = additions.map((filepath) => { + const [parent, filename] = filepath.split("/"); + const details = directories.items + .find((folder) => folder.name === parent) + .items.find((file) => file.name === filename); + + const { token_count_estimate = 0, cached = false } = details; + return cached ? token_count_estimate : 0; + }); + const tokenEstimates = additions.map((filepath) => { + const [parent, filename] = filepath.split("/"); + const details = directories.items + .find((folder) => folder.name === parent) + .items.find((file) => file.name === filename); + + const { token_count_estimate = 0 } = details; + return token_count_estimate; + }); + + const totalTokens = tokenEstimates.reduce((a, b) => a + b, 0); + const cachedTotal = cachedTokens.reduce((a, b) => a + b, 0); + const dollarValue = 0.0004 * ((totalTokens - cachedTotal) / 1_000); + + return { + dollarValue, + dollarText: + dollarValue < 0.01 ? "< $0.01" : `about ${dollarFormat(dollarValue)}`, + }; + } + + const { dollarValue, dollarText } = estimateCosts(); + return ( + +
+
+

+ Are you sure you want to embed these documents? +

+ +
+ {dollarValue <= 0 ? ( +

+ You will be embedding {additions.length} new documents into this + workspace. +
+ This will not incur any costs for OpenAI credits. +

+ ) : ( +

+ You will be embedding {additions.length} new documents into this + workspace.
+ This will cost {dollarText} in OpenAI credits. +

+ )} +
+ +
+ + +
+
+
+ + ); +} + +export function useManageWorkspaceModal() { + const [showing, setShowing] = useState(false); + const showModal = () => { + setShowing(true); + }; + const hideModal = () => { + setShowing(false); + }; + + return { showing, showModal, hideModal }; +} diff --git a/frontend/src/components/Modals/NewWorkspace.jsx b/frontend/src/components/Modals/NewWorkspace.jsx new file mode 100644 index 000000000..44aed7d99 --- /dev/null +++ b/frontend/src/components/Modals/NewWorkspace.jsx @@ -0,0 +1,104 @@ +import React, { useRef, useState } from "react"; +import { X } from "react-feather"; +import Workspace from "../../models/workspace"; + +const noop = () => false; +export default function NewWorkspaceModal({ hideModal = noop }) { + const formEl = useRef(null); + const [error, setError] = useState(null); + const handleCreate = async (e) => { + setError(null); + e.preventDefault(); + const data = {}; + const form = new FormData(formEl.current); + for (var [key, value] of form.entries()) data[key] = value; + const { workspace, message } = await Workspace.new(data); + if (!!workspace) window.location.reload(); + setError(message); + }; + + return ( +
+
+
+
+
+

+ Create a New Workspace +

+ +
+
+
+
+
+ + +
+ {error && ( +

+ Error: {error} +

+ )} +

+ After creating a workspace you will be able to add and remove + documents from it. +

+
+
+
+ + +
+
+
+
+
+ ); +} + +export function useNewWorkspaceModal() { + const [showing, setShowing] = useState(false); + const showModal = () => { + setShowing(true); + }; + const hideModal = () => { + setShowing(false); + }; + + return { showing, showModal, hideModal }; +} diff --git a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx new file mode 100644 index 000000000..af9ddefb8 --- /dev/null +++ b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx @@ -0,0 +1,82 @@ +import React, { useState, useEffect } from "react"; +import { Book, Settings } from "react-feather"; +import * as Skeleton from "react-loading-skeleton"; +import "react-loading-skeleton/dist/skeleton.css"; +import Workspace from "../../../models/workspace"; +import ManageWorkspace, { + useManageWorkspaceModal, +} from "../../Modals/ManageWorkspace"; +import paths from "../../../utils/paths"; +import { useParams } from "react-router-dom"; + +export default function ActiveWorkspaces() { + const { slug } = useParams(); + const [loading, setLoading] = useState(true); + const [workspaces, setWorkspaces] = useState([]); + const [selectedWs, setSelectedWs] = useState(null); + const { showing, showModal, hideModal } = useManageWorkspaceModal(); + + useEffect(() => { + async function getWorkspaces() { + const workspaces = await Workspace.all(); + setLoading(false); + setWorkspaces(workspaces); + } + getWorkspaces(); + }, []); + + if (loading) { + return ( + <> + + + ); + } + + return ( + <> + {workspaces.map((workspace) => { + const isActive = workspace.slug === slug; + return ( +
+ + +

+ {workspace.name} +

+ + +
+ ); + })} + {showing && !!selectedWs && ( + + )} + + ); +} diff --git a/frontend/src/components/Sidebar/IndexCount.jsx b/frontend/src/components/Sidebar/IndexCount.jsx new file mode 100644 index 000000000..3565ca5f3 --- /dev/null +++ b/frontend/src/components/Sidebar/IndexCount.jsx @@ -0,0 +1,34 @@ +import pluralize from "pluralize"; +import React, { useEffect, useState } from "react"; +import System from "../../models/system"; +import { numberWithCommas } from "../../utils/numbers"; + +export default function IndexCount() { + const [indexes, setIndexes] = useState(null); + useEffect(() => { + async function indexCount() { + setIndexes(await System.totalIndexes()); + } + indexCount(); + }, []); + + if (indexes === null || indexes === 0) { + return ( +
+
+

+
+
+ ); + } + + return ( +
+
+

+ {numberWithCommas(indexes)} {pluralize("index", indexes)} +

+
+
+ ); +} diff --git a/frontend/src/components/Sidebar/LLMStatus.jsx b/frontend/src/components/Sidebar/LLMStatus.jsx new file mode 100644 index 000000000..dc06c4dcf --- /dev/null +++ b/frontend/src/components/Sidebar/LLMStatus.jsx @@ -0,0 +1,49 @@ +import React, { useEffect, useState } from "react"; +import { AlertCircle, Circle } from "react-feather"; +import System from "../../models/system"; + +export default function LLMStatus() { + const [status, setStatus] = useState(null); + useEffect(() => { + async function checkPing() { + setStatus(await System.ping()); + } + checkPing(); + }, []); + + if (status === null) { + return ( +
+

LLM

+
+

unknown

+ +
+
+ ); + } + + // TODO: add modal or toast on click to identify why this is broken + // need to likely start server. + if (status === false) { + return ( +
+

LLM

+
+

offline

+ +
+
+ ); + } + + return ( +
+

LLM

+
+

online

+ +
+
+ ); +} diff --git a/frontend/src/components/Sidebar/index.jsx b/frontend/src/components/Sidebar/index.jsx new file mode 100644 index 000000000..6d172e20f --- /dev/null +++ b/frontend/src/components/Sidebar/index.jsx @@ -0,0 +1,133 @@ +import React, { useRef } from "react"; +import { BookOpen, Briefcase, Cpu, GitHub, Key, Plus } from "react-feather"; +import IndexCount from "./IndexCount"; +import LLMStatus from "./LLMStatus"; +import KeysModal, { useKeysModal } from "../Modals/Keys"; +import NewWorkspaceModal, { + useNewWorkspaceModal, +} from "../Modals/NewWorkspace"; +import ActiveWorkspaces from "./ActiveWorkspaces"; +import paths from "../../utils/paths"; + +export default function Sidebar() { + const sidebarRef = useRef(null); + const { + showing: showingKeyModal, + showModal: showKeyModal, + hideModal: hideKeyModal, + } = useKeysModal(); + const { + showing: showingNewWsModal, + showModal: showNewWsModal, + hideModal: hideNewWsModal, + } = useNewWorkspaceModal(); + + // const handleWidthToggle = () => { + // if (!sidebarRef.current) return false; + // sidebarRef.current.classList.add('translate-x-[-100%]') + // } + + return ( + <> +
+ {/* */} + +
+ {/* Header Information */} +
+

+ AnythingLLM +

+
+ +
+
+ + {/* Primary Body */} +
+
+
+
+ +
+ +
+
+ +
+
+
+ {showingKeyModal && } + {showingNewWsModal && } + + ); +} diff --git a/frontend/src/components/UserIcon/index.jsx b/frontend/src/components/UserIcon/index.jsx new file mode 100644 index 000000000..a98b11925 --- /dev/null +++ b/frontend/src/components/UserIcon/index.jsx @@ -0,0 +1,27 @@ +import React, { useRef, useEffect } from "react"; +import JAZZ from "@metamask/jazzicon"; + +export default function Jazzicon({ size = 10, user }) { + const divRef = useRef(null); + const seed = user?.uid + ? toPseudoRandomInteger(user.uid) + : Math.floor(100000 + Math.random() * 900000); + const result = JAZZ(size, seed); + + useEffect(() => { + if (!divRef || !divRef.current) return null; + + divRef.current.appendChild(result); + }, []); // eslint-disable-line react-hooks/exhaustive-deps + + return
; +} + +function toPseudoRandomInteger(uidString = "") { + var numberArray = [uidString.length]; + for (var i = 0; i < uidString.length; i++) { + numberArray[i] = uidString.charCodeAt(i); + } + + return numberArray.reduce((a, b) => a + b, 0); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx new file mode 100644 index 000000000..db63e4f75 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx @@ -0,0 +1,106 @@ +import { useEffect, useRef, memo, useState } from "react"; +import { AlertTriangle } from "react-feather"; +import Jazzicon from "../../../../UserIcon"; +import { v4 } from "uuid"; +import { decode as HTMLDecode } from "he"; + +function HistoricalMessage({ + message, + role, + workspace, + sources = [], + error = false, +}) { + const replyRef = useRef(null); + useEffect(() => { + if (replyRef.current) + replyRef.current.scrollIntoView({ behavior: "smooth", block: "end" }); + }, [replyRef.current]); + + if (role === "user") { + return ( +
+
+ + {message} + +
+ +
+ ); + } + + if (error) { + return ( +
+ +
+ + Could not + respond to message. + +
+
+ ); + } + + return ( +
+ +
+ + {message} + + +
+
+ ); +} + +const Citations = ({ sources = [] }) => { + const [show, setShow] = useState(false); + if (sources.length === 0) return null; + + return ( +
+ + {show && ( + <> +
+ {sources.map((source) => { + const { id = null, title, url } = source; + const handleClick = () => { + if (!url) return false; + window.open(url, "_blank"); + }; + return ( + + ); + })} +
+

+ *citation may not be relevant to end result. +

+ + )} +
+ ); +}; + +export default memo(HistoricalMessage); diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx new file mode 100644 index 000000000..3cbe8957c --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx @@ -0,0 +1,112 @@ +import { memo, useEffect, useRef, useState } from "react"; +import { AlertTriangle } from "react-feather"; +import Jazzicon from "../../../../UserIcon"; +import { decode as HTMLDecode } from "he"; + +function PromptReply({ + uuid, + reply, + pending, + error, + workspace, + sources = [], + closed = true, +}) { + const replyRef = useRef(null); + useEffect(() => { + if (replyRef.current) + replyRef.current.scrollIntoView({ behavior: "smooth", block: "end" }); + }, [replyRef.current]); + + if (!reply && !sources.length === 0 && !pending && !error) return null; + if (pending) { + return ( +
+ +
+ +
+ +
+
+ ); + } + + if (error) { + return ( +
+ +
+
+ + Could not + respond to message. + + Reason: {error || "unknown"} +
+
+
+ ); + } + + return ( +
+ +
+

+ {reply} + {!closed && |} +

+ +
+
+ ); +} + +const Citations = ({ sources = [] }) => { + const [show, setShow] = useState(false); + if (sources.length === 0) return null; + + return ( +
+ + {show && ( + <> +
+ {sources.map((source) => { + const { id = null, title, url } = source; + const handleClick = () => { + if (!url) return false; + window.open(url, "_blank"); + }; + return ( + + ); + })} +
+

+ *citation may not be relevant to end result. +

+ + )} +
+ ); +}; + +export default memo(PromptReply); diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx new file mode 100644 index 000000000..0d0222265 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx @@ -0,0 +1,70 @@ +import { Frown } from "react-feather"; +import HistoricalMessage from "./HistoricalMessage"; +import PromptReply from "./PromptReply"; +// import paths from '../../../../../utils/paths'; + +export default function ChatHistory({ history = [], workspace }) { + if (history.length === 0) { + return ( +
+
+ +

No chat history found.

+
+

+ Send your first message to get started. +

+
+ ); + } + + return ( +
+ {history.map( + ( + { + uuid = null, + content, + sources = [], + role, + closed = true, + pending = false, + error = false, + animate = false, + }, + index + ) => { + const isLastBotReply = + index === history.length - 1 && role === "assistant"; + if (isLastBotReply && animate) { + return ( + + ); + } + return ( + + ); + } + )} +
+ ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx new file mode 100644 index 000000000..59c6fe8d5 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx @@ -0,0 +1,106 @@ +import React, { useState, useRef } from "react"; +import { Loader, Menu, Send, X } from "react-feather"; + +export default function PromptInput({ + workspace, + message, + submit, + onChange, + inputDisabled, + buttonDisabled, +}) { + const [showMenu, setShowMenu] = useState(false); + const formRef = useRef(null); + const [_, setFocused] = useState(false); + const handleSubmit = (e) => { + setFocused(false); + submit(e); + }; + const captureEnter = (event) => { + if (event.keyCode == 13) { + if (!event.shiftKey) { + submit(event); + } + } + }; + const adjustTextArea = (event) => { + const element = event.target; + element.style.height = "1px"; + element.style.height = + event.target.value.length !== 0 + ? 25 + element.scrollHeight + "px" + : "1px"; + }; + + const setTextCommand = (command = "") => { + onChange({ target: { value: `${command} ${message}` } }); + }; + + return ( +
+
+
+ {/* Toggle selector? */} + {/* */} +