Feature/drupalwiki collector (#3693)

* Implement DrupalWiki collector

* Add attachment downloading and processing functionality (#3)

* linting

* Linting
Add citation image
small refactors
add URL for citation identifier

---------

Co-authored-by: em <eugen.mayer@kontextwork.de>
Co-authored-by: rexjohannes <53578137+rexjohannes@users.noreply.github.com>
Co-authored-by: Eugen Mayer <136934+EugenMayer@users.noreply.github.com>
This commit is contained in:
Timothy Carambat
2025-04-21 09:17:24 -07:00
committed by GitHub
parent fe59e22397
commit fd4929b4d2
15 changed files with 782 additions and 12 deletions

View File

@@ -10,7 +10,14 @@ const { Telemetry } = require("./telemetry");
const DocumentSyncQueue = {
featureKey: "experimental_live_file_sync",
// update the validFileTypes and .canWatch properties when adding elements here.
validFileTypes: ["link", "youtube", "confluence", "github", "gitlab"],
validFileTypes: [
"link",
"youtube",
"confluence",
"github",
"gitlab",
"drupalwiki",
],
defaultStaleAfter: 604800000,
maxRepeatFailures: 5, // How many times a run can fail in a row before pruning.
writable: [],
@@ -52,6 +59,7 @@ const DocumentSyncQueue = {
if (chunkSource.startsWith("confluence://")) return true; // If is a confluence document link
if (chunkSource.startsWith("github://")) return true; // If is a GitHub file reference
if (chunkSource.startsWith("gitlab://")) return true; // If is a GitLab file reference
if (chunkSource.startsWith("drupalwiki://")) return true; // If is a DrupalWiki document link
return false;
},