eliott/seanime
1
0
Fork 0
mirror of https://github.com/5rahim/seanime synced 2026-04-18 22:24:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): new matching system

Browse Source 
fix: removed reliance on online proxy list
This commit is contained in:
5rahim
2026-02-01 14:31:48 +01:00
parent 57ffd20a8c
commit a959c4c204
43 changed files with 4646 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
codegen/generated/handlers.json
View File

@@ -9088,6 +9088,15 @@
"required": true,
"descriptions": []
},
{
"name": "EnhanceWithOfflineDatabase",
"jsonName": "enhanceWithOfflineDatabase",
"goType": "bool",
"usedStructType": "",
"typescriptType": "boolean",
"required": true,
"descriptions": []
},
{
"name": "SkipLockedFiles",
"jsonName": "skipLockedFiles",

8
codegen/generated/hooks.json
View File

@@ -4255,10 +4255,10 @@
{
"name": "AllMedia",
"jsonName": "allMedia",
"goType": "[]anilist.CompleteAnime",
"typescriptType": "Array\u003cAL_CompleteAnime\u003e",
"usedTypescriptType": "AL_CompleteAnime",
"usedStructName": "anilist.CompleteAnime",
"goType": "[]anime.NormalizedMedia",
"typescriptType": "Array\u003cAnime_NormalizedMedia\u003e",
"usedTypescriptType": "Anime_NormalizedMedia",
"usedStructName": "anime.NormalizedMedia",
"required": false,
"public": true,
"comments": []

695
codegen/generated/public_structs.json
View File

@@ -24221,6 +24221,154 @@
"hook_resolver.Event"
]
},
{
"filepath": "../internal/api/animeofflinedb/animeofflinedb.go",
"filename": "animeofflinedb.go",
"name": "DatabaseRoot",
"formattedName": "DatabaseRoot",
"package": "animeofflinedb",
"fields": [
{
"name": "Data",
"jsonName": "data",
"goType": "[]AnimeEntry",
"typescriptType": "Array\u003cAnimeEntry\u003e",
"usedTypescriptType": "AnimeEntry",
"usedStructName": "animeofflinedb.AnimeEntry",
"required": false,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/api/animeofflinedb/animeofflinedb.go",
"filename": "animeofflinedb.go",
"name": "AnimeEntry",
"formattedName": "AnimeEntry",
"package": "animeofflinedb",
"fields": [
{
"name": "Sources",
"jsonName": "sources",
"goType": "[]string",
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": true,
"comments": []
},
{
"name": "Title",
"jsonName": "title",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
},
{
"name": "Type",
"jsonName": "type",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": [
" TV, MOVIE, OVA, ONA, SPECIAL, UNKNOWN"
]
},
{
"name": "Episodes",
"jsonName": "episodes",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "Status",
"jsonName": "status",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": [
" FINISHED, ONGOING, UPCOMING, UNKNOWN"
]
},
{
"name": "AnimeSeason",
"jsonName": "animeSeason",
"goType": "AnimeSeason",
"typescriptType": "AnimeSeason",
"usedTypescriptType": "AnimeSeason",
"usedStructName": "animeofflinedb.AnimeSeason",
"required": true,
"public": true,
"comments": []
},
{
"name": "Picture",
"jsonName": "picture",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
},
{
"name": "Thumbnail",
"jsonName": "thumbnail",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
},
{
"name": "Synonyms",
"jsonName": "synonyms",
"goType": "[]string",
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/api/animeofflinedb/animeofflinedb.go",
"filename": "animeofflinedb.go",
"name": "AnimeSeason",
"formattedName": "AnimeSeason",
"package": "animeofflinedb",
"fields": [
{
"name": "Season",
"jsonName": "season",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": [
" SPRING, SUMMER, FALL, WINTER, UNDEFINED"
]
},
{
"name": "Year",
"jsonName": "year",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/api/anizip/anizip.go",
"filename": "anizip.go",
@@ -24700,11 +24848,13 @@
"comments": []
},
{
"name": "userAgent",
"jsonName": "userAgent",
"goType": "string",
"typescriptType": "string",
"required": true,
"name": "client",
"jsonName": "client",
"goType": "req.Client",
"typescriptType": "Client",
"usedTypescriptType": "Client",
"usedStructName": "req.Client",
"required": false,
"public": false,
"comments": []
},
@@ -29468,6 +29618,15 @@
"required": true,
"public": true,
"comments": []
},
{
"name": "ScannerUseLegacyMatching",
"jsonName": "scannerUseLegacyMatching",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
}
],
"comments": []
@@ -47955,11 +48114,315 @@
"name": "NormalizedMedia",
"formattedName": "Anime_NormalizedMedia",
"package": "anime",
"fields": [],
"comments": [],
"embeddedStructNames": [
"anilist.BaseAnime"
]
"fields": [
{
"name": "ID",
"jsonName": "ID",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "IdMal",
"jsonName": "IdMal",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
},
{
"name": "Title",
"jsonName": "Title",
"goType": "NormalizedMediaTitle",
"typescriptType": "Anime_NormalizedMediaTitle",
"usedTypescriptType": "Anime_NormalizedMediaTitle",
"usedStructName": "anime.NormalizedMediaTitle",
"required": false,
"public": true,
"comments": []
},
{
"name": "Synonyms",
"jsonName": "Synonyms",
"goType": "[]string",
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": true,
"comments": []
},
{
"name": "Format",
"jsonName": "Format",
"goType": "anilist.MediaFormat",
"typescriptType": "AL_MediaFormat",
"usedTypescriptType": "AL_MediaFormat",
"usedStructName": "anilist.MediaFormat",
"required": false,
"public": true,
"comments": []
},
{
"name": "Status",
"jsonName": "Status",
"goType": "anilist.MediaStatus",
"typescriptType": "AL_MediaStatus",
"usedTypescriptType": "AL_MediaStatus",
"usedStructName": "anilist.MediaStatus",
"required": false,
"public": true,
"comments": []
},
{
"name": "Season",
"jsonName": "Season",
"goType": "anilist.MediaSeason",
"typescriptType": "AL_MediaSeason",
"usedTypescriptType": "AL_MediaSeason",
"usedStructName": "anilist.MediaSeason",
"required": false,
"public": true,
"comments": []
},
{
"name": "Year",
"jsonName": "Year",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
},
{
"name": "StartDate",
"jsonName": "StartDate",
"goType": "NormalizedMediaDate",
"typescriptType": "Anime_NormalizedMediaDate",
"usedTypescriptType": "Anime_NormalizedMediaDate",
"usedStructName": "anime.NormalizedMediaDate",
"required": false,
"public": true,
"comments": []
},
{
"name": "Episodes",
"jsonName": "Episodes",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
},
{
"name": "BannerImage",
"jsonName": "BannerImage",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "CoverImage",
"jsonName": "CoverImage",
"goType": "NormalizedMediaCoverImage",
"typescriptType": "Anime_NormalizedMediaCoverImage",
"usedTypescriptType": "Anime_NormalizedMediaCoverImage",
"usedStructName": "anime.NormalizedMediaCoverImage",
"required": false,
"public": true,
"comments": []
},
{
"name": "NextAiringEpisode",
"jsonName": "NextAiringEpisode",
"goType": "NormalizedMediaNextAiringEpisode",
"typescriptType": "Anime_NormalizedMediaNextAiringEpisode",
"usedTypescriptType": "Anime_NormalizedMediaNextAiringEpisode",
"usedStructName": "anime.NormalizedMediaNextAiringEpisode",
"required": false,
"public": true,
"comments": []
},
{
"name": "fetched",
"jsonName": "fetched",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": false,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/library/anime/normalized_media.go",
"filename": "normalized_media.go",
"name": "NormalizedMediaTitle",
"formattedName": "Anime_NormalizedMediaTitle",
"package": "anime",
"fields": [
{
"name": "Romaji",
"jsonName": "Romaji",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "English",
"jsonName": "English",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "Native",
"jsonName": "Native",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "UserPreferred",
"jsonName": "UserPreferred",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/library/anime/normalized_media.go",
"filename": "normalized_media.go",
"name": "NormalizedMediaDate",
"formattedName": "Anime_NormalizedMediaDate",
"package": "anime",
"fields": [
{
"name": "Year",
"jsonName": "Year",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
},
{
"name": "Month",
"jsonName": "Month",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
},
{
"name": "Day",
"jsonName": "Day",
"goType": "int",
"typescriptType": "number",
"required": false,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/library/anime/normalized_media.go",
"filename": "normalized_media.go",
"name": "NormalizedMediaCoverImage",
"formattedName": "Anime_NormalizedMediaCoverImage",
"package": "anime",
"fields": [
{
"name": "ExtraLarge",
"jsonName": "ExtraLarge",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "Large",
"jsonName": "Large",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "Medium",
"jsonName": "Medium",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
},
{
"name": "Color",
"jsonName": "Color",
"goType": "string",
"typescriptType": "string",
"required": false,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/library/anime/normalized_media.go",
"filename": "normalized_media.go",
"name": "NormalizedMediaNextAiringEpisode",
"formattedName": "Anime_NormalizedMediaNextAiringEpisode",
"package": "anime",
"fields": [
{
"name": "AiringAt",
"jsonName": "AiringAt",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "TimeUntilAiring",
"jsonName": "TimeUntilAiring",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "Episode",
"jsonName": "Episode",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
}
],
"comments": []
},
{
"filepath": "../internal/library/anime/normalized_media.go",
@@ -51196,10 +51659,10 @@
{
"name": "AllMedia",
"jsonName": "allMedia",
"goType": "[]anilist.CompleteAnime",
"typescriptType": "Array\u003cAL_CompleteAnime\u003e",
"usedTypescriptType": "AL_CompleteAnime",
"usedStructName": "anilist.CompleteAnime",
"goType": "[]anime.NormalizedMedia",
"typescriptType": "Array\u003cAnime_NormalizedMedia\u003e",
"usedTypescriptType": "Anime_NormalizedMedia",
"usedStructName": "anime.NormalizedMedia",
"required": false,
"public": true,
"comments": []
@@ -51637,17 +52100,6 @@
"public": true,
"comments": []
},
{
"name": "CompleteAnimeCache",
"jsonName": "CompleteAnimeCache",
"goType": "anilist.CompleteAnimeCache",
"typescriptType": "AL_CompleteAnimeCache",
"usedTypescriptType": "AL_CompleteAnimeCache",
"usedStructName": "anilist.CompleteAnimeCache",
"required": false,
"public": true,
"comments": []
},
{
"name": "Logger",
"jsonName": "Logger",
@@ -51700,6 +52152,24 @@
"required": true,
"public": true,
"comments": []
},
{
"name": "Debug",
"jsonName": "Debug",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
},
{
"name": "UseLegacyMatching",
"jsonName": "UseLegacyMatching",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
}
],
"comments": []
@@ -51714,10 +52184,10 @@
{
"name": "AllMedia",
"jsonName": "AllMedia",
"goType": "[]anilist.CompleteAnime",
"typescriptType": "Array\u003cAL_CompleteAnime\u003e",
"usedTypescriptType": "AL_CompleteAnime",
"usedStructName": "anilist.CompleteAnime",
"goType": "[]anime.NormalizedMedia",
"typescriptType": "Array\u003cAnime_NormalizedMedia\u003e",
"usedTypescriptType": "Anime_NormalizedMedia",
"usedStructName": "anime.NormalizedMedia",
"required": false,
"public": true,
"comments": []
@@ -51754,6 +52224,19 @@
"public": true,
"comments": []
},
{
"name": "NormalizedTitlesCache",
"jsonName": "NormalizedTitlesCache",
"goType": "map[int][]NormalizedTitle",
"typescriptType": "Record\u003cnumber, Array\u003cScanner_NormalizedTitle\u003e\u003e",
"usedTypescriptType": "Scanner_NormalizedTitle",
"usedStructName": "scanner.NormalizedTitle",
"required": false,
"public": true,
"comments": [
" mediaId -\u003e normalized titles"
]
},
{
"name": "ScanLogger",
"jsonName": "ScanLogger",
@@ -51765,6 +52248,17 @@
"public": true,
"comments": []
},
{
"name": "TokenIndex",
"jsonName": "TokenIndex",
"goType": "map[string][]anime.NormalizedMedia",
"typescriptType": "Record\u003cstring, Array\u003cAnime_NormalizedMedia\u003e\u003e",
"usedTypescriptType": "Anime_NormalizedMedia",
"usedStructName": "anime.NormalizedMedia",
"required": false,
"public": true,
"comments": []
},
{
"name": "engTitles",
"jsonName": "engTitles",
@@ -51772,7 +52266,9 @@
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": false,
"comments": []
"comments": [
" legacy"
]
},
{
"name": "romTitles",
@@ -51781,7 +52277,9 @@
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": false,
"comments": []
"comments": [
" legacy"
]
},
{
"name": "synonyms",
@@ -51790,18 +52288,9 @@
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": false,
"comments": []
},
{
"name": "allMedia",
"jsonName": "allMedia",
"goType": "[]anilist.CompleteAnime",
"typescriptType": "Array\u003cAL_CompleteAnime\u003e",
"usedTypescriptType": "AL_CompleteAnime",
"usedStructName": "anilist.CompleteAnime",
"required": false,
"public": false,
"comments": []
"comments": [
" legacy"
]
}
],
"comments": []
@@ -51816,10 +52305,10 @@
{
"name": "AllMedia",
"jsonName": "AllMedia",
"goType": "[]anilist.CompleteAnime",
"typescriptType": "Array\u003cAL_CompleteAnime\u003e",
"usedTypescriptType": "AL_CompleteAnime",
"usedStructName": "anilist.CompleteAnime",
"goType": "[]anime.NormalizedMedia",
"typescriptType": "Array\u003cAnime_NormalizedMedia\u003e",
"usedTypescriptType": "Anime_NormalizedMedia",
"usedStructName": "anime.NormalizedMedia",
"required": false,
"public": true,
"comments": []
@@ -51887,6 +52376,15 @@
"public": true,
"comments": []
},
{
"name": "UseLegacyEnhanced",
"jsonName": "UseLegacyEnhanced",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
},
{
"name": "PlatformRef",
"jsonName": "PlatformRef",
@@ -52154,6 +52652,15 @@
"public": true,
"comments": []
},
{
"name": "EnhanceWithOfflineDatabase",
"jsonName": "EnhanceWithOfflineDatabase",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
},
{
"name": "PlatformRef",
"jsonName": "PlatformRef",
@@ -52245,6 +52752,15 @@
"public": true,
"comments": []
},
{
"name": "UseLegacyMatching",
"jsonName": "UseLegacyMatching",
"goType": "bool",
"typescriptType": "boolean",
"required": true,
"public": true,
"comments": []
},
{
"name": "MatchingThreshold",
"jsonName": "MatchingThreshold",
@@ -52252,7 +52768,9 @@
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
"comments": [
" only used by legacy"
]
},
{
"name": "MatchingAlgorithm",
@@ -52261,7 +52779,9 @@
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
"comments": [
" only used by legacy"
]
},
{
"name": "WithShelving",
@@ -52342,6 +52862,83 @@
" ScanLogger is a custom logger struct for scanning operations."
]
},
{
"filepath": "../internal/library/scanner/title_normalization.go",
"filename": "title_normalization.go",
"name": "NormalizedTitle",
"formattedName": "Scanner_NormalizedTitle",
"package": "scanner",
"fields": [
{
"name": "Original",
"jsonName": "Original",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
},
{
"name": "Normalized",
"jsonName": "Normalized",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": []
},
{
"name": "Tokens",
"jsonName": "Tokens",
"goType": "[]string",
"typescriptType": "Array\u003cstring\u003e",
"required": false,
"public": true,
"comments": []
},
{
"name": "Season",
"jsonName": "Season",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "Part",
"jsonName": "Part",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "Year",
"jsonName": "Year",
"goType": "int",
"typescriptType": "number",
"required": true,
"public": true,
"comments": []
},
{
"name": "CleanBaseTitle",
"jsonName": "CleanBaseTitle",
"goType": "string",
"typescriptType": "string",
"required": true,
"public": true,
"comments": [
" Title without season/part/year info"
]
}
],
"comments": [
" NormalizedTitle holds the normalized form and extracted metadata"
]
},
{
"filepath": "../internal/library/scanner/watcher.go",
"filename": "watcher.go",

304
internal/api/animeofflinedb/animeofflinedb.go Normal file
View File

@@ -0,0 +1,304 @@
package animeofflinedb
import (
"bufio"
"errors"
"net/http"
"seanime/internal/api/anilist"
"seanime/internal/library/anime"
"strconv"
"strings"
"sync"
"github.com/goccy/go-json"
)
const (
DatabaseURL = "https://github.com/manami-project/anime-offline-database/releases/download/latest/anime-offline-database.jsonl"
)
type animeEntry struct {
Sources []string `json:"sources"`
Title string `json:"title"`
Type string `json:"type"`
Episodes int `json:"episodes"`
Status string `json:"status"`
AnimeSeason animeSeason `json:"animeSeason"`
Picture string `json:"picture"`
Thumbnail string `json:"thumbnail"`
Synonyms []string `json:"synonyms"`
}
type animeSeason struct {
Season string `json:"season"`
Year int `json:"year"`
}
const (
anilistPrefix = "https://anilist.co/anime/"
malPrefix = "https://myanimelist.net/anime/"
)
var (
normalizedMediaCache []*anime.NormalizedMedia
normalizedMediaCacheMu sync.RWMutex
)
// FetchAndConvertDatabase fetches the database and converts entries to NormalizedMedia.
// Only entries with valid AniList IDs are included.
// Entries that already exist in existingMediaIDs are excluded.
func FetchAndConvertDatabase(existingMediaIDs map[int]bool) ([]*anime.NormalizedMedia, error) {
// check cache first
normalizedMediaCacheMu.RLock()
if normalizedMediaCache != nil {
// filter cached results by existingMediaIDs
result := filterByExistingIDs(normalizedMediaCache, existingMediaIDs)
normalizedMediaCacheMu.RUnlock()
return result, nil
}
normalizedMediaCacheMu.RUnlock()
resp, err := http.Get(DatabaseURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.New("failed to fetch database: " + resp.Status)
}
// stream and convert directly to NormalizedMedia
// estimate ~20000 entries with anilist ids
allMedia := make([]*anime.NormalizedMedia, 0, 20000)
result := make([]*anime.NormalizedMedia, 0, 20000)
scanner := bufio.NewScanner(resp.Body)
buf := make([]byte, 0, 64*1024)
scanner.Buffer(buf, 1024*1024)
lineNum := 0
for scanner.Scan() {
lineNum++
if lineNum == 1 {
continue // skip metadata line
}
line := scanner.Bytes()
if len(line) == 0 {
continue
}
// parse entry
var entry animeEntry
if err := json.Unmarshal(line, &entry); err != nil {
continue
}
// convert immediately and discard raw entry
media := convertEntryToNormalizedMedia(&entry)
if media == nil {
continue // no anilist id
}
// add to cache (all media with anilist ids)
allMedia = append(allMedia, media)
// check if should be included in result
if existingMediaIDs == nil || !existingMediaIDs[media.ID] {
result = append(result, media)
}
// entry goes out of scope here and can be GC'd
}
if err := scanner.Err(); err != nil {
return nil, err
}
// cache all media for future calls?
normalizedMediaCacheMu.Lock()
normalizedMediaCache = allMedia
normalizedMediaCacheMu.Unlock()
return result, nil
}
// filterByExistingIDs filters cached media by existing IDs
func filterByExistingIDs(media []*anime.NormalizedMedia, existingMediaIDs map[int]bool) []*anime.NormalizedMedia {
if existingMediaIDs == nil || len(existingMediaIDs) == 0 {
return media
}
result := make([]*anime.NormalizedMedia, 0, len(media))
for _, m := range media {
if !existingMediaIDs[m.ID] {
result = append(result, m)
}
}
return result
}
// ClearCache clears the normalized media cache
func ClearCache() {
normalizedMediaCacheMu.Lock()
normalizedMediaCache = nil
normalizedMediaCacheMu.Unlock()
}
// convertEntryToNormalizedMedia converts an animeEntry to NormalizedMedia.
// Returns nil if the entry has no anilist id.
func convertEntryToNormalizedMedia(e *animeEntry) *anime.NormalizedMedia {
// extract anilist id
anilistID := extractAnilistID(e.Sources)
if anilistID == 0 {
return nil
}
malID := extractMALID(e.Sources)
var malIDPtr *int
if malID > 0 {
malIDPtr = &malID
}
// convert type to anilist.MediaFormat
var format *anilist.MediaFormat
switch e.Type {
case "TV":
f := anilist.MediaFormatTv
format = &f
case "MOVIE":
f := anilist.MediaFormatMovie
format = &f
case "OVA":
f := anilist.MediaFormatOva
format = &f
case "ONA":
f := anilist.MediaFormatOna
format = &f
case "SPECIAL":
f := anilist.MediaFormatSpecial
format = &f
}
// convert status to anilist.MediaStatus
var status *anilist.MediaStatus
switch e.Status {
case "FINISHED":
s := anilist.MediaStatusFinished
status = &s
case "ONGOING":
s := anilist.MediaStatusReleasing
status = &s
case "UPCOMING":
s := anilist.MediaStatusNotYetReleased
status = &s
}
// convert season to anilist.MediaSeason
var season *anilist.MediaSeason
switch e.AnimeSeason.Season {
case "SPRING":
s := anilist.MediaSeasonSpring
season = &s
case "SUMMER":
s := anilist.MediaSeasonSummer
season = &s
case "FALL":
s := anilist.MediaSeasonFall
season = &s
case "WINTER":
s := anilist.MediaSeasonWinter
season = &s
}
// reuse the same string pointer for all title fields
title := e.Title
titleObj := &anime.NormalizedMediaTitle{
Romaji: &title,
English: &title,
UserPreferred: &title,
}
// build synonyms
var synonyms []*string
if len(e.Synonyms) > 0 {
synonyms = make([]*string, len(e.Synonyms))
for i := range e.Synonyms {
synonyms[i] = &e.Synonyms[i]
}
}
// build start date
var startDate *anime.NormalizedMediaDate
if e.AnimeSeason.Year > 0 {
year := e.AnimeSeason.Year
startDate = &anime.NormalizedMediaDate{
Year: &year,
}
}
var episodes *int
if e.Episodes > 0 {
ep := e.Episodes
episodes = &ep
}
var year *int
if e.AnimeSeason.Year > 0 {
y := e.AnimeSeason.Year
year = &y
}
var coverImage *anime.NormalizedMediaCoverImage
if e.Thumbnail != "" || e.Picture != "" {
coverImage = &anime.NormalizedMediaCoverImage{
Large: &e.Picture,
Medium: &e.Thumbnail,
}
}
return anime.NewNormalizedMediaFromOfflineDB(
anilistID,
malIDPtr,
titleObj,
synonyms,
format,
status,
season,
year,
startDate,
episodes,
coverImage,
)
}
func extractAnilistID(sources []string) int {
for _, source := range sources {
if strings.HasPrefix(source, anilistPrefix) {
idStr := source[len(anilistPrefix):]
// handle potential trailing slashes or query params
if idx := strings.IndexAny(idStr, "/?"); idx != -1 {
idStr = idStr[:idx]
}
if id, err := strconv.Atoi(idStr); err == nil {
return id
}
}
}
return 0
}
func extractMALID(sources []string) int {
for _, source := range sources {
if strings.HasPrefix(source, malPrefix) {
idStr := source[len(malPrefix):]
if idx := strings.IndexAny(idStr, "/?"); idx != -1 {
idStr = idStr[:idx]
}
if id, err := strconv.Atoi(idStr); err == nil {
return id
}
}
}
return 0
}

266
internal/api/animeofflinedb/animeofflinedb_test.go Normal file
View File

@@ -0,0 +1,266 @@
package animeofflinedb
import (
"testing"
)
func TestGetAnilistID(t *testing.T) {
t.Parallel()
tests := []struct {
name string
sources []string
want int
}{
{
name: "Death Note",
sources: []string{
"https://anidb.net/anime/4563",
"https://anilist.co/anime/1535",
"https://anime-planet.com/anime/death-note",
"https://myanimelist.net/anime/1535",
},
want: 1535,
},
{
name: "No AniList source",
sources: []string{
"https://anidb.net/anime/4563",
"https://myanimelist.net/anime/1535",
},
want: 0,
},
{
name: "Empty sources",
sources: []string{},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e := &AnimeEntry{Sources: tt.sources}
got := e.GetAnilistID()
if got != tt.want {
t.Errorf("GetAnilistID() = %v, want %v", got, tt.want)
}
})
}
}
func TestGetMALID(t *testing.T) {
t.Parallel()
tests := []struct {
name string
sources []string
want int
}{
{
name: "Death Note",
sources: []string{
"https://anidb.net/anime/4563",
"https://anilist.co/anime/1535",
"https://myanimelist.net/anime/1535",
},
want: 1535,
},
{
name: "No MAL source",
sources: []string{
"https://anidb.net/anime/4563",
"https://anilist.co/anime/1535",
},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e := &AnimeEntry{Sources: tt.sources}
got := e.GetMALID()
if got != tt.want {
t.Errorf("GetMALID() = %v, want %v", got, tt.want)
}
})
}
}
func TestToNormalizedMedia(t *testing.T) {
t.Parallel()
tests := []struct {
name string
entry AnimeEntry
wantNil bool
wantID int
wantTitle string
wantEps int
wantFormat string
}{
{
name: "Valid entry",
entry: AnimeEntry{
Sources: []string{
"https://anilist.co/anime/1535",
"https://myanimelist.net/anime/1535",
},
Title: "Death Note",
Type: "TV",
Episodes: 37,
Status: "FINISHED",
AnimeSeason: AnimeSeason{
Season: "FALL",
Year: 2006,
},
Synonyms: []string{"DN", "デスノート"},
},
wantNil: false,
wantID: 1535,
wantTitle: "Death Note",
wantEps: 37,
wantFormat: "TV",
},
{
name: "No AniList ID",
entry: AnimeEntry{
Sources: []string{
"https://myanimelist.net/anime/1535",
},
Title: "Death Note",
},
wantNil: true,
},
{
name: "Movie format",
entry: AnimeEntry{
Sources: []string{
"https://anilist.co/anime/199",
},
Title: "Spirited Away",
Type: "MOVIE",
Episodes: 1,
},
wantNil: false,
wantID: 199,
wantTitle: "Spirited Away",
wantEps: 1,
wantFormat: "MOVIE",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.entry.ToNormalizedMedia()
if tt.wantNil {
if got != nil {
t.Errorf("ToNormalizedMedia() expected nil, got %+v", got)
}
return
}
if got == nil {
t.Fatal("ToNormalizedMedia() returned nil, expected non-nil")
}
if got.ID != tt.wantID {
t.Errorf("ID = %v, want %v", got.ID, tt.wantID)
}
if got.GetTitleSafe() != tt.wantTitle {
t.Errorf("Title = %v, want %v", got.GetTitleSafe(), tt.wantTitle)
}
if tt.wantEps > 0 {
if got.Episodes == nil || *got.Episodes != tt.wantEps {
t.Errorf("Episodes = %v, want %v", got.Episodes, tt.wantEps)
}
}
})
}
}
func TestConvertToNormalizedMedia(t *testing.T) {
t.Parallel()
db := &DatabaseRoot{
Data: []AnimeEntry{
{
Sources: []string{"https://anilist.co/anime/1"},
Title: "Test Anime 1",
Type: "TV",
Episodes: 12,
},
{
Sources: []string{"https://anilist.co/anime/2"},
Title: "Test Anime 2",
Type: "MOVIE",
Episodes: 1,
},
{
Sources: []string{"https://myanimelist.net/anime/3"}, // No AniList ID
Title: "Test Anime 3",
Type: "TV",
Episodes: 24,
},
},
}
// Exclude anime with ID 1
existing := map[int]bool{1: true}
result := ConvertToNormalizedMedia(db, existing)
// Should only include anime 2 (anime 1 is excluded, anime 3 has no AniList ID)
if len(result) != 1 {
t.Errorf("Expected 1 result, got %d", len(result))
}
if len(result) > 0 && result[0].ID != 2 {
t.Errorf("Expected anime ID 2, got %d", result[0].ID)
}
}
// TestFetchDatabase tests fetching the actual database (integration)
func TestFetchDatabase(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
// Clear any cached data first
ClearCache()
db, err := FetchDatabase()
if err != nil {
t.Fatalf("FetchDatabase() error = %v", err)
}
if db == nil {
t.Fatal("FetchDatabase() returned nil")
}
if len(db.Data) == 0 {
t.Error("FetchDatabase() returned empty data")
}
// Check that we can find a known anime (Death Note)
var foundDeathNote bool
for _, entry := range db.Data {
if entry.GetAnilistID() == 1535 {
foundDeathNote = true
if entry.Title != "Death Note" {
t.Errorf("Expected 'Death Note', got '%s'", entry.Title)
}
break
}
}
if !foundDeathNote {
t.Error("Could not find Death Note (AniList ID 1535) in database")
}
t.Logf("Fetched %d anime entries from database", len(db.Data))
ClearCache()
}

66
internal/api/filler/filler.go
View File

@@ -4,9 +4,11 @@ import (
"fmt"
"seanime/internal/util"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/adrg/strutil/metrics"
"github.com/gocolly/colly"
"github.com/imroc/req/v3"
"github.com/rs/zerolog"
)
@@ -34,17 +36,19 @@ type (
type (
AnimeFillerList struct {
baseUrl string
userAgent string
logger *zerolog.Logger
baseUrl string
client *req.Client
logger *zerolog.Logger
}
)
func NewAnimeFillerList(logger *zerolog.Logger) *AnimeFillerList {
return &AnimeFillerList{
baseUrl: "https://www.animefillerlist.com",
userAgent: util.GetRandomUserAgent(),
logger: logger,
baseUrl: "https://www.animefillerlist.com",
client: req.C().
SetTimeout(10 * time.Second).
ImpersonateChrome(),
logger: logger,
}
}
@@ -52,23 +56,25 @@ func (af *AnimeFillerList) Search(opts SearchOptions) (result *SearchResult, err
defer util.HandlePanicInModuleWithError("api/metadata/filler/Search", &err)
c := colly.NewCollector(
colly.UserAgent(af.userAgent),
)
ret := make([]*SearchResult, 0)
c.OnHTML("div.Group > ul > li > a", func(e *colly.HTMLElement) {
ret = append(ret, &SearchResult{
Slug: e.Attr("href"),
Title: e.Text,
})
})
err = c.Visit(fmt.Sprintf("%s/shows", af.baseUrl))
resp, err := af.client.R().Get(fmt.Sprintf("%s/shows", af.baseUrl))
if err != nil {
return nil, err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
}
doc.Find("div.Group > ul > li > a").Each(func(i int, s *goquery.Selection) {
ret = append(ret, &SearchResult{
Slug: s.AttrOr("href", ""),
Title: s.Text(),
})
})
if len(ret) == 0 {
return nil, fmt.Errorf("no results found")
@@ -161,23 +167,25 @@ func (af *AnimeFillerList) FindFillerData(slug string) (ret *Data, err error) {
defer util.HandlePanicInModuleWithError("api/metadata/filler/FindFillerEpisodes", &err)
c := colly.NewCollector(
colly.UserAgent(af.userAgent),
)
ret = &Data{
FillerEpisodes: make([]string, 0),
}
fillerEps := make([]string, 0)
c.OnHTML("tr.filler", func(e *colly.HTMLElement) {
fillerEps = append(fillerEps, e.ChildText("td.Number"))
})
err = c.Visit(fmt.Sprintf("%s%s", af.baseUrl, slug))
resp, err := af.client.R().Get(fmt.Sprintf("%s%s", af.baseUrl, slug))
if err != nil {
return nil, err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
}
fillerEps := make([]string, 0)
doc.Find("tr.filler").Each(func(i int, s *goquery.Selection) {
fillerEps = append(fillerEps, s.Find("td.Number").Text())
})
ret.FillerEpisodes = fillerEps

2
internal/database/models/models.go
View File

@@ -96,6 +96,8 @@ type LibrarySettings struct {
AutoSaveCurrentMediaOffline bool `gorm:"column:auto_save_current_media_offline" json:"autoSaveCurrentMediaOffline"`
// v3+
UseFallbackMetadataProvider bool `gorm:"column:use_fallback_metadata_provider" json:"useFallbackMetadataProvider"`
// v3.5+
ScannerUseLegacyMatching bool `gorm:"column:scanner_use_legacy_matching" json:"scannerUseLegacyMatching"`
}
func (o *LibrarySettings) GetLibraryPaths() (ret []string) {

6
internal/doh/doh.go
View File

@@ -25,13 +25,13 @@ func HandleDoH(dohUrl string, logger *zerolog.Logger) {
return
}
// Override the default resolver
net.DefaultResolver = resolver
// Test the resolver
_, err = resolver.LookupIPAddr(context.Background(), "ipv4.google.com")
if err != nil {
logger.Error().Err(err).Msgf("doh: DoH resolver failed lookup: %s", dohUrl)
return
}
// Override the default resolver
net.DefaultResolver = resolver
}

77
internal/extension_repo/goja_plugin_types/app.d.ts vendored
View File

@@ -1610,7 +1610,7 @@ declare namespace $app {
interface ScanMediaFetcherCompletedEvent {
next(): void;
allMedia?: Array<AL_CompleteAnime>;
allMedia?: Array<Anime_NormalizedMedia>;
unknownMediaIds?: Array<number>;
}
@@ -3278,29 +3278,58 @@ declare namespace $app {
* - Filepath: internal/library/anime/normalized_media.go
*/
interface Anime_NormalizedMedia {
id: number;
idMal?: number;
siteUrl?: string;
status?: AL_MediaStatus;
season?: AL_MediaSeason;
type?: AL_MediaType;
format?: AL_MediaFormat;
seasonYear?: number;
bannerImage?: string;
episodes?: number;
synonyms?: Array<string>;
isAdult?: boolean;
countryOfOrigin?: string;
meanScore?: number;
description?: string;
genres?: Array<string>;
duration?: number;
trailer?: AL_BaseAnime_Trailer;
title?: AL_BaseAnime_Title;
coverImage?: AL_BaseAnime_CoverImage;
startDate?: AL_BaseAnime_StartDate;
endDate?: AL_BaseAnime_EndDate;
nextAiringEpisode?: AL_BaseAnime_NextAiringEpisode;
ID: number;
IdMal?: number;
Title?: Anime_NormalizedMediaTitle;
Synonyms?: Array<string>;
Format?: AL_MediaFormat;
Status?: AL_MediaStatus;
Season?: AL_MediaSeason;
Year?: number;
StartDate?: Anime_NormalizedMediaDate;
Episodes?: number;
BannerImage?: string;
CoverImage?: Anime_NormalizedMediaCoverImage;
NextAiringEpisode?: Anime_NormalizedMediaNextAiringEpisode;
fetched: boolean;
}
/**
* - Filepath: internal/library/anime/normalized_media.go
*/
interface Anime_NormalizedMediaCoverImage {
ExtraLarge?: string;
Large?: string;
Medium?: string;
Color?: string;
}
/**
* - Filepath: internal/library/anime/normalized_media.go
*/
interface Anime_NormalizedMediaDate {
Year?: number;
Month?: number;
Day?: number;
}
/**
* - Filepath: internal/library/anime/normalized_media.go
*/
interface Anime_NormalizedMediaNextAiringEpisode {
AiringAt: number;
TimeUntilAiring: number;
Episode: number;
}
/**
* - Filepath: internal/library/anime/normalized_media.go
*/
interface Anime_NormalizedMediaTitle {
Romaji?: string;
English?: string;
Native?: string;
UserPreferred?: string;
}
/**

35
internal/handlers/proxy.go
View File

@@ -2,7 +2,6 @@ package handlers
import (
"bytes"
"crypto/tls"
"io"
"net/http"
url2 "net/url"
@@ -13,22 +12,15 @@ import (
"github.com/5rahim/hls-m3u8/m3u8"
"github.com/goccy/go-json"
"github.com/imroc/req/v3"
"github.com/labstack/echo/v4"
"github.com/rs/zerolog/log"
)
var proxyUA = util.GetRandomUserAgent()
var videoProxyClient2 = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
ForceAttemptHTTP2: false, // Fixes issues on Linux
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
Timeout: 60 * time.Second,
}
var videoProxyClient2 = req.C().
SetTimeout(60 * time.Second).
EnableInsecureSkipVerify().
ImpersonateChrome()
func (h *Handler) VideoProxy(c echo.Context) (err error) {
defer util.HandlePanicInModuleWithError("util/VideoProxy", &err)
@@ -37,12 +29,7 @@ func (h *Handler) VideoProxy(c echo.Context) (err error) {
headers := c.QueryParam("headers")
authToken := c.QueryParam("token")
// Always use GET request internally, even for HEAD requests
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Error().Err(err).Msg("proxy: Error creating request")
return echo.NewHTTPError(http.StatusInternalServerError)
}
r := videoProxyClient2.R()
var headerMap map[string]string
if headers != "" {
@@ -51,17 +38,17 @@ func (h *Handler) VideoProxy(c echo.Context) (err error) {
return echo.NewHTTPError(http.StatusInternalServerError)
}
for key, value := range headerMap {
req.Header.Set(key, value)
r.SetHeader(key, value)
}
}
req.Header.Set("User-Agent", proxyUA)
req.Header.Set("Accept", "*/*")
r.SetHeader("Accept", "*/*")
if rangeHeader := c.Request().Header.Get("Range"); rangeHeader != "" {
req.Header.Set("Range", rangeHeader)
r.SetHeader("Range", rangeHeader)
}
resp, err := videoProxyClient2.Do(req)
// Always use GET request internally, even for HEAD requests
resp, err := r.Get(url)
if err != nil {
log.Error().Err(err).Msg("proxy: Error sending request")

41
internal/handlers/scan.go
View File

@@ -19,9 +19,10 @@ import (
func (h *Handler) HandleScanLocalFiles(c echo.Context) error {
type body struct {
Enhanced bool `json:"enhanced"`
SkipLockedFiles bool `json:"skipLockedFiles"`
SkipIgnoredFiles bool `json:"skipIgnoredFiles"`
Enhanced bool `json:"enhanced"`
EnhanceWithOfflineDatabase bool `json:"enhanceWithOfflineDatabase"`
SkipLockedFiles bool `json:"skipLockedFiles"`
SkipIgnoredFiles bool `json:"skipIgnoredFiles"`
}
var b body
@@ -67,22 +68,24 @@ func (h *Handler) HandleScanLocalFiles(c echo.Context) error {
// Create a new scanner
sc := scanner.Scanner{
DirPath: libraryPath,
OtherDirPaths: additionalLibraryPaths,
Enhanced: b.Enhanced,
PlatformRef: h.App.AnilistPlatformRef,
Logger: h.App.Logger,
WSEventManager: h.App.WSEventManager,
ExistingLocalFiles: existingLfs,
SkipLockedFiles: b.SkipLockedFiles,
SkipIgnoredFiles: b.SkipIgnoredFiles,
ScanSummaryLogger: scanSummaryLogger,
ScanLogger: scanLogger,
MetadataProviderRef: h.App.MetadataProviderRef,
MatchingAlgorithm: h.App.Settings.GetLibrary().ScannerMatchingAlgorithm,
MatchingThreshold: h.App.Settings.GetLibrary().ScannerMatchingThreshold,
WithShelving: true,
ExistingShelvedFiles: existingShelvedLfs,
DirPath: libraryPath,
OtherDirPaths: additionalLibraryPaths,
Enhanced: b.Enhanced,
EnhanceWithOfflineDatabase: b.EnhanceWithOfflineDatabase,
PlatformRef: h.App.AnilistPlatformRef,
Logger: h.App.Logger,
WSEventManager: h.App.WSEventManager,
ExistingLocalFiles: existingLfs,
SkipLockedFiles: b.SkipLockedFiles,
SkipIgnoredFiles: b.SkipIgnoredFiles,
ScanSummaryLogger: scanSummaryLogger,
ScanLogger: scanLogger,
MetadataProviderRef: h.App.MetadataProviderRef,
MatchingAlgorithm: h.App.Settings.GetLibrary().ScannerMatchingAlgorithm,
MatchingThreshold: h.App.Settings.GetLibrary().ScannerMatchingThreshold,
UseLegacyMatching: h.App.Settings.GetLibrary().ScannerUseLegacyMatching,
WithShelving: true,
ExistingShelvedFiles: existingShelvedLfs,
}
// Scan the library

192
internal/library/anime/localfile_helper.go
View File

@@ -270,12 +270,19 @@ func (f *LocalFile) GetParsedTitle() string {
return ""
}
func (f *LocalFile) GetFolderTitle() string {
func (f *LocalFile) GetFolderTitle(all ...bool) string {
folderTitles := make([]string, 0)
if f.ParsedFolderData != nil && len(f.ParsedFolderData) > 0 {
// Go through each folder data and keep the ones with a title
data := lo.Filter(f.ParsedFolderData, func(fpd *LocalFileParsedData, _ int) bool {
return len(fpd.Title) > 0
// remove non-anime titles
cleanTitle := strings.TrimSpace(strings.ToLower(fpd.Title))
if len(all) == 0 || !all[0] {
if _, ok := comparison.IgnoredFilenames[cleanTitle]; ok {
return false
}
}
return len(cleanTitle) > 0
})
if len(data) == 0 {
return ""
@@ -297,7 +304,7 @@ func (f *LocalFile) GetTitleVariations() []*string {
folderSeason := 0
// Get the season from the folder data
if f.ParsedFolderData != nil && len(f.ParsedFolderData) > 0 {
if len(f.ParsedFolderData) > 0 {
v, found := lo.Find(f.ParsedFolderData, func(fpd *LocalFileParsedData) bool {
return len(fpd.Season) > 0
})
@@ -319,141 +326,107 @@ func (f *LocalFile) GetTitleVariations() []*string {
part := 0
// Get the part from the folder data
if f.ParsedFolderData != nil && len(f.ParsedFolderData) > 0 {
if len(f.ParsedFolderData) > 0 {
v, found := lo.Find(f.ParsedFolderData, func(fpd *LocalFileParsedData) bool {
return len(fpd.Part) > 0
})
if found {
if res, ok := util.StringToInt(v.Season); ok {
if res, ok := util.StringToInt(v.Part); ok {
part = res
}
}
}
// Devnote: This causes issues when an episode title contains "Part"
//// Get the part from the filename
//if len(f.ParsedData.Part) > 0 {
// if res, ok := util.StringToInt(f.ParsedData.Part); ok {
// part = res
// }
//}
folderTitle := f.GetFolderTitle()
if comparison.ValueContainsIgnoredKeywords(folderTitle) {
folderTitle = ""
}
if len(f.ParsedData.Title) == 0 && len(folderTitle) == 0 {
return make([]*string, 0)
}
titleVariations := make([]string, 0)
titleVariations := make([]string, 0, 20) // Pre-allocate for efficiency
bothTitles := len(f.ParsedData.Title) > 0 && len(folderTitle) > 0 // Both titles are present (filename and folder)
noSeasonsOrParts := folderSeason == 0 && season == 0 && part == 0 // No seasons or parts are present
bothTitlesSimilar := bothTitles && strings.Contains(folderTitle, f.ParsedData.Title) // The folder title contains the filename title
eitherSeason := folderSeason > 0 || season > 0 // Either season is present
eitherSeasonFirst := folderSeason == 1 || season == 1 // Either season is 1
bothTitles := len(f.ParsedData.Title) > 0 && len(folderTitle) > 0
noSeasonsOrParts := folderSeason == 0 && season == 0 && part == 0
bothTitlesSimilar := bothTitles && strings.Contains(folderTitle, f.ParsedData.Title)
eitherSeason := folderSeason > 0 || season > 0
eitherSeasonFirst := folderSeason == 1 || season == 1
// Part
// Collect base titles to use
baseTitles := make([]string, 0, 4)
if len(f.ParsedData.Title) > 0 {
baseTitles = append(baseTitles, f.ParsedData.Title)
}
if len(folderTitle) > 0 && folderTitle != f.ParsedData.Title {
baseTitles = append(baseTitles, folderTitle)
}
// Always add the raw base titles
for _, t := range baseTitles {
titleVariations = append(titleVariations, t)
}
// Part variations
if part > 0 {
if len(folderTitle) > 0 {
for _, t := range baseTitles {
titleVariations = append(titleVariations,
buildTitle(folderTitle, "Part", strconv.Itoa(part)),
buildTitle(folderTitle, "Part", util.IntegerToOrdinal(part)),
buildTitle(folderTitle, "Cour", strconv.Itoa(part)),
buildTitle(folderTitle, "Cour", util.IntegerToOrdinal(part)),
buildTitle(t, "Part", strconv.Itoa(part)),
buildTitle(t, "Part", util.IntegerToOrdinal(part)),
buildTitle(t, "Cour", strconv.Itoa(part)),
)
}
if len(f.ParsedData.Title) > 0 {
titleVariations = append(titleVariations,
buildTitle(f.ParsedData.Title, "Part", strconv.Itoa(part)),
buildTitle(f.ParsedData.Title, "Part", util.IntegerToOrdinal(part)),
buildTitle(f.ParsedData.Title, "Cour", strconv.Itoa(part)),
buildTitle(f.ParsedData.Title, "Cour", util.IntegerToOrdinal(part)),
)
}
}
// Title, no seasons, no parts, or season 1
// e.g. "Bungou Stray Dogs"
// e.g. "Bungou Stray Dogs Season 1"
if noSeasonsOrParts || eitherSeasonFirst {
if len(f.ParsedData.Title) > 0 { // Add filename title
titleVariations = append(titleVariations, f.ParsedData.Title)
}
if len(folderTitle) > 0 { // Both titles are present and similar, add folder title
titleVariations = append(titleVariations, folderTitle)
}
}
// Part & Season
// e.g. "Spy x Family Season 1 Part 2"
if part > 0 && eitherSeason {
if len(folderTitle) > 0 {
if season > 0 {
titleVariations = append(titleVariations,
buildTitle(folderTitle, "Season", strconv.Itoa(season), "Part", strconv.Itoa(part)),
)
} else if folderSeason > 0 {
titleVariations = append(titleVariations,
buildTitle(folderTitle, "Season", strconv.Itoa(folderSeason), "Part", strconv.Itoa(part)),
)
}
}
if len(f.ParsedData.Title) > 0 {
if season > 0 {
titleVariations = append(titleVariations,
buildTitle(f.ParsedData.Title, "Season", strconv.Itoa(season), "Part", strconv.Itoa(part)),
)
} else if folderSeason > 0 {
titleVariations = append(titleVariations,
buildTitle(f.ParsedData.Title, "Season", strconv.Itoa(folderSeason), "Part", strconv.Itoa(part)),
)
// Roman numerals for parts 1-3
if partRoman := intToRoman(part); partRoman != "" {
titleVariations = append(titleVariations, buildTitle(t, "Part", partRoman))
}
}
}
// Season is present
// Season variations
if eitherSeason {
arr := make([]string, 0)
seas := folderSeason // Default to folder parsed season
if season > 0 { // Use filename parsed season if present
seas := folderSeason
if season > 0 {
seas = season
}
// Both titles are present
if bothTitles {
// Add both titles
arr = append(arr, f.ParsedData.Title)
arr = append(arr, folderTitle)
if !bothTitlesSimilar { // Combine both titles if they are not similar
arr = append(arr, fmt.Sprintf("%s %s", folderTitle, f.ParsedData.Title))
}
} else if len(folderTitle) > 0 { // Only folder title is present
arr = append(arr, folderTitle)
} else if len(f.ParsedData.Title) > 0 { // Only filename title is present
arr = append(arr, f.ParsedData.Title)
for _, t := range baseTitles {
// Standard formats
titleVariations = append(titleVariations,
buildTitle(t, "Season", strconv.Itoa(seas)), // "Title Season 2"
buildTitle(t, "S"+strconv.Itoa(seas)), // "Title S2"
buildTitle(t, fmt.Sprintf("S%02d", seas)), // "Title S02"
buildTitle(t, util.IntegerToOrdinal(seas), "Season"), // "Title 2nd Season"
fmt.Sprintf("%s %d", t, seas), // "Title 2" (common pattern)
)
}
for _, t := range arr {
titleVariations = append(titleVariations,
buildTitle(t, "Season", strconv.Itoa(seas)),
buildTitle(t, "S"+strconv.Itoa(seas)),
buildTitle(t, util.IntegerToOrdinal(seas), "Season"),
)
// Combined with part
if part > 0 {
for _, t := range baseTitles {
titleVariations = append(titleVariations,
buildTitle(t, "Season", strconv.Itoa(seas), "Part", strconv.Itoa(part)),
buildTitle(t, fmt.Sprintf("S%d", seas), fmt.Sprintf("Part %d", part)),
)
}
}
}
// Season 1 or no season info. base titles already added
if noSeasonsOrParts || eitherSeasonFirst {
// Already added base titles above
// For season 1, also add without the "Season 1" suffix as many first seasons
// don't have season indicators in their official titles
}
// Combined folder + filename title variations
// e.g. "Anime/S02/Episode.mkv"
if bothTitles && !bothTitlesSimilar {
combined := fmt.Sprintf("%s %s", folderTitle, f.ParsedData.Title)
titleVariations = append(titleVariations, combined)
}
// Deduplicate
titleVariations = lo.Uniq(titleVariations)
// If there are no title variations, use the folder title or the parsed title
// If there are still no title variations, fall back to raw titles
if len(titleVariations) == 0 {
if len(folderTitle) > 0 {
titleVariations = append(titleVariations, folderTitle)
@@ -464,5 +437,16 @@ func (f *LocalFile) GetTitleVariations() []*string {
}
return lo.ToSlicePtr(titleVariations)
}
// intToRoman converts small integers (1-10) to Roman numerals
func intToRoman(n int) string {
romans := map[int]string{
1: "I", 2: "II", 3: "III", 4: "IV", 5: "V",
6: "VI", 7: "VII", 8: "VIII", 9: "IX", 10: "X",
}
if r, ok := romans[n]; ok {
return r
}
return ""
}

89
internal/library/anime/localfile_helper_test.go
View File

@@ -1,14 +1,15 @@
package anime_test
import (
"github.com/davecgh/go-spew/spew"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"path/filepath"
"runtime"
"seanime/internal/library/anime"
"seanime/internal/util"
"strings"
"testing"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
)
func TestLocalFile_GetNormalizedPath(t *testing.T) {
@@ -19,12 +20,12 @@ func TestLocalFile_GetNormalizedPath(t *testing.T) {
expectedResult string
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
expectedResult: "e:/anime/bungou stray dogs 5th season/bungou stray dogs/[subsplease] bungou stray dogs - 61 (1080p) [f609b947].mkv",
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
expectedResult: "e:/anime/shakugan no shana/shakugan no shana i/opening/op01.mkv",
},
@@ -37,7 +38,7 @@ func TestLocalFile_GetNormalizedPath(t *testing.T) {
if assert.NotNil(t, lf) {
if assert.Equal(t, tt.expectedResult, lf.GetNormalizedPath()) {
spew.Dump(lf.GetNormalizedPath())
util.Spew(lf.GetNormalizedPath())
}
}
@@ -55,19 +56,19 @@ func TestLocalFile_IsInDir(t *testing.T) {
expectedResult bool
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Bungou Stray Dogs 5th Season",
expectedResult: true,
expectedResult: runtime.GOOS == "windows",
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Shakugan No Shana",
expectedResult: true,
expectedResult: runtime.GOOS == "windows",
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Shakugan No Shana I",
expectedResult: false,
@@ -81,7 +82,7 @@ func TestLocalFile_IsInDir(t *testing.T) {
if assert.NotNil(t, lf) {
if assert.Equal(t, tt.expectedResult, lf.IsInDir(tt.dir)) {
spew.Dump(lf.IsInDir(tt.dir))
util.Spew(lf.IsInDir(tt.dir))
}
}
@@ -99,22 +100,22 @@ func TestLocalFile_IsAtRootOf(t *testing.T) {
expectedResult bool
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Bungou Stray Dogs 5th Season",
expectedResult: false,
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Shakugan No Shana",
expectedResult: false,
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
dir: "E:/ANIME/Shakugan No Shana/Shakugan No Shana I/Opening",
expectedResult: true,
expectedResult: runtime.GOOS == "windows",
},
}
@@ -145,14 +146,14 @@ func TestLocalFile_Equals(t *testing.T) {
expectedResult bool
}{
{
filePath1: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath1: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath2: "E:/ANIME/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/Anime",
expectedResult: true,
expectedResult: runtime.GOOS == "windows",
},
{
filePath1: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath2: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 62 (1080p) [F609B947].mkv",
filePath1: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath2: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 62 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
expectedResult: false,
},
@@ -181,62 +182,52 @@ func TestLocalFile_GetTitleVariations(t *testing.T) {
expectedTitles []string
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Bungou Stray Dogs",
"Bungou Stray Dogs 5th Season",
"Bungou Stray Dogs Season 5",
"Bungou Stray Dogs S5",
"Bungou Stray Dogs S05",
"Bungou Stray Dogs 5",
},
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Shakugan No Shana I",
},
},
{
filePath: "E:\\ANIME\\Neon Genesis Evangelion Death & Rebirth\\[Anime Time] Neon Genesis Evangelion - Rebirth.mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Neon Genesis Evangelion - Rebirth",
"Neon Genesis Evangelion Death & Rebirth",
},
},
{
filePath: "E:\\ANIME\\Omoi, Omoware, Furi, Furare\\[GJM] Love Me, Love Me Not (BD 1080p) [841C23CD].mkv",
filePath: "E:/ANIME/Omoi, Omoware, Furi, Furare/[GJM] Love Me, Love Me Not (BD 1080p) [841C23CD].mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Love Me, Love Me Not",
"Omoi, Omoware, Furi, Furare",
"Omoi, Omoware, Furi, Furare Love Me, Love Me Not",
},
},
{
filePath: "E:\\ANIME\\Violet Evergarden Gaiden Eien to Jidou Shuki Ningyou\\Violet.Evergarden.Gaiden.2019.1080..Dual.Audio.BDRip.10.bits.DD.x265-EMBER.mkv",
filePath: "E:/ANIME/Violet Evergarden Gaiden Eien to Jidou Shuki Ningyou/Violet.Evergarden.Gaiden.2019.1080..Dual.Audio.BDRip.10.bits.DD.x265-EMBER.mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Violet Evergarden Gaiden Eien to Jidou Shuki Ningyou",
"Violet Evergarden Gaiden 2019",
"Violet Evergarden Gaiden Eien to Jidou Shuki Ningyou Violet Evergarden Gaiden 2019",
},
},
{
filePath: "E:\\ANIME\\Violet Evergarden S01+Movies+OVA 1080p Dual Audio BDRip 10 bits DD x265-EMBER\\01. Season 1 + OVA\\S01E01-'I Love You' and Auto Memory Dolls [F03E1F7A].mkv",
filePath: "E:/ANIME/Violet Evergarden S01+Movies+OVA 1080p Dual Audio BDRip 10 bits DD x265-EMBER/01. Season 1 + OVA/S01E01-'I Love You' and Auto Memory Dolls [F03E1F7A].mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Violet Evergarden",
"Violet Evergarden S1",
"Violet Evergarden Season 1",
"Violet Evergarden 1st Season",
},
},
{
filePath: "E:\\ANIME\\Golden Kamuy 4th Season\\[Judas] Golden Kamuy (Season 4) [1080p][HEVC x265 10bit][Multi-Subs]\\[Judas] Golden Kamuy - S04E01.mkv",
libraryPath: "E:/ANIME",
expectedTitles: []string{
"Golden Kamuy S4",
"Golden Kamuy Season 4",
"Golden Kamuy 4th Season",
"Violet Evergarden S01",
"Violet Evergarden 1",
},
},
}
@@ -249,7 +240,7 @@ func TestLocalFile_GetTitleVariations(t *testing.T) {
tv := lo.Map(lf.GetTitleVariations(), func(item *string, _ int) string { return *item })
if assert.ElementsMatch(t, tt.expectedTitles, tv) {
spew.Dump(lf.GetTitleVariations())
util.Spew(lf.GetTitleVariations())
}
}
@@ -266,12 +257,12 @@ func TestLocalFile_GetParsedTitle(t *testing.T) {
expectedParsedTitle string
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\Bungou Stray Dogs\\[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/Bungou Stray Dogs/[SubsPlease] Bungou Stray Dogs - 61 (1080p) [F609B947].mkv",
libraryPath: "E:/ANIME",
expectedParsedTitle: "Bungou Stray Dogs",
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
expectedParsedTitle: "Shakugan No Shana I",
},
@@ -284,7 +275,7 @@ func TestLocalFile_GetParsedTitle(t *testing.T) {
if assert.NotNil(t, lf) {
if assert.Equal(t, tt.expectedParsedTitle, lf.GetParsedTitle()) {
spew.Dump(lf.GetParsedTitle())
util.Spew(lf.GetParsedTitle())
}
}
@@ -301,12 +292,12 @@ func TestLocalFile_GetFolderTitle(t *testing.T) {
expectedFolderTitle string
}{
{
filePath: "E:\\Anime\\Bungou Stray Dogs 5th Season\\S05E11 - Episode Title.mkv",
filePath: "E:/Anime/Bungou Stray Dogs 5th Season/S05E11 - Episode Title.mkv",
libraryPath: "E:/ANIME",
expectedFolderTitle: "Bungou Stray Dogs",
},
{
filePath: "E:\\Anime\\Shakugan No Shana\\Shakugan No Shana I\\Opening\\OP01.mkv",
filePath: "E:/Anime/Shakugan No Shana/Shakugan No Shana I/Opening/OP01.mkv",
libraryPath: "E:/ANIME",
expectedFolderTitle: "Shakugan No Shana I",
},
@@ -319,7 +310,7 @@ func TestLocalFile_GetFolderTitle(t *testing.T) {
if assert.NotNil(t, lf) {
if assert.Equal(t, tt.expectedFolderTitle, lf.GetFolderTitle()) {
spew.Dump(lf.GetFolderTitle())
util.Spew(lf.GetFolderTitle())
}
}

277
internal/library/anime/normalized_media.go
View File

@@ -1,12 +1,58 @@
package anime
import (
"context"
"seanime/internal/api/anilist"
"seanime/internal/util/comparison"
"seanime/internal/util/limiter"
"seanime/internal/util/result"
"github.com/samber/lo"
)
type NormalizedMedia struct {
*anilist.BaseAnime
ID int
IdMal *int
Title *NormalizedMediaTitle
Synonyms []*string
Format *anilist.MediaFormat
Status *anilist.MediaStatus
Season *anilist.MediaSeason
Year *int
StartDate *NormalizedMediaDate
Episodes *int
BannerImage *string
CoverImage *NormalizedMediaCoverImage
//Relations *anilist.CompleteAnimeById_Media_CompleteAnime_Relations
NextAiringEpisode *NormalizedMediaNextAiringEpisode
// Whether it was fetched from AniList
fetched bool
}
type NormalizedMediaTitle struct {
Romaji *string
English *string
Native *string
UserPreferred *string
}
type NormalizedMediaDate struct {
Year *int
Month *int
Day *int
}
type NormalizedMediaCoverImage struct {
ExtraLarge *string
Large *string
Medium *string
Color *string
}
type NormalizedMediaNextAiringEpisode struct {
AiringAt int
TimeUntilAiring int
Episode int
}
type NormalizedMediaCache struct {
@@ -14,11 +60,236 @@ type NormalizedMediaCache struct {
}
func NewNormalizedMedia(m *anilist.BaseAnime) *NormalizedMedia {
return &NormalizedMedia{
BaseAnime: m,
var startDate *NormalizedMediaDate
if m.GetStartDate() != nil {
startDate = &NormalizedMediaDate{
Year: m.GetStartDate().GetYear(),
Month: m.GetStartDate().GetMonth(),
Day: m.GetStartDate().GetDay(),
}
}
var title *NormalizedMediaTitle
if m.GetTitle() != nil {
title = &NormalizedMediaTitle{
Romaji: m.GetTitle().GetRomaji(),
English: m.GetTitle().GetEnglish(),
Native: m.GetTitle().GetNative(),
UserPreferred: m.GetTitle().GetUserPreferred(),
}
}
var coverImage *NormalizedMediaCoverImage
if m.GetCoverImage() != nil {
coverImage = &NormalizedMediaCoverImage{
ExtraLarge: m.GetCoverImage().GetExtraLarge(),
Large: m.GetCoverImage().GetLarge(),
Medium: m.GetCoverImage().GetMedium(),
Color: m.GetCoverImage().GetColor(),
}
}
var nextAiringEpisode *NormalizedMediaNextAiringEpisode
if m.GetNextAiringEpisode() != nil {
nextAiringEpisode = &NormalizedMediaNextAiringEpisode{
AiringAt: m.GetNextAiringEpisode().GetAiringAt(),
TimeUntilAiring: m.GetNextAiringEpisode().GetTimeUntilAiring(),
Episode: m.GetNextAiringEpisode().GetEpisode(),
}
}
return &NormalizedMedia{
ID: m.GetID(),
IdMal: m.GetIDMal(),
Title: title,
Synonyms: m.GetSynonyms(),
Format: m.GetFormat(),
Status: m.GetStatus(),
Season: m.GetSeason(),
Year: m.GetSeasonYear(),
StartDate: startDate,
Episodes: m.GetEpisodes(),
BannerImage: m.GetBannerImage(),
CoverImage: coverImage,
NextAiringEpisode: nextAiringEpisode,
fetched: true,
}
}
// NewNormalizedMediaFromOfflineDB creates a NormalizedMedia from the anime-offline-database.
// The media is marked as not fetched (fetched=false) since it lacks some AniList-specific data.
func NewNormalizedMediaFromOfflineDB(
id int,
idMal *int,
title *NormalizedMediaTitle,
synonyms []*string,
format *anilist.MediaFormat,
status *anilist.MediaStatus,
season *anilist.MediaSeason,
year *int,
startDate *NormalizedMediaDate,
episodes *int,
coverImage *NormalizedMediaCoverImage,
) *NormalizedMedia {
return &NormalizedMedia{
ID: id,
IdMal: idMal,
Title: title,
Synonyms: synonyms,
Format: format,
Status: status,
Season: season,
Year: year,
StartDate: startDate,
Episodes: episodes,
CoverImage: coverImage,
fetched: false,
}
}
func FetchNormalizedMedia(anilistClient anilist.AnilistClient, l *limiter.Limiter, cache *anilist.CompleteAnimeCache, m *NormalizedMedia) error {
if anilistClient == nil || m == nil {
return nil
}
if m.fetched {
return nil
}
if cache != nil {
if complete, found := cache.Get(m.ID); found {
*m = *NewNormalizedMedia(complete.ToBaseAnime())
}
}
l.Wait()
complete, err := anilistClient.CompleteAnimeByID(context.Background(), &m.ID)
if err != nil {
return err
}
if cache != nil {
cache.Set(m.ID, complete.GetMedia())
}
*m = *NewNormalizedMedia(complete.GetMedia().ToBaseAnime())
m.fetched = true
return nil
}
func NewNormalizedMediaCache() *NormalizedMediaCache {
return &NormalizedMediaCache{result.NewCache[int, *NormalizedMedia]()}
}
// Helper methods
func (m *NormalizedMedia) GetTitleSafe() string {
if m.Title == nil {
return ""
}
if m.Title.UserPreferred != nil {
return *m.Title.UserPreferred
}
if m.Title.English != nil {
return *m.Title.English
}
if m.Title.Romaji != nil {
return *m.Title.Romaji
}
if m.Title.Native != nil {
return *m.Title.Native
}
return ""
}
func (m *NormalizedMedia) HasRomajiTitle() bool {
return m.Title != nil && m.Title.Romaji != nil
}
func (m *NormalizedMedia) HasEnglishTitle() bool {
return m.Title != nil && m.Title.English != nil
}
func (m *NormalizedMedia) HasSynonyms() bool {
return len(m.Synonyms) > 0
}
func (m *NormalizedMedia) GetAllTitles() []*string {
titles := make([]*string, 0)
if m.Title == nil {
return titles
}
if m.Title.Romaji != nil {
titles = append(titles, m.Title.Romaji)
}
if m.Title.English != nil {
titles = append(titles, m.Title.English)
}
if m.Title.Native != nil {
titles = append(titles, m.Title.Native)
}
if m.Title.UserPreferred != nil {
titles = append(titles, m.Title.UserPreferred)
}
titles = append(titles, m.Synonyms...)
return titles
}
// GetPossibleSeasonNumber returns the possible season number for that media and -1 if it doesn't have one.
// It looks at the synonyms and returns the highest season number found.
func (m *NormalizedMedia) GetPossibleSeasonNumber() int {
if m == nil || len(m.Synonyms) == 0 {
return -1
}
titles := lo.Filter(m.Synonyms, func(s *string, i int) bool { return comparison.ValueContainsSeason(*s) })
if m.HasEnglishTitle() {
titles = append(titles, m.Title.English)
}
if m.HasRomajiTitle() {
titles = append(titles, m.Title.Romaji)
}
seasons := lo.Map(titles, func(s *string, i int) int { return comparison.ExtractSeasonNumber(*s) })
return lo.Max(seasons)
}
func (m *NormalizedMedia) FetchMediaTree(
rel anilist.FetchMediaTreeRelation,
anilistClient anilist.AnilistClient,
rl *limiter.Limiter,
tree *anilist.CompleteAnimeRelationTree,
cache *anilist.CompleteAnimeCache,
) error {
if m == nil {
return nil
}
rl.Wait()
res, err := anilistClient.CompleteAnimeByID(context.Background(), &m.ID)
if err != nil {
return err
}
return res.GetMedia().FetchMediaTree(rel, anilistClient, rl, tree, cache)
}
// GetCurrentEpisodeCount returns the current episode number for that media and -1 if it doesn't have one.
// i.e. -1 is returned if the media has no episodes AND the next airing episode is not set.
func (m *NormalizedMedia) GetCurrentEpisodeCount() int {
ceil := -1
if m.Episodes != nil {
ceil = *m.Episodes
}
if m.NextAiringEpisode != nil {
if m.NextAiringEpisode.Episode > 0 {
ceil = m.NextAiringEpisode.Episode - 1
}
}
return ceil
}
// GetTotalEpisodeCount returns the total episode number for that media and -1 if it doesn't have one
func (m *NormalizedMedia) GetTotalEpisodeCount() int {
ceil := -1
if m.Episodes != nil {
ceil = *m.Episodes
}
return ceil
}

127
internal/library/scanner/efficient_dice.go Normal file
View File

@@ -0,0 +1,127 @@
package scanner
import (
"strings"
"sync"
)
// EfficientDice is Sorensen-Dice implementation that minimizes allocs
type EfficientDice struct {
ngramSize int
caseSensitive bool
ngramBufferA []uint64
ngramBufferB []uint64
ngramCountMapA map[uint64]int16
ngramCountMapB map[uint64]int16
}
var EfficientDicePool = sync.Pool{
New: func() interface{} {
return &EfficientDice{
ngramSize: 2,
caseSensitive: false,
ngramBufferA: make([]uint64, 0, 64),
ngramBufferB: make([]uint64, 0, 64),
ngramCountMapA: make(map[uint64]int16, 64),
ngramCountMapB: make(map[uint64]int16, 64),
}
},
}
func GetEfficientDice() *EfficientDice {
return EfficientDicePool.Get().(*EfficientDice)
}
func PutEfficientDice(d *EfficientDice) {
d.reset()
EfficientDicePool.Put(d)
}
// reset clears internal state for reuse
func (d *EfficientDice) reset() {
d.ngramBufferA = d.ngramBufferA[:0]
d.ngramBufferB = d.ngramBufferB[:0]
for k := range d.ngramCountMapA {
delete(d.ngramCountMapA, k)
}
for k := range d.ngramCountMapB {
delete(d.ngramCountMapB, k)
}
}
// Compare calculates the Sorensen-Dice coefficient between two strings
func (d *EfficientDice) Compare(a, b string) float64 {
if a == "" && b == "" {
return 1.0
}
if a == "" || b == "" {
return 0.0
}
// normalize case
if !d.caseSensitive {
a = strings.ToLower(a)
b = strings.ToLower(b)
}
// bigrams for both strings
d.generateNgrams(a, &d.ngramBufferA, d.ngramCountMapA)
d.generateNgrams(b, &d.ngramBufferB, d.ngramCountMapB)
if len(d.ngramBufferA) == 0 && len(d.ngramBufferB) == 0 {
return 1.0
}
if len(d.ngramBufferA) == 0 || len(d.ngramBufferB) == 0 {
return 0.0
}
// calculate intersection size
intersection := 0
for ngram, countA := range d.ngramCountMapA {
if countB, exists := d.ngramCountMapB[ngram]; exists {
// take minimum count for multiset intersection
if countA < countB {
intersection += int(countA)
} else {
intersection += int(countB)
}
}
}
// coefficient
totalA := len(d.ngramBufferA)
totalB := len(d.ngramBufferB)
return float64(2*intersection) / float64(totalA+totalB)
}
// generateNgrams generates bigrams from a string, stores them as uint64 hashes
func (d *EfficientDice) generateNgrams(s string, buffer *[]uint64, countMap map[uint64]int16) {
*buffer = (*buffer)[:0]
for k := range countMap {
delete(countMap, k)
}
runes := []rune(s)
if len(runes) < d.ngramSize {
// single character strings
if len(runes) == 1 {
hash := uint64(runes[0])
*buffer = append(*buffer, hash)
countMap[hash] = 1
}
return
}
for i := 0; i <= len(runes)-d.ngramSize; i++ {
// encode bigram as uint64: first rune in high 32 bits, second in low 32 bits
hash := (uint64(runes[i]) << 32) | uint64(runes[i+1])
*buffer = append(*buffer, hash)
countMap[hash]++
}
}
func CompareStrings(a, b string) float64 {
d := GetEfficientDice()
defer PutEfficientDice(d)
return d.Compare(a, b)
}

3
internal/library/scanner/hook_events.go
View File

@@ -1,7 +1,6 @@
package scanner
import (
"seanime/internal/api/anilist"
"seanime/internal/hook_resolver"
"seanime/internal/library/anime"
)
@@ -65,7 +64,7 @@ type ScanMediaFetcherStartedEvent struct {
type ScanMediaFetcherCompletedEvent struct {
hook_resolver.Event
// All media fetched from AniList, to be matched against the local files.
AllMedia []*anilist.CompleteAnime `json:"allMedia"`
AllMedia []*anime.NormalizedMedia `json:"allMedia"`
// Media IDs that are not in the user's collection.
UnknownMediaIds []int `json:"unknownMediaIds"`
}

8
internal/library/scanner/hydrator.go
View File

@@ -23,8 +23,9 @@ import (
// FileHydrator hydrates the metadata of all (matched) LocalFiles.
// LocalFiles should already have their media ID hydrated.
type FileHydrator struct {
LocalFiles []*anime.LocalFile // Local files to hydrate
AllMedia []*anime.NormalizedMedia // All media used to hydrate local files
LocalFiles []*anime.LocalFile // Local files to hydrate
AllMedia []*anime.NormalizedMedia // All media used to hydrate local files
// Used by media tree analysis
CompleteAnimeCache *anilist.CompleteAnimeCache
PlatformRef *util.Ref[platform.Platform]
MetadataProviderRef *util.Ref[metadata_provider.Provider]
@@ -108,6 +109,9 @@ func (fh *FileHydrator) hydrateGroupMetadata(
return
}
// Make sure the media is fetched
_ = anime.FetchNormalizedMedia(fh.PlatformRef.Get().GetAnilistClient(), fh.AnilistRateLimiter, fh.CompleteAnimeCache, media)
// Tree contains media relations
tree := anilist.NewCompleteAnimeRelationTree()
// Tree analysis used for episode normalization

11
internal/library/scanner/hydrator_test.go
View File

@@ -79,7 +79,7 @@ func TestFileHydrator_HydrateMetadata(t *testing.T) {
// +---------------------+
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: allMedia,
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
@@ -92,11 +92,10 @@ func TestFileHydrator_HydrateMetadata(t *testing.T) {
// +---------------------+
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
CompleteAnimeCache: nil,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
}
err = matcher.MatchLocalFilesWithMedia()

975
internal/library/scanner/matcher.go
View File

File diff suppressed because it is too large Load Diff

804
internal/library/scanner/matcher_test.go
View File

@@ -3,19 +3,21 @@ package scanner
import (
"context"
"seanime/internal/api/anilist"
"seanime/internal/api/metadata_provider"
"seanime/internal/database/db"
"seanime/internal/extension"
"seanime/internal/library/anime"
"seanime/internal/platforms/anilist_platform"
"seanime/internal/test_utils"
"seanime/internal/util"
"seanime/internal/util/limiter"
"testing"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
)
// Add more media to this file if needed
// scanner_test_mock_data.json
func TestMatcher_MatchLocalFileWithMedia(t *testing.T) {
func TestMatcher1(t *testing.T) {
anilistClient := anilist.TestGetMockAnilistClient()
animeCollection, err := anilistClient.AnimeCollectionWithRelations(context.Background(), nil)
@@ -69,7 +71,7 @@ func TestMatcher_MatchLocalFileWithMedia(t *testing.T) {
// +---------------------+
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: allMedia,
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
@@ -78,12 +80,12 @@ func TestMatcher_MatchLocalFileWithMedia(t *testing.T) {
// +---------------------+
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
CompleteAnimeCache: nil,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
Debug: true,
}
err = matcher.MatchLocalFilesWithMedia()
@@ -101,7 +103,7 @@ func TestMatcher_MatchLocalFileWithMedia(t *testing.T) {
}
func TestMatcher_MatchLocalFileWithMedia2(t *testing.T) {
func TestMatcher2(t *testing.T) {
test_utils.InitTestProvider(t, test_utils.Anilist())
anilistClient := anilist.NewAnilistClient(test_utils.ConfigData.Provider.AnilistJwt, "")
@@ -136,13 +138,13 @@ func TestMatcher_MatchLocalFileWithMedia2(t *testing.T) {
},
expectedMediaId: 21699,
},
{
name: "Demon Slayer: Kimetsu no Yaiba Entertainment District Arc - 142329",
paths: []string{
"E:/Anime/Kimetsu no Yaiba Yuukaku-hen/[Salieri] Demon Slayer - Kimetsu No Yaiba - S3 - Entertainment District - BD (1080P) (HDR) [Dual-Audio]/[Salieri] Demon Slayer S3 - Kimetsu No Yaiba- Entertainment District - 03 (1080P) (HDR) [Dual-Audio].mkv",
},
expectedMediaId: 142329,
},
//{
// name: "Demon Slayer: Kimetsu no Yaiba Entertainment District Arc - 142329",
// paths: []string{
// "E:/Anime/Kimetsu no Yaiba Yuukaku-hen/[Salieri] Demon Slayer - Kimetsu No Yaiba - S3 - Entertainment District - BD (1080P) (HDR) [Dual-Audio]/[Salieri] Demon Slayer S3 - Kimetsu No Yaiba- Entertainment District - 03 (1080P) (HDR) [Dual-Audio].mkv",
// },
// expectedMediaId: 142329, // mislabeled?
//},
{
name: "KnY 145139",
paths: []string{
@@ -211,7 +213,7 @@ func TestMatcher_MatchLocalFileWithMedia2(t *testing.T) {
// +---------------------+
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: allMedia,
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
@@ -220,12 +222,12 @@ func TestMatcher_MatchLocalFileWithMedia2(t *testing.T) {
// +---------------------+
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
CompleteAnimeCache: nil,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
Debug: true,
}
err = matcher.MatchLocalFilesWithMedia()
@@ -242,3 +244,753 @@ func TestMatcher_MatchLocalFileWithMedia2(t *testing.T) {
}
}
func TestMatcher3(t *testing.T) {
test_utils.InitTestProvider(t, test_utils.Anilist())
anilistClient := anilist.NewAnilistClient(test_utils.ConfigData.Provider.AnilistJwt, "")
animeCollection, err := anilistClient.AnimeCollectionWithRelations(context.Background(), &test_utils.ConfigData.Provider.AnilistUsername)
if err != nil {
t.Fatal(err.Error())
}
dir := "E:/Anime"
tests := []struct {
name string
paths []string
expectedMediaId int
// Optional ids of other media that should be in the collection to test conflict resolution
otherMediaIds []int
}{
{
name: "Frieren - Simple title matching - 154587",
paths: []string{
"E:/Anime/Frieren/Frieren - 01.mkv",
},
expectedMediaId: 154587,
},
{
name: "Jujutsu Kaisen Season 2 - Ordinal season format - 145064",
paths: []string{
"E:/Anime/Jujutsu Kaisen Season 2/[SubsPlease] Jujutsu Kaisen 2nd Season - 01 (1080p) [12345678].mkv",
},
expectedMediaId: 145064,
otherMediaIds: []int{113415},
},
{
name: "Dungeon Meshi - 153518",
paths: []string{
"E:/Anime/Dungeon Meshi/Dungeon Meshi - 01.mkv",
},
expectedMediaId: 153518,
},
{
name: "Violet Evergarden - 21827",
paths: []string{
"E:/Anime/Violet Evergarden/[SubsPlease] Violet Evergarden - 01 (1080p) [A1B2C3D4].mkv",
"E:/Anime/Violet Evergarden/[SubsPlease] Violet Evergarden - 02 (1080p) [E5F6G7H8].mkv",
},
expectedMediaId: 21827,
},
{
name: "Flying Witch - 21284",
paths: []string{
"E:/Anime/Flying Witch/[Erai-raws] Flying Witch - 01 [1080p][HEVC][Multiple Subtitle].mkv",
},
expectedMediaId: 21284,
},
{
name: "Durarara - 6746",
paths: []string{
"E:/Anime/Durarara/Durarara.S01E01.1080p.BluRay.x264-GROUP.mkv",
"E:/Anime/Durarara/Durarara.S01E02.1080p.BluRay.x264-GROUP.mkv",
},
expectedMediaId: 6746,
},
{
name: "HIGH CARD - 135778",
paths: []string{
"E:/Anime/HIGH CARD (01-12) [1080p] [Dual-Audio]/[ASW] HIGH CARD - 01 [1080p HEVC x265 10Bit][AAC].mkv",
},
expectedMediaId: 135778,
},
{
name: "Baccano - 2251",
paths: []string{
"E:/Anime/Baccano!/[Judas] Baccano! - S01E01.mkv",
"E:/Anime/Baccano!/[Judas] Baccano! - S01E05.mkv",
},
expectedMediaId: 2251,
},
{
name: "Kimi ni Todoke - 6045",
paths: []string{
"E:/Anime/Kimi ni Todoke/Kimi.ni.Todoke.S01.1080p.BluRay.10-Bit.Dual-Audio.FLAC.x265-YURASUKA/Kimi.ni.Todoke.S01E01.mkv",
},
expectedMediaId: 6045,
},
{
name: "Zom 100 - 159831",
paths: []string{
"E:/Anime/Zom 100/Zom.100.Bucket.List.of.the.Dead.S01.1080p.BluRay.Remux.Dual.Audio.x265-EMBER/S01E01-Zom 100 [12345678].mkv",
},
expectedMediaId: 159831,
},
{
name: "Kimi ni Todoke 2ND SEASON - 9656",
paths: []string{
"E:/Anime/Kimi ni Todoke 2ND SEASON/[SubsPlease] Kimi ni Todoke 2nd Season - 01 (1080p).mkv",
},
expectedMediaId: 9656,
otherMediaIds: []int{6045},
},
{
name: "Durarara!!x2 Shou - 20652",
paths: []string{
"E:/Anime/Durarara x2 Shou/[HorribleSubs] Durarara!! x2 Shou - 01 [1080p].mkv",
},
expectedMediaId: 20652,
otherMediaIds: []int{6746},
},
{
name: "HIGH CARD Season 2 - 163151",
paths: []string{
"E:/Anime/HIGH CARD Season 2/[SubsPlease] HIGH CARD Season 2 - 01 (1080p) [ABCD1234].mkv",
},
expectedMediaId: 163151,
otherMediaIds: []int{135778},
},
{
name: "86 EIGHTY-SIX Part 2 - 131586",
paths: []string{
"E:/Anime/86 Eighty-Six Part 2/[SubsPlease] 86 Eighty-Six Part 2 - 01 (1080p).mkv",
},
expectedMediaId: 131586,
otherMediaIds: []int{116589},
},
{
name: "Evangelion 1.0 - 2759",
paths: []string{
"E:/Anime/Evangelion Rebuild/Evangelion.1.0.You.Are.Not.Alone.2007.1080p.BluRay.x264-GROUP.mkv",
},
expectedMediaId: 2759,
},
{
name: "Evangelion 2.0 - 3784",
paths: []string{
"E:/Anime/Evangelion Rebuild/Evangelion.2.22.You.Can.Not.Advance.2009.1080p.BluRay.x265-GROUP.mkv",
},
expectedMediaId: 3784,
otherMediaIds: []int{2759, 3786}, // Include Eva 1.0 and Eva 3.0+1.0 for conflict testing
},
{
// One Piece Film Gold
name: "One Piece Film Gold - 21335",
paths: []string{
"E:/Anime/One Piece Movies/One.Piece.Film.Gold.2016.1080p.BluRay.x264-GROUP.mkv",
},
expectedMediaId: 21335,
},
{
name: "Violet Evergarden - 21827",
paths: []string{
"E:/Anime/Violet Evergarden/Season 01/Violet Evergarden - S01E01 - Episode Title.mkv",
},
expectedMediaId: 21827,
},
{
name: "Flying Witch (2016) - 21284",
paths: []string{
"E:/Anime/Flying Witch (2016)/Season 01/Flying Witch (2016) - S01E01 - Stone Seeker.mkv",
},
expectedMediaId: 21284,
},
{
name: "Baccano! with punctuation - 2251",
paths: []string{
"E:/Anime/Baccano!/Baccano! - 01 [BD 1080p] [5.1 Dual Audio].mkv",
},
expectedMediaId: 2251,
},
{
name: "86 - Eighty Six with dashes - 116589",
paths: []string{
"E:/Anime/86 - Eighty Six/86 - Eighty Six - 01 - Undertaker.mkv",
},
expectedMediaId: 116589,
},
{
name: "Evangelion 3.0+1.0 - 3786",
paths: []string{
"E:/Anime/Evangelion 3.0+1.0/Evangelion.3.0+1.0.Thrice.Upon.a.Time.2021.1080p.AMZN.WEB-DL.DDP5.1.x264-GROUP.mkv",
},
expectedMediaId: 3786,
},
{
name: "Insomniacs After School x265 - 143653",
paths: []string{
"E:/Anime/Kimi wa Houkago Insomnia/[ASW] Kimi wa Houkago Insomnia - 01 [1080p HEVC][AAC].mkv",
},
expectedMediaId: 143653,
},
{
name: "Kimi wa Houkago Insomnia 10bit - 143653",
paths: []string{
"E:/Anime/Insomniacs After School/Insomniacs.After.School.S01E01.1080p.WEB-DL.10bit.x265-GROUP.mkv",
},
expectedMediaId: 143653,
},
{
name: "One Piece Stampede WEB-DL - 105143",
paths: []string{
"E:/Anime/One Piece Movies/One.Piece.Stampede.2019.1080p.NF.WEB-DL.DDP5.1.H.264-GROUP.mkv",
},
expectedMediaId: 105143,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Add media to collection if it doesn't exist
allMedia := animeCollection.GetAllAnime()
// Helper to ensure media exists in collection
hasMedia := false
for _, media := range allMedia {
if media.ID == tt.expectedMediaId {
hasMedia = true
break
}
}
if !hasMedia {
anilist.TestAddAnimeCollectionWithRelationsEntry(animeCollection, tt.expectedMediaId, anilist.TestModifyAnimeCollectionEntryInput{Status: lo.ToPtr(anilist.MediaListStatusCurrent)}, anilistClient)
allMedia = animeCollection.GetAllAnime()
}
// Ensure other media exists
for _, id := range tt.otherMediaIds {
hasMedia := false
for _, media := range allMedia {
if media.ID == id {
hasMedia = true
break
}
}
if !hasMedia {
anilist.TestAddAnimeCollectionWithRelationsEntry(animeCollection, id, anilist.TestModifyAnimeCollectionEntryInput{Status: lo.ToPtr(anilist.MediaListStatusCurrent)}, anilistClient)
allMedia = animeCollection.GetAllAnime()
}
}
scanLogger, err := NewConsoleScanLogger()
if err != nil {
t.Fatal("expected result, got error:", err.Error())
}
// +---------------------+
// | Local Files |
// +---------------------+
var lfs []*anime.LocalFile
for _, path := range tt.paths {
lf := anime.NewLocalFile(path, dir)
lfs = append(lfs, lf)
}
// +---------------------+
// | MediaContainer |
// +---------------------+
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
// +---------------------+
// | Matcher |
// +---------------------+
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
}
err = matcher.MatchLocalFilesWithMedia()
if assert.NoError(t, err, "Error while matching local files") {
for _, lf := range lfs {
if lf.MediaId != tt.expectedMediaId {
t.Errorf("FAILED: expected media id %d, got %d for file %s", tt.expectedMediaId, lf.MediaId, lf.Name)
} else {
t.Logf("SUCCESS: local file: %s -> media id: %d", lf.Name, lf.MediaId)
}
}
}
})
}
}
// TestMatcher4 tests complex scenarios like abbreviated titles,
// series with multiple seasons/parts, and special characters.
func TestMatcher4(t *testing.T) {
test_utils.InitTestProvider(t, test_utils.Anilist())
anilistClient := anilist.NewAnilistClient(test_utils.ConfigData.Provider.AnilistJwt, "")
animeCollection, err := anilistClient.AnimeCollectionWithRelations(context.Background(), &test_utils.ConfigData.Provider.AnilistUsername)
if err != nil {
t.Fatal(err.Error())
}
dir := "E:/Anime"
tests := []struct {
name string
paths []string
expectedMediaId int
otherMediaIds []int
}{
// Abbreviated titles
{
name: "Bunny Girl Senpai abbreviated - 101291",
paths: []string{
"E:/Anime/Bunny Girl Senpai/[SubsPlease] Bunny Girl Senpai - 01 (1080p).mkv",
},
expectedMediaId: 101291,
},
{
// Romaji title
name: "Seishun Buta Yarou full title - 101291",
paths: []string{
"E:/Anime/Seishun Buta Yarou/Seishun.Buta.Yarou.wa.Bunny.Girl.Senpai.no.Yume.wo.Minai.S01E01.1080p.BluRay.x264.mkv",
},
expectedMediaId: 101291,
},
// Mushoku Tensei parts/seasons
{
name: "Mushoku Tensei S2 - 146065",
paths: []string{
"E:/Anime/Mushoku Tensei S2/[SubsPlease] Mushoku Tensei S2 - 01 (1080p) [EC64C8B1].mkv",
},
expectedMediaId: 146065,
otherMediaIds: []int{108465, 127720, 166873}, // Part 1, Cour 2, Season 2 Part 2
},
{
// Season 2 Part 2 (Erai-raws)
name: "Mushoku Tensei II Part 2 Erai-raws - 166873",
paths: []string{
"E:/Anime/Mushoku Tensei II Part 2/[Erai-raws] Mushoku Tensei II Part 2 - 06 [1080p][HEVC][Multiple Subtitle][7509990E].mkv",
},
expectedMediaId: 166873, // Season 2 Part 2
otherMediaIds: []int{108465, 146065},
},
{
// Jobless Reincarnation (English)
name: "Jobless Reincarnation S2 - 146065",
paths: []string{
"E:/Anime/Jobless Reincarnation/Mushoku.Tensei.Jobless.Reincarnation.S02E01.1080p.CR.WEB-DL.x264.mkv",
},
expectedMediaId: 146065,
otherMediaIds: []int{108465},
},
// Bungo Stray Dogs seasons
{
name: "Bungou Stray Dogs S1 - 21311",
paths: []string{
"E:/Anime/Bungou Stray Dogs/[Judas] Bungo Stray Dogs - S01E01.mkv",
},
expectedMediaId: 21311,
otherMediaIds: []int{21679}, // S2
},
{
name: "Bungou Stray Dogs S2 - 21679",
paths: []string{
"E:/Anime/Bungou Stray Dogs 2nd Season/Bungou.Stray.Dogs.S02E01.1080p.BluRay.x264-GROUP.mkv",
},
expectedMediaId: 21679,
otherMediaIds: []int{21311}, // S1
},
{
name: "BSD 5th Season abbreviated - 163263",
paths: []string{
"E:/Anime/BSD S5/[SubsPlease] Bungou Stray Dogs 5th Season - 01 (1080p).mkv",
},
expectedMediaId: 163263,
otherMediaIds: []int{21311, 21679}, // S1, S2
},
// Golden Kamuy
{
name: "Golden Kamuy S3 - 110355",
paths: []string{
"E:/Anime/Golden Kamuy 3rd Season/Golden.Kamuy.S03E01.1080p.WEB-DL.x264.mkv",
},
expectedMediaId: 110355,
otherMediaIds: []int{102977}, // S2
},
// Blue Lock
{
name: "Blue Lock S1 - 137822",
paths: []string{
"E:/Anime/Blue Lock/[SubsPlease] Blue Lock - 01 (1080p).mkv",
},
expectedMediaId: 137822,
otherMediaIds: []int{163146}, // S2
},
{
name: "Blue Lock 2nd Season - 163146",
paths: []string{
"E:/Anime/Blue Lock 2nd Season/[SubsPlease] Blue Lock 2nd Season - 01 (1080p) [HASH].mkv",
},
expectedMediaId: 163146,
otherMediaIds: []int{137822}, // S1
},
{
name: "Violet Evergarden Gaiden - 109190",
paths: []string{
"E:/Anime/Violet Evergarden Gaiden/Violet.Evergarden.Eternity.and.the.Auto.Memory.Doll.2019.1080p.BluRay.x264.mkv",
},
expectedMediaId: 109190,
otherMediaIds: []int{21827}, // Main series
},
{
name: "Zom 100 short name - 159831",
paths: []string{
"E:/Anime/Zom 100/[ASW] Zom 100 - 01 [1080p HEVC].mkv",
},
expectedMediaId: 159831,
},
{
name: "Insomniacs main series not special - 143653",
paths: []string{
"E:/Anime/Kimi wa Houkago Insomnia/[Erai-raws] Kimi wa Houkago Insomnia - 01 [1080p].mkv",
},
expectedMediaId: 143653,
otherMediaIds: []int{160205}, // Special Animation PV
},
{
name: "Kekkai Sensen - 20727",
paths: []string{
"E:/Anime/[Anime Time] Kekkai Sensen (Blood Blockade Battlefront) S01+02+OVA+Extra [Dual Audio][BD][1080p][HEVC 10bit x265][AAC][Eng Sub]/Blood Blockade Battlefront/NC/Blood Blockade Battlefront - NCED.mkv",
},
expectedMediaId: 20727,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
allMedia := animeCollection.GetAllAnime()
hasMedia := false
for _, media := range allMedia {
if media.ID == tt.expectedMediaId {
hasMedia = true
break
}
}
if !hasMedia {
anilist.TestAddAnimeCollectionWithRelationsEntry(animeCollection, tt.expectedMediaId, anilist.TestModifyAnimeCollectionEntryInput{Status: lo.ToPtr(anilist.MediaListStatusCurrent)}, anilistClient)
allMedia = animeCollection.GetAllAnime()
}
for _, id := range tt.otherMediaIds {
hasMedia := false
for _, media := range allMedia {
if media.ID == id {
hasMedia = true
break
}
}
if !hasMedia {
anilist.TestAddAnimeCollectionWithRelationsEntry(animeCollection, id, anilist.TestModifyAnimeCollectionEntryInput{Status: lo.ToPtr(anilist.MediaListStatusCurrent)}, anilistClient)
allMedia = animeCollection.GetAllAnime()
}
}
scanLogger, err := NewConsoleScanLogger()
if err != nil {
t.Fatal("expected result, got error:", err.Error())
}
var lfs []*anime.LocalFile
for _, path := range tt.paths {
lf := anime.NewLocalFile(path, dir)
lfs = append(lfs, lf)
}
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
Debug: true,
}
err = matcher.MatchLocalFilesWithMedia()
if assert.NoError(t, err, "Error while matching local files") {
for _, lf := range lfs {
if lf.MediaId != tt.expectedMediaId {
t.Errorf("FAILED: expected media id %d, got %d for file %s", tt.expectedMediaId, lf.MediaId, lf.Name)
} else {
t.Logf("SUCCESS: local file: %s -> media id: %d", lf.Name, lf.MediaId)
}
}
}
})
}
}
// TestMatcherWithOfflineDB tests matching using the anime-offline-database.
// MediaFetcher is initialized with DisableAnimeCollection=true and Enhanced=true.
func TestMatcherWithOfflineDB(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test")
}
test_utils.InitTestProvider(t, test_utils.Anilist())
anilistClient := anilist.TestGetMockAnilistClient()
logger := util.NewLogger()
database, err := db.NewDatabase(test_utils.ConfigData.Path.DataDir, test_utils.ConfigData.Database.Name, logger)
if err != nil {
t.Fatal(err)
}
anilistClientRef := util.NewRef(anilistClient)
extensionBankRef := util.NewRef(extension.NewUnifiedBank())
anilistPlatform := anilist_platform.NewAnilistPlatform(anilistClientRef, extensionBankRef, logger, database)
anilistPlatform.SetUsername(test_utils.ConfigData.Provider.AnilistUsername)
metadataProvider := metadata_provider.GetFakeProvider(t, database)
completeAnimeCache := anilist.NewCompleteAnimeCache()
anilistRateLimiter := limiter.NewAnilistLimiter()
scanLogger, err := NewConsoleScanLogger()
if err != nil {
t.Fatal("expected result, got error:", err.Error())
}
dir := "E:/Anime"
t.Log("Initializing MediaFetcher with anime-offline-database...")
mf, err := NewMediaFetcher(t.Context(), &MediaFetcherOptions{
Enhanced: true,
EnhanceWithOfflineDatabase: true, // Use offline database
PlatformRef: util.NewRef(anilistPlatform),
LocalFiles: []*anime.LocalFile{}, // Empty, we don't need local files for fetching
CompleteAnimeCache: completeAnimeCache,
MetadataProviderRef: util.NewRef(metadataProvider),
Logger: logger,
AnilistRateLimiter: anilistRateLimiter,
ScanLogger: scanLogger,
DisableAnimeCollection: true, // Only use offline database
})
if err != nil {
t.Fatal("Failed to create MediaFetcher:", err.Error())
}
t.Logf("MediaFetcher initialized with %d media entries", len(mf.AllMedia))
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: mf.AllMedia,
ScanLogger: scanLogger,
})
tests := []struct {
name string
paths []string
expectedMediaId int
}{
{
name: "Death Note - 1535",
paths: []string{
"E:/Anime/Death Note/[SubsPlease] Death Note - 01 (1080p).mkv",
"E:/Anime/Death Note/[SubsPlease] Death Note - 02 (1080p).mkv",
},
expectedMediaId: 1535,
},
{
name: "Fullmetal Alchemist Brotherhood - 5114",
paths: []string{
"E:/Anime/Fullmetal Alchemist Brotherhood/[HorribleSubs] Fullmetal Alchemist Brotherhood - 01 [1080p].mkv",
},
expectedMediaId: 5114,
},
{
name: "Attack on Titan S1 - 16498",
paths: []string{
"E:/Anime/Attack on Titan/Shingeki.no.Kyojin.S01E01.1080p.BluRay.x264.mkv",
},
expectedMediaId: 16498,
},
{
name: "Demon Slayer S1 - 101922",
paths: []string{
"E:/Anime/Kimetsu no Yaiba/[SubsPlease] Kimetsu no Yaiba - 01 (1080p).mkv",
},
expectedMediaId: 101922,
},
{
name: "Jujutsu Kaisen S1 - 113415",
paths: []string{
"E:/Anime/Jujutsu Kaisen/[SubsPlease] Jujutsu Kaisen - 01 (1080p).mkv",
},
expectedMediaId: 113415,
},
{
name: "Spy x Family S1 - 140960",
paths: []string{
"E:/Anime/Spy x Family/[SubsPlease] Spy x Family - 01 (1080p).mkv",
},
expectedMediaId: 140960,
},
{
name: "One Punch Man S1 - 21087",
paths: []string{
"E:/Anime/One Punch Man/[HorribleSubs] One Punch Man - 01 [1080p].mkv",
},
expectedMediaId: 21087,
},
{
name: "My Hero Academia S1 - 21459",
paths: []string{
"E:/Anime/Boku no Hero Academia/[SubsPlease] Boku no Hero Academia - 01 (1080p).mkv",
},
expectedMediaId: 21459,
},
{
name: "Spirited Away - 199",
paths: []string{
"E:/Anime/Spirited Away/Spirited.Away.2001.1080p.BluRay.x264.mkv",
},
expectedMediaId: 199,
},
{
name: "Your Name - 21519",
paths: []string{
"E:/Anime/Your Name/Kimi.no.Na.wa.2016.1080p.BluRay.x264.mkv",
},
expectedMediaId: 21519,
},
{
name: "Steins Gate - 9253",
paths: []string{
"E:/Anime/Steins Gate/Steins.Gate.S01E01.1080p.BluRay.x264.mkv",
},
expectedMediaId: 9253,
},
{
name: "Re Zero S1 - 21355",
paths: []string{
"E:/Anime/Re Zero/[SubsPlease] Re Zero kara Hajimeru Isekai Seikatsu - 01 (1080p).mkv",
},
expectedMediaId: 21355,
},
{
name: "Mob Psycho 100 S1 - 21507",
paths: []string{
"E:/Anime/Mob Psycho 100/[HorribleSubs] Mob Psycho 100 - 01 [1080p].mkv",
},
expectedMediaId: 21507,
},
{
name: "Chainsaw Man - 127230",
paths: []string{
"E:/Anime/Chainsaw Man/[SubsPlease] Chainsaw Man - 01 (1080p).mkv",
},
expectedMediaId: 127230,
},
{
name: "KonoSuba S1 - 21202",
paths: []string{
"E:/Anime/KonoSuba/[HorribleSubs] Kono Subarashii Sekai ni Shukufuku wo! - 01 [1080p].mkv",
},
expectedMediaId: 21202,
},
{
name: "FMAB alternate name - 5114",
paths: []string{
"E:/Anime/FMAB/FMAB.S01E01.1080p.BluRay.x264.mkv",
},
expectedMediaId: 5114,
},
{
name: "Kekkai Sensen - 20727",
paths: []string{
"E:/Anime/[Anime Time] Kekkai Sensen (Blood Blockade Battlefront) S01+02+OVA+Extra [Dual Audio][BD][1080p][HEVC 10bit x265][AAC][Eng Sub]/Blood Blockade Battlefront/NC/Blood Blockade Battlefront - NCED.mkv",
},
expectedMediaId: 20727,
},
{
name: "ACCA 13-ku Kansatsu-ka - 21823",
paths: []string{
"E:/Anime/ACCA 13-ku Kansatsu-ka/[Judas] ACCA 13-ku Kansatsu-ka (Season 1) [BD 1080p][HEVC x265 10bit][Dual-Audio][Eng-Subs]/Extras/[Judas] ACCA 13-ku Kansatsu-ka - Ending.mkv",
},
expectedMediaId: 21823,
},
{
name: "Akebi-chan no Sailor Fuku - 131548",
paths: []string{
"E:/Anime/Akebi-chan no Sailor Fuku/[Anime Time] Akebi-chan no Sailor Fuku - 01 [1080p][HEVC 10bit x265][AAC][Multi Sub].mkv",
},
expectedMediaId: 131548,
},
{
name: "Pluto - 99088",
paths: []string{
"E:/Anime/PLUTO/Pluto S01 1080p Dual Audio WEBRip DD+ x265-EMBER/S01E01-Episode 1 [59596368].mkv",
},
expectedMediaId: 99088,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create local files for this test case
var lfs []*anime.LocalFile
for _, path := range tt.paths {
lf := anime.NewLocalFile(path, dir)
lfs = append(lfs, lf)
}
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
Logger: logger,
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
Debug: true,
}
err := matcher.MatchLocalFilesWithMedia()
if err != nil {
t.Fatal("Error while matching:", err.Error())
}
for _, lf := range lfs {
if lf.MediaId == tt.expectedMediaId {
t.Logf("SUCCESS: %s -> media id: %d", lf.Name, lf.MediaId)
} else if lf.MediaId == 0 {
t.Errorf("UNMATCHED: %s (expected %d)", lf.Name, tt.expectedMediaId)
} else {
t.Errorf("WRONG MATCH: %s -> got %d, expected %d", lf.Name, lf.MediaId, tt.expectedMediaId)
}
}
})
}
}

165
internal/library/scanner/media_container.go
View File

@@ -1,115 +1,130 @@
package scanner
import (
"github.com/rs/zerolog"
"github.com/samber/lo"
lop "github.com/samber/lo/parallel"
"seanime/internal/api/anilist"
"seanime/internal/library/anime"
"seanime/internal/util/comparison"
"strings"
"github.com/rs/zerolog"
"github.com/samber/lo"
)
type (
MediaContainerOptions struct {
AllMedia []*anilist.CompleteAnime
AllMedia []*anime.NormalizedMedia
ScanLogger *ScanLogger
}
// MediaContainer holds all the NormalizedMedia that will be used by the Matcher.
// It creates an inverted index for fast candidate lookup based on title tokens.
// Note: It doesn't care that the NormalizedMedia are not fully fetched.
// Before v3.5, it was used to flatten relations into NormalizedMedia.
MediaContainer struct {
NormalizedMedia []*anime.NormalizedMedia
ScanLogger *ScanLogger
engTitles []*string
romTitles []*string
synonyms []*string
allMedia []*anilist.CompleteAnime
NormalizedMedia []*anime.NormalizedMedia
NormalizedTitlesCache map[int][]*NormalizedTitle // mediaId -> normalized titles
ScanLogger *ScanLogger
// Inverted Index for fast candidate lookup
// Token -> media that contain this token in their title
TokenIndex map[string][]*anime.NormalizedMedia
engTitles []*string // legacy
romTitles []*string // legacy
synonyms []*string // legacy
}
)
// NewMediaContainer will create a list of all English titles, Romaji titles, and synonyms from all anilist.BaseAnime (used by Matcher).
//
// The list will include all anilist.BaseAnime and their relations (prequels, sequels, spin-offs, etc...) as NormalizedMedia.
//
// It also provides helper functions to get a NormalizedMedia from a title or synonym (used by FileHydrator).
// NewMediaContainer creates a new MediaContainer from a list of NormalizedMedia that will be used by the Matcher.
func NewMediaContainer(opts *MediaContainerOptions) *MediaContainer {
mc := new(MediaContainer)
mc.ScanLogger = opts.ScanLogger
mc.NormalizedMedia = make([]*anime.NormalizedMedia, 0)
mc.NormalizedMedia = opts.AllMedia
normalizedMediaMap := make(map[int]*anime.NormalizedMedia)
// pre-compute normalized titles for all media
mc.NormalizedTitlesCache = make(map[int][]*NormalizedTitle, len(mc.NormalizedMedia))
for _, m := range opts.AllMedia {
normalizedMediaMap[m.ID] = anime.NewNormalizedMedia(m.ToBaseAnime())
if m.Relations != nil && m.Relations.Edges != nil && len(m.Relations.Edges) > 0 {
for _, edgeM := range m.Relations.Edges {
if edgeM.Node == nil || edgeM.Node.Format == nil || edgeM.RelationType == nil {
continue
// Initialize token index
mc.TokenIndex = make(map[string][]*anime.NormalizedMedia)
for _, m := range mc.NormalizedMedia {
normalized := make([]*NormalizedTitle, 0)
// Keep track of which tokens this media has been added to to avoid duplicates
seenTokens := make(map[string]struct{})
addTitle := func(t *string, isMain bool) {
if t != nil && *t != "" {
norm := NormalizeTitle(*t)
norm.IsMain = isMain
normalized = append(normalized, norm)
// Populate index
tokens := GetSignificantTokens(norm.Tokens)
for _, token := range tokens {
if _, ok := seenTokens[token]; !ok {
mc.TokenIndex[token] = append(mc.TokenIndex[token], m)
seenTokens[token] = struct{}{}
}
}
if *edgeM.Node.Format != anilist.MediaFormatMovie &&
*edgeM.Node.Format != anilist.MediaFormatOva &&
*edgeM.Node.Format != anilist.MediaFormatSpecial &&
*edgeM.Node.Format != anilist.MediaFormatTv {
continue
}
if *edgeM.RelationType != anilist.MediaRelationPrequel &&
*edgeM.RelationType != anilist.MediaRelationSequel &&
*edgeM.RelationType != anilist.MediaRelationSpinOff &&
*edgeM.RelationType != anilist.MediaRelationAlternative &&
*edgeM.RelationType != anilist.MediaRelationParent {
continue
}
// DEVNOTE: Edges fetched from the AniList AnimeCollection query do not contain NextAiringEpisode
// Make sure we don't overwrite the original media in the map that contains NextAiringEpisode
if _, found := normalizedMediaMap[edgeM.Node.ID]; !found {
normalizedMediaMap[edgeM.Node.ID] = anime.NewNormalizedMedia(edgeM.Node)
}
}
if m.Title != nil {
addTitle(m.Title.Romaji, true)
addTitle(m.Title.English, true)
addTitle(m.Title.Native, false)
addTitle(m.Title.UserPreferred, true)
}
if m.Synonyms != nil {
for _, syn := range m.Synonyms {
addTitle(syn, false)
}
}
mc.NormalizedTitlesCache[m.ID] = normalized
seenTokens = nil
}
// ------------------------------------------
// Legacy stuff
engTitles := make([]*string, 0, len(mc.NormalizedMedia))
romTitles := make([]*string, 0, len(mc.NormalizedMedia))
synonymsSlice := make([]*string, 0, len(mc.NormalizedMedia)*2)
for _, m := range mc.NormalizedMedia {
if m.Title.English != nil && len(*m.Title.English) > 0 {
engTitles = append(engTitles, m.Title.English)
}
if m.Title.Romaji != nil && len(*m.Title.Romaji) > 0 {
romTitles = append(romTitles, m.Title.Romaji)
}
if m.Synonyms != nil {
for _, syn := range m.Synonyms {
if syn != nil && comparison.ValueContainsSeason(*syn) {
synonymsSlice = append(synonymsSlice, syn)
}
}
}
}
for _, m := range normalizedMediaMap {
mc.NormalizedMedia = append(mc.NormalizedMedia, m)
}
engTitles := lop.Map(mc.NormalizedMedia, func(m *anime.NormalizedMedia, index int) *string {
if m.Title.English != nil {
return m.Title.English
}
return new(string)
})
romTitles := lop.Map(mc.NormalizedMedia, func(m *anime.NormalizedMedia, index int) *string {
if m.Title.Romaji != nil {
return m.Title.Romaji
}
return new(string)
})
_synonymsArr := lop.Map(mc.NormalizedMedia, func(m *anime.NormalizedMedia, index int) []*string {
if m.Synonyms != nil {
return m.Synonyms
}
return make([]*string, 0)
})
synonyms := lo.Flatten(_synonymsArr)
engTitles = lo.Filter(engTitles, func(s *string, i int) bool { return s != nil && len(*s) > 0 })
romTitles = lo.Filter(romTitles, func(s *string, i int) bool { return s != nil && len(*s) > 0 })
synonyms = lo.Filter(synonyms, func(s *string, i int) bool { return comparison.ValueContainsSeason(*s) })
mc.engTitles = engTitles
mc.romTitles = romTitles
mc.synonyms = synonyms
mc.allMedia = opts.AllMedia
mc.synonyms = synonymsSlice
// ------------------------------------------
if mc.ScanLogger != nil {
mc.ScanLogger.LogMediaContainer(zerolog.InfoLevel).
Any("inputCount", len(opts.AllMedia)).
Any("mediaCount", len(mc.NormalizedMedia)).
Any("titles", len(mc.engTitles)+len(mc.romTitles)+len(mc.synonyms)).
Any("indexSize", len(mc.TokenIndex)).
Msg("Created media container")
}
return mc
}
// Legacy helper function
func (mc *MediaContainer) GetMediaFromTitleOrSynonym(title *string) (*anime.NormalizedMedia, bool) {
if title == nil {
return nil, false
@@ -134,13 +149,3 @@ func (mc *MediaContainer) GetMediaFromTitleOrSynonym(title *string) (*anime.Norm
return res, found
}
func (mc *MediaContainer) GetMediaFromId(id int) (*anime.NormalizedMedia, bool) {
res, found := lo.Find(mc.NormalizedMedia, func(m *anime.NormalizedMedia) bool {
if m.ID == id {
return true
}
return false
})
return res, found
}

133
internal/library/scanner/media_fetcher.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"seanime/internal/api/anilist"
"seanime/internal/api/animeofflinedb"
"seanime/internal/api/mal"
"seanime/internal/api/metadata"
"seanime/internal/api/metadata_provider"
@@ -23,7 +24,7 @@ import (
// MediaFetcher holds all anilist.BaseAnime that will be used for the comparison process
type MediaFetcher struct {
AllMedia []*anilist.CompleteAnime
AllMedia []*anime.NormalizedMedia
CollectionMediaIds []int
UnknownMediaIds []int // Media IDs that are not in the user's collection
AnimeCollectionWithRelations *anilist.AnimeCollectionWithRelations
@@ -31,15 +32,16 @@ type MediaFetcher struct {
}
type MediaFetcherOptions struct {
Enhanced bool
PlatformRef *util.Ref[platform.Platform]
MetadataProviderRef *util.Ref[metadata_provider.Provider]
LocalFiles []*anime.LocalFile
CompleteAnimeCache *anilist.CompleteAnimeCache
Logger *zerolog.Logger
AnilistRateLimiter *limiter.Limiter
DisableAnimeCollection bool
ScanLogger *ScanLogger
Enhanced bool
EnhanceWithOfflineDatabase bool
PlatformRef *util.Ref[platform.Platform]
MetadataProviderRef *util.Ref[metadata_provider.Provider]
LocalFiles []*anime.LocalFile
CompleteAnimeCache *anilist.CompleteAnimeCache
Logger *zerolog.Logger
AnilistRateLimiter *limiter.Limiter
DisableAnimeCollection bool
ScanLogger *ScanLogger
}
// NewMediaFetcher
@@ -89,13 +91,14 @@ func NewMediaFetcher(ctx context.Context, opts *MediaFetcherOptions) (ret *Media
mf.AnimeCollectionWithRelations = animeCollectionWithRelations
mf.AllMedia = make([]*anilist.CompleteAnime, 0)
// Temporary slice to hold CompleteAnime before conversion
allCompleteAnime := make([]*anilist.CompleteAnime, 0)
if !opts.DisableAnimeCollection {
// For each collection entry, append the media to AllMedia
for _, list := range animeCollectionWithRelations.GetMediaListCollection().GetLists() {
for _, entry := range list.GetEntries() {
mf.AllMedia = append(mf.AllMedia, entry.GetMedia())
allCompleteAnime = append(allCompleteAnime, entry.GetMedia())
// +---------------------+
// | Cache |
@@ -108,25 +111,25 @@ func NewMediaFetcher(ctx context.Context, opts *MediaFetcherOptions) (ret *Media
if mf.ScanLogger != nil {
mf.ScanLogger.LogMediaFetcher(zerolog.DebugLevel).
Int("count", len(mf.AllMedia)).
Int("count", len(allCompleteAnime)).
Msg("Fetched media from AniList collection")
}
//--------------------------------------------
// Get the media IDs from the collection
mf.CollectionMediaIds = lop.Map(mf.AllMedia, func(m *anilist.CompleteAnime, index int) int {
mf.CollectionMediaIds = lop.Map(allCompleteAnime, func(m *anilist.CompleteAnime, index int) int {
return m.ID
})
//--------------------------------------------
// +---------------------+
// | Enhanced |
// | Enhanced (Legacy) |
// +---------------------+
// If enhancing is on, scan media from local files and get their relations
if opts.Enhanced {
// If enhancing (legacy) is on, scan media from local files and get their relations
if opts.Enhanced && !opts.EnhanceWithOfflineDatabase {
_, ok := FetchMediaFromLocalFiles(
ctx,
@@ -138,27 +141,66 @@ func NewMediaFetcher(ctx context.Context, opts *MediaFetcherOptions) (ret *Media
mf.ScanLogger,
)
if ok {
// We assume the CompleteAnimeCache is populated. We overwrite AllMedia with the cache content.
// This is because the cache will contain all media from the user's collection AND scanned ones
mf.AllMedia = make([]*anilist.CompleteAnime, 0)
// We assume the CompleteAnimeCache is populated.
// Safe to overwrite allCompleteAnime with the cache content
// because the cache will contain all media from the user's collection AND scanned ones
allCompleteAnime = make([]*anilist.CompleteAnime, 0)
opts.CompleteAnimeCache.Range(func(key int, value *anilist.CompleteAnime) bool {
mf.AllMedia = append(mf.AllMedia, value)
allCompleteAnime = append(allCompleteAnime, value)
return true
})
}
}
mf.AllMedia = NormalizedMediaFromAnilistComplete(allCompleteAnime)
// +-------------------------+
// | Enhanced (Offline DB) |
// +-------------------------+
// When enhanced mode is on, fetch anime-offline-database to provide more matching candidates
if opts.Enhanced && opts.EnhanceWithOfflineDatabase {
if mf.ScanLogger != nil {
mf.ScanLogger.LogMediaFetcher(zerolog.DebugLevel).
Msg("Fetching anime-offline-database for enhanced matching")
}
// build existing media IDs map for filtering
existingMediaIDs := make(map[int]bool, len(mf.AllMedia))
for _, m := range mf.AllMedia {
existingMediaIDs[m.ID] = true
}
offlineMedia, err := animeofflinedb.FetchAndConvertDatabase(existingMediaIDs)
if err != nil {
if mf.ScanLogger != nil {
mf.ScanLogger.LogMediaFetcher(zerolog.WarnLevel).
Err(err).
Msg("Failed to fetch anime-offline-database, continuing without it")
}
} else {
if mf.ScanLogger != nil {
mf.ScanLogger.LogMediaFetcher(zerolog.DebugLevel).
Int("offlineMediaCount", len(offlineMedia)).
Msg("Added media from anime-offline-database")
}
// Append offline media to AllMedia
mf.AllMedia = append(mf.AllMedia, offlineMedia...)
}
}
// +---------------------+
// | Unknown media |
// +---------------------+
// Media that are not in the user's collection
// Get the media that are not in the user's collection
unknownMedia := lo.Filter(mf.AllMedia, func(m *anilist.CompleteAnime, _ int) bool {
unknownMedia := lo.Filter(mf.AllMedia, func(m *anime.NormalizedMedia, _ int) bool {
return !lo.Contains(mf.CollectionMediaIds, m.ID)
})
// Get the media IDs that are not in the user's collection
mf.UnknownMediaIds = lop.Map(unknownMedia, func(m *anilist.CompleteAnime, _ int) int {
mf.UnknownMediaIds = lop.Map(unknownMedia, func(m *anime.NormalizedMedia, _ int) int {
return m.ID
})
@@ -181,6 +223,51 @@ func NewMediaFetcher(ctx context.Context, opts *MediaFetcherOptions) (ret *Media
return mf, nil
}
func NormalizedMediaFromAnilistComplete(c []*anilist.CompleteAnime) []*anime.NormalizedMedia {
normalizedMediaMap := make(map[int]*anime.NormalizedMedia)
// Convert CompleteAnime to NormalizedMedia and flatten relations
for _, m := range c {
if _, found := normalizedMediaMap[m.ID]; !found {
normalizedMediaMap[m.ID] = anime.NewNormalizedMedia(m.ToBaseAnime())
}
// Process relations
if m.Relations != nil && m.Relations.Edges != nil && len(m.Relations.Edges) > 0 {
for _, edgeM := range m.Relations.Edges {
if edgeM.Node == nil || edgeM.Node.Format == nil || edgeM.RelationType == nil {
continue
}
if *edgeM.Node.Format != anilist.MediaFormatMovie &&
*edgeM.Node.Format != anilist.MediaFormatOva &&
*edgeM.Node.Format != anilist.MediaFormatSpecial &&
*edgeM.Node.Format != anilist.MediaFormatTv {
continue
}
if *edgeM.RelationType != anilist.MediaRelationPrequel &&
*edgeM.RelationType != anilist.MediaRelationSequel &&
*edgeM.RelationType != anilist.MediaRelationSpinOff &&
*edgeM.RelationType != anilist.MediaRelationAlternative &&
*edgeM.RelationType != anilist.MediaRelationParent {
continue
}
// Make sure we don't overwrite the original media in the map
if _, found := normalizedMediaMap[edgeM.Node.ID]; !found {
normalizedMediaMap[edgeM.Node.ID] = anime.NewNormalizedMedia(edgeM.Node)
}
}
}
}
ret := make([]*anime.NormalizedMedia, 0, len(normalizedMediaMap))
for _, m := range normalizedMediaMap {
ret = append(ret, m)
}
return ret
}
//----------------------------------------------------------------------------------------------------------------------
// FetchMediaFromLocalFiles gets media and their relations from local file titles.

2
internal/library/scanner/media_tree_analysis.go
View File

@@ -106,7 +106,7 @@ func NewMediaTreeAnalysis(opts *MediaTreeAnalysisOptions) (*MediaTreeAnalysis, e
}
branches, _ := p.Wait()
if branches == nil || len(branches) == 0 {
if len(branches) == 0 {
return nil, errors.New("no branches found")
}

42
internal/library/scanner/pools.go Normal file
View File

@@ -0,0 +1,42 @@
package scanner
import (
"sync"
)
// Object pools to reduce allocs during scanning
// stringSlicePool provides reusable string slices for tokenization
var stringSlicePool = sync.Pool{
New: func() interface{} {
s := make([]string, 0, 16)
return &s
},
}
func getStringSlice() *[]string {
return stringSlicePool.Get().(*[]string)
}
func putStringSlice(s *[]string) {
*s = (*s)[:0]
stringSlicePool.Put(s)
}
// tokenSetPool provides reusable maps for token set operations
var tokenSetPool = sync.Pool{
New: func() interface{} {
return make(map[string]struct{}, 16)
},
}
func getTokenSet() map[string]struct{} {
return tokenSetPool.Get().(map[string]struct{})
}
func putTokenSet(m map[string]struct{}) {
// clear the map
for k := range m {
delete(m, k)
}
tokenSetPool.Put(m)
}

72
internal/library/scanner/scan.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"errors"
"os"
"runtime"
"runtime/debug"
"seanime/internal/api/anilist"
"seanime/internal/api/metadata_provider"
"seanime/internal/events"
@@ -25,20 +27,22 @@ import (
)
type Scanner struct {
DirPath string
OtherDirPaths []string
Enhanced bool
PlatformRef *util.Ref[platform.Platform]
Logger *zerolog.Logger
WSEventManager events.WSEventManagerInterface
ExistingLocalFiles []*anime.LocalFile
SkipLockedFiles bool
SkipIgnoredFiles bool
ScanSummaryLogger *summary.ScanSummaryLogger
ScanLogger *ScanLogger
MetadataProviderRef *util.Ref[metadata_provider.Provider]
MatchingThreshold float64
MatchingAlgorithm string
DirPath string
OtherDirPaths []string
Enhanced bool
EnhanceWithOfflineDatabase bool
PlatformRef *util.Ref[platform.Platform]
Logger *zerolog.Logger
WSEventManager events.WSEventManagerInterface
ExistingLocalFiles []*anime.LocalFile
SkipLockedFiles bool
SkipIgnoredFiles bool
ScanSummaryLogger *summary.ScanSummaryLogger
ScanLogger *ScanLogger
MetadataProviderRef *util.Ref[metadata_provider.Provider]
UseLegacyMatching bool
MatchingThreshold float64 // only used by legacy
MatchingAlgorithm string // only used by legacy
// If true, locked files whose library path doesn't exist will be put aside
WithShelving bool
ExistingShelvedFiles []*anime.LocalFile
@@ -301,7 +305,7 @@ func (scn *Scanner) Scan(ctx context.Context) (lfs []*anime.LocalFile, err error
scn.WSEventManager.SendEvent(events.EventScanProgress, 40)
if scn.Enhanced {
scn.WSEventManager.SendEvent(events.EventScanStatus, "Fetching media detected from file titles...")
scn.WSEventManager.SendEvent(events.EventScanStatus, "Fetching additional matching data...")
} else {
scn.WSEventManager.SendEvent(events.EventScanStatus, "Fetching media...")
}
@@ -312,15 +316,16 @@ func (scn *Scanner) Scan(ctx context.Context) (lfs []*anime.LocalFile, err error
// Fetch media needed for matching
mf, err := NewMediaFetcher(ctx, &MediaFetcherOptions{
Enhanced: scn.Enhanced,
PlatformRef: scn.PlatformRef,
MetadataProviderRef: scn.MetadataProviderRef,
LocalFiles: localFiles,
CompleteAnimeCache: completeAnimeCache,
Logger: scn.Logger,
AnilistRateLimiter: anilistRateLimiter,
DisableAnimeCollection: false,
ScanLogger: scn.ScanLogger,
Enhanced: scn.Enhanced,
EnhanceWithOfflineDatabase: scn.EnhanceWithOfflineDatabase,
PlatformRef: scn.PlatformRef,
MetadataProviderRef: scn.MetadataProviderRef,
LocalFiles: localFiles,
CompleteAnimeCache: completeAnimeCache,
Logger: scn.Logger,
AnilistRateLimiter: anilistRateLimiter,
DisableAnimeCollection: false,
ScanLogger: scn.ScanLogger,
})
if err != nil {
return nil, err
@@ -349,14 +354,14 @@ func (scn *Scanner) Scan(ctx context.Context) (lfs []*anime.LocalFile, err error
// Create a new matcher
matcher := &Matcher{
LocalFiles: localFiles,
MediaContainer: mc,
CompleteAnimeCache: completeAnimeCache,
Logger: scn.Logger,
ScanLogger: scn.ScanLogger,
ScanSummaryLogger: scn.ScanSummaryLogger,
Algorithm: scn.MatchingAlgorithm,
Threshold: scn.MatchingThreshold,
LocalFiles: localFiles,
MediaContainer: mc,
Logger: scn.Logger,
ScanLogger: scn.ScanLogger,
ScanSummaryLogger: scn.ScanSummaryLogger,
Algorithm: scn.MatchingAlgorithm,
Threshold: scn.MatchingThreshold,
UseLegacyMatching: scn.UseLegacyMatching,
}
scn.WSEventManager.SendEvent(events.EventScanProgress, 60)
@@ -466,6 +471,9 @@ func (scn *Scanner) Scan(ctx context.Context) (lfs []*anime.LocalFile, err error
hook.GlobalHookManager.OnScanCompleted().Trigger(completedEvent)
localFiles = completedEvent.LocalFiles
runtime.GC()
debug.FreeOSMemory()
return localFiles, nil
}

13
internal/library/scanner/scan_logger_test.go
View File

@@ -74,7 +74,7 @@ func TestScanLogger(t *testing.T) {
// +---------------------+
mc := NewMediaContainer(&MediaContainerOptions{
AllMedia: allMedia,
AllMedia: NormalizedMediaFromAnilistComplete(allMedia),
ScanLogger: scanLogger,
})
@@ -87,12 +87,11 @@ func TestScanLogger(t *testing.T) {
// +---------------------+
matcher := &Matcher{
LocalFiles: lfs,
MediaContainer: mc,
CompleteAnimeCache: completeAnimeCache,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: scanLogger,
ScanSummaryLogger: nil,
}
err = matcher.MatchLocalFilesWithMedia()

519
internal/library/scanner/title_normalization.go Normal file
View File

@@ -0,0 +1,519 @@
package scanner
import (
"regexp"
"seanime/internal/util/comparison"
"strconv"
"strings"
"unicode"
)
// Noise words that should be weighted less
var noiseWords = map[string]struct{}{
"the": {}, "a": {}, "an": {}, "of": {}, "to": {}, "in": {}, "for": {},
"on": {}, "with": {}, "at": {}, "by": {}, "from": {}, "as": {}, "is": {},
"it": {}, "that": {}, "this": {}, "be": {}, "are": {}, "was": {}, "were": {},
// japanese particles/common words
"no": {}, "wa": {}, "wo": {}, "ga": {}, "ni": {}, "de": {}, "ka": {},
"mo": {}, "ya": {}, "e": {}, "he": {},
// common anime title words
"anime": {}, "ova": {}, "ona": {}, "oad": {}, "tv": {}, "movie": {},
"nc": {}, "nced": {}, "ncop": {},
"extras": {}, "ending": {}, "opening": {}, "preview": {},
}
var ordinalToNumber = map[string]int{
"first": 1, "1st": 1,
"second": 2, "2nd": 2,
"third": 3, "3rd": 3,
"fourth": 4, "4th": 4,
"fifth": 5, "5th": 5,
"sixth": 6, "6th": 6,
"seventh": 7, "7th": 7,
"eighth": 8, "8th": 8,
"ninth": 9, "9th": 9,
"tenth": 10, "10th": 10,
}
// Roman numerals
// Note: skip I and X bc they are ambiguous
var romanToNumber = map[string]string{
"ii": "2", "iii": "3", "iv": "4", "v": "5",
"vi": "6", "vii": "7", "viii": "8", "ix": "9",
"xi": "11", "xii": "12", "xiii": "13",
}
// isSeparatorChar returns true for characters that should be normalized to spaces
func isSeparatorChar(r rune) bool {
switch r {
case '_', '.', '-', ':', ';', ',', '|':
return true
}
return false
}
func isAlphanumOrSpace(r rune) bool {
return (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == ' '
}
// collapseWhitespace collapses multiple whitespace characters into single spaces
// and trims leading/trailing whitespace. avoids allocating an intermediate slice.
func collapseWhitespace(s string) string {
if s == "" {
return s
}
var sb strings.Builder
sb.Grow(len(s)) // estimate capacity
inSpace := true // start as true to trim leading whitespace
for _, r := range s {
isSpace := r == ' ' || r == '\t' || r == '\n' || r == '\r'
if isSpace {
if !inSpace {
sb.WriteByte(' ')
inSpace = true
}
} else {
sb.WriteRune(r)
inSpace = false
}
}
result := sb.String()
// trim trailing space if present
if len(result) > 0 && result[len(result)-1] == ' ' {
result = result[:len(result)-1]
}
return result
}
// Season patterns
var (
// "Season 2", "S2", "S02", "2nd Season", etc.
seasonPatternExplicit = regexp.MustCompile(`(?i)\b(?:season|s|series)\s*0*(\d+)\b`)
seasonPatternOrdinal = regexp.MustCompile(`(?i)\b(\d+)(?:st|nd|rd|th)\s*(?:part|season|series)\b`)
seasonPatternSuffix = regexp.MustCompile(`(?i)\b(\d+)\s*(?:期|シーズン)\b`)
// Part patterns, e.g. "Part 2", "Part II", "Cour 2", "2nd Part"
partPatternExplicit = regexp.MustCompile(`(?i)\b(?:part|cour)\s*0*(\d+)\b`)
partPatternOrdinal = regexp.MustCompile(`(?i)\b(\d+)(?:st|nd|rd|th)\s*(?:part|cour)\b`)
partPatternRoman = regexp.MustCompile(`(?i)\b(?:part|cour)\s+(i{1,3}|iv|vi?i?i?|ix|x)\b`)
// Year patterns
yearParenRegex = regexp.MustCompile(`\((\d{4})\)`)
yearStandaloneRegex = regexp.MustCompile(`\b(19\d{2}|20\d{2})\b`)
)
// NormalizedTitle holds the normalized form and extracted metadata
type NormalizedTitle struct {
Original string
Normalized string
Tokens []string
Season int
Part int
Year int
CleanBaseTitle string // Title without season/part/year info
IsMain bool // Whether this title is a main title (romaji,english)
}
// NormalizeTitle creates a normalized version of a title for matching
func NormalizeTitle(title string) *NormalizedTitle {
if title == "" {
return &NormalizedTitle{}
}
result := &NormalizedTitle{
Original: title,
Season: -1,
Part: -1,
Year: -1,
}
// Extract metadata
result.Season = comparison.ExtractSeasonNumber(title)
result.Part = ExtractPartNumber(title)
result.Year = ExtractYear(title)
// Normalize the full title
normalizedFull := normalizeString(title)
result.Normalized = normalizedFull
// Tokenize
result.Tokens = tokenize(normalizedFull)
// Create a clean base title (without season/part/year markers)
cleanTitle := removeSeasonPartMarkers(title)
// Normalize the clean title
result.CleanBaseTitle = normalizeString(cleanTitle)
return result
}
func normalizeString(s string) string {
s = strings.ToLower(s)
// Macrons to double vowels
s = strings.ReplaceAll(s, "ō", "ou")
s = strings.ReplaceAll(s, "ū", "uu")
// Character replacements
s = strings.ReplaceAll(s, "@", "a")
s = strings.ReplaceAll(s, "×", "x")
s = strings.ReplaceAll(s, "꞉", ":")
s = strings.ReplaceAll(s, "＊", "*")
s = replaceWord(s, "the animation", "")
s = replaceWord(s, "the", "")
s = replaceWord(s, "episode", "")
s = replaceWord(s, "oad", "ova")
s = replaceWord(s, "oav", "ova")
s = replaceWord(s, "specials", "sp")
s = replaceWord(s, "special", "sp")
s = strings.ReplaceAll(s, "(tv)", "")
s = replaceWord(s, "&", "and")
// Replace smart quotes and apostrophes
s = strings.ReplaceAll(s, "'", "")
s = strings.ReplaceAll(s, "’", "")
s = strings.ReplaceAll(s, "`", "")
s = strings.ReplaceAll(s, "\"", "")
s = strings.ReplaceAll(s, "“", "")
s = strings.ReplaceAll(s, "”", "")
// Normalize separators to spaces
// normalize separators and non-alphanumeric characters
var sb strings.Builder
sb.Grow(len(s))
prevWasSpace := false
for _, r := range s {
if isSeparatorChar(r) || !isAlphanumOrSpace(r) {
// convert to space but avoid consecutive spaces
if !prevWasSpace {
sb.WriteByte(' ')
prevWasSpace = true
}
} else {
sb.WriteRune(r)
prevWasSpace = (r == ' ')
}
}
s = sb.String()
// Remove season markers entirely from normalized title
// Season/part numbers are extracted separately for scoring
// We don't want "Title S2" to match "Other Title 2" just because of the bare "2"
s = seasonPatternExplicit.ReplaceAllString(s, " ") // "Season X", "SX", "S0X"
s = seasonPatternOrdinal.ReplaceAllString(s, " ") // "2nd Season", "3rd Season"
s = seasonPatternSuffix.ReplaceAllString(s, " ") // Japanese "2期", "シーズン"
// Remove part markers entirely
s = partPatternExplicit.ReplaceAllString(s, " ") // "Part X", "Cour X"
s = partPatternOrdinal.ReplaceAllString(s, " ") // "2nd Part"
s = partPatternRoman.ReplaceAllString(s, " ") // "Part II"
// Collapse whitespace
s = collapseWhitespace(s)
// Devnote: intentionally keep Roman numerals (II, III, etc.) in the normalized title
// They help distinguish sequels like "Overlord II" from "Overlord"
// Season extraction handles them separately for scoring
return s
}
// replaceWord replaces all occurrences of old with new in s but only if old is a whole word.
// It matches case-insensitively since s is expected to be lower-cased, but the implementation relies on exact match of old.
// Assumes s is already lower-cased if old is lower-cased.
func replaceWord(s string, oldStr, newStr string) string {
if s == "" || oldStr == "" {
return s
}
var sb strings.Builder
// estimate the size to be roughly the same
sb.Grow(len(s))
start := 0
oldLen := len(oldStr)
for {
idx := strings.Index(s[start:], oldStr)
if idx == -1 {
sb.WriteString(s[start:])
break
}
absIdx := start + idx
// Check boundaries
isStartBoundary := absIdx == 0 || !isAlphanumeric(rune(s[absIdx-1]))
isEndBoundary := absIdx+oldLen == len(s) || !isAlphanumeric(rune(s[absIdx+oldLen]))
if isStartBoundary && isEndBoundary {
sb.WriteString(s[start:absIdx])
sb.WriteString(newStr)
start = absIdx + oldLen
} else {
sb.WriteString(s[start : absIdx+1]) // advance by 1 to avoid infinite loop on same match
start = absIdx + 1
}
}
return sb.String()
}
func isAlphanumeric(r rune) bool {
return (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z')
}
// tokenize splits a normalized string into tokens
func tokenize(s string) []string {
if s == "" {
return nil
}
// count tokens first to preallocate exact size
count := 0
inField := false
for _, r := range s {
isSpace := r == ' ' || r == '\t' || r == '\n' || r == '\r'
if isSpace {
inField = false
} else if !inField {
count++
inField = true
}
}
if count == 0 {
return nil
}
// allocate and fill
result := make([]string, 0, count)
inField = false
fieldStart := 0
for i, r := range s {
isSpace := r == ' ' || r == '\t' || r == '\n' || r == '\r'
if isSpace {
if inField {
result = append(result, s[fieldStart:i])
inField = false
}
} else {
if !inField {
fieldStart = i
inField = true
}
}
}
if inField {
result = append(result, s[fieldStart:])
}
return result
}
// removeSeasonPartMarkers removes season/part indicators from a title
func removeSeasonPartMarkers(title string) string {
s := title
// Remove explicit season markers
s = seasonPatternExplicit.ReplaceAllString(s, " ")
s = seasonPatternOrdinal.ReplaceAllString(s, " ")
s = seasonPatternSuffix.ReplaceAllString(s, " ")
// Remove part markers
s = partPatternExplicit.ReplaceAllString(s, " ")
s = partPatternOrdinal.ReplaceAllString(s, " ")
s = partPatternRoman.ReplaceAllString(s, " ")
// Remove year in parentheses
s = yearParenRegex.ReplaceAllString(s, " ")
// Clean up whitespace without allocating intermediate slice
s = collapseWhitespace(s)
return s
}
// ExtractPartNumber extracts the part number from a title string
func ExtractPartNumber(val string) int {
val = strings.ToLower(val)
// Check explicit patterns first: "Part 2", "Cour 2"
matches := partPatternExplicit.FindStringSubmatch(val)
if len(matches) > 1 {
if num, err := strconv.Atoi(matches[1]); err == nil {
return num
}
}
// Check ordinal patterns, e.g. "2nd Part", "2nd Cour"
matches = partPatternOrdinal.FindStringSubmatch(val)
if len(matches) > 1 {
if num, err := strconv.Atoi(matches[1]); err == nil {
return num
}
}
// Check roman numeral patterns, e.g. "Part II", not I or X since they're ambiguous
matches = partPatternRoman.FindStringSubmatch(val)
if len(matches) > 1 {
romanNum := strings.ToLower(matches[1])
if romanNum == "i" || romanNum == "x" {
return -1
}
if numStr, ok := romanToNumber[romanNum]; ok {
if num, err := strconv.Atoi(numStr); err == nil {
return num
}
}
}
return -1
}
// ExtractYear extracts a year from a title string
func ExtractYear(val string) int {
// Match years in parentheses first, e.g. "(2024)"
matches := yearParenRegex.FindStringSubmatch(val)
if len(matches) > 1 {
if year, err := strconv.Atoi(matches[1]); err == nil && year >= 1900 && year <= 2100 {
return year
}
}
// Match standalone years, look for 4-digit numbers that could be years
matches = yearStandaloneRegex.FindStringSubmatch(val)
if len(matches) > 1 {
if year, err := strconv.Atoi(matches[1]); err == nil {
return year
}
}
return -1
}
// GetSignificantTokens returns tokens that are not noise words
func GetSignificantTokens(tokens []string) []string {
result := make([]string, 0, len(tokens))
for _, token := range tokens {
if _, isNoise := noiseWords[token]; !isNoise && len(token) > 1 {
result = append(result, token)
}
}
return result
}
// getSignificantTokensInto filters tokens that are not noise words into the provided slice.
// This avoids allocations when the caller can reuse a slice.
func getSignificantTokensInto(tokens []string, dst []string) []string {
for _, token := range tokens {
if _, isNoise := noiseWords[token]; !isNoise && len(token) > 1 {
dst = append(dst, token)
}
}
return dst
}
func IsNoiseWord(word string) bool {
_, isNoise := noiseWords[strings.ToLower(word)]
return isNoise
}
// TokenMatchRatio calculates the ratio of matching tokens between two token sets.
func TokenMatchRatio(tokensA, tokensB []string) float64 {
if len(tokensA) == 0 || len(tokensB) == 0 {
return 0.0
}
// Create a set of tokensB for O(1) lookup
setB := getTokenSet()
defer putTokenSet(setB)
for _, t := range tokensB {
setB[t] = struct{}{}
}
// Count matches
matches := 0
for _, t := range tokensA {
if _, found := setB[t]; found {
matches++
}
}
// Return ratio based on the smaller set (more lenient for subset matching)
minLen := len(tokensA)
if len(tokensB) < minLen {
minLen = len(tokensB)
}
return float64(matches) / float64(minLen)
}
// WeightedTokenMatchRatio calculates match ratio with noise words weighted less.
func WeightedTokenMatchRatio(tokensA, tokensB []string) float64 {
if len(tokensA) == 0 || len(tokensB) == 0 {
return 0.0
}
setB := getTokenSet()
defer putTokenSet(setB)
for _, t := range tokensB {
setB[t] = struct{}{}
}
totalWeight := 0.0
matchedWeight := 0.0
for _, t := range tokensA {
weight := 1.0
if IsNoiseWord(t) {
weight = 0.3 // Noise words contribute less
}
totalWeight += weight
if _, found := setB[t]; found {
matchedWeight += weight
}
}
if totalWeight == 0 {
return 0.0
}
return matchedWeight / totalWeight
}
// ContainsAllTokens returns true if all tokens from subset are in superset
func ContainsAllTokens(subset, superset []string) bool {
if len(subset) == 0 {
return true
}
if len(superset) == 0 {
return false
}
setSuper := getTokenSet()
defer putTokenSet(setSuper)
for _, t := range superset {
setSuper[t] = struct{}{}
}
for _, t := range subset {
if _, found := setSuper[t]; !found {
return false
}
}
return true
}
func RemoveNonAlphanumeric(s string) string {
var result strings.Builder
for _, r := range s {
if unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsSpace(r) {
result.WriteRune(r)
}
}
return result.String()
}

39
internal/library/scanner/title_normalization_bench_test.go Normal file
View File

@@ -0,0 +1,39 @@
package scanner
import (
"testing"
)
// BenchmarkNormalizeTitle benchmarks the title normalization
func BenchmarkNormalizeTitle(b *testing.B) {
titles := []string{
"Attack on Titan Season 2",
"Kono Subarashii Sekai ni Shukufuku wo! 2",
"Boku no Hero Academia 5th Season",
"[SubsPlease] Mushoku Tensei S2 - 01 (1080p) [EC64C8B1].mkv",
"Overlord III",
"Steins;Gate 0",
"Jujutsu Kaisen 2nd Season",
"86 - Eighty Six Part 2",
"The Melancholy of Haruhi Suzumiya (2009)",
"KonoSuba.God's.Blessing.On.This.Wonderful.World.S02E01.1080p.BluRay.10-Bit.Dual-Audio.FLAC2.0.x265-YURASUKA",
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, title := range titles {
NormalizeTitle(title)
}
}
}
// BenchmarkNormalizeTitleParallel benchmarks parallel title normalization
func BenchmarkNormalizeTitleParallel(b *testing.B) {
title := "Kono Subarashii Sekai ni Shukufuku wo! Season 2 Part 1"
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
NormalizeTitle(title)
}
})
}

116
internal/library/scanner/title_normalization_test.go Normal file
View File

@@ -0,0 +1,116 @@
package scanner
import (
"testing"
)
func TestNormalizeTitle(t *testing.T) {
tests := []struct {
name string
input string
want string
wantBase string
season int
part int
}{
{
name: "Basic title",
input: "Attack on Titan",
want: "attack on titan",
wantBase: "attack on titan",
season: -1,
part: -1,
},
{
// Season markers are stripped from normalized title for accurate matching
// Season info is extracted into the Season field instead
name: "Title with season",
input: "Attack on Titan Season 2",
want: "attack on titan",
wantBase: "attack on titan",
season: 2,
part: -1,
},
{
// Season and part markers are stripped from normalized title
// They're extracted into Season and Part fields
name: "Title with part",
input: "Attack on Titan Season 3 Part 2",
want: "attack on titan",
wantBase: "attack on titan",
season: 3,
part: 2,
},
{
// Roman numerals are kept in normalized title for sequel distinction
// e.g. help distinguish "Overlord II" from "Overlord"
name: "Roman numeral season",
input: "Overlord III",
want: "overlord iii",
wantBase: "overlord iii",
season: 3, // ExtractSeasonNumber should extract this
},
{
name: "Special characters",
input: "Steins;Gate",
want: "steins gate",
wantBase: "steins gate",
},
{
name: "Smart quotes",
input: "Kino's Journey",
want: "kinos journey",
wantBase: "kinos journey",
},
{
name: "The Animation suffix",
input: "Persona 4 The Animation",
want: "persona 4",
wantBase: "persona 4",
},
{
name: "Case sensitivity",
input: "ATTACK ON TITAN",
want: "attack on titan",
wantBase: "attack on titan",
},
{
name: "With 'The'",
input: "The Melancholy of Haruhi Suzumiya",
want: "melancholy of haruhi suzumiya",
wantBase: "melancholy of haruhi suzumiya",
},
{
name: "With 'Episode'",
input: "One Piece Episode 1000",
want: "one piece 1000",
wantBase: "one piece 1000",
},
{
name: "OAD/OVA",
input: "Attack on Titan OAD",
want: "attack on titan ova",
wantBase: "attack on titan ova",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NormalizeTitle(tt.input)
if got.Normalized != tt.want {
t.Errorf("NormalizeTitle(%q).Normalized = %q, want %q", tt.input, got.Normalized, tt.want)
}
// check base title only if expected is provided (some cases might be tricky with what 'base' implies)
if tt.wantBase != "" && got.CleanBaseTitle != tt.wantBase {
t.Errorf("NormalizeTitle(%q).CleanBaseTitle = %q, want %q", tt.input, got.CleanBaseTitle, tt.wantBase)
}
// Check season extraction if specified
if tt.season != 0 && got.Season != tt.season {
t.Errorf("NormalizeTitle(%q).Season = %d, want %d", tt.input, got.Season, tt.season)
}
// Check part extraction if specified
if tt.part != 0 && got.Part != tt.part {
t.Errorf("NormalizeTitle(%q).Part = %d, want %d", tt.input, got.Part, tt.part)
}
})
}
}

6
internal/library/summary/scan_summary.go
View File

@@ -134,10 +134,10 @@ func (l *ScanSummaryLogger) GenerateSummary() *ScanSummary {
mediaIsInCollection := false
for _, m := range l.AllMedia {
if m.ID == mediaId {
mediaTitle = m.GetPreferredTitle()
mediaTitle = m.GetTitleSafe()
mediaImage = ""
if m.GetCoverImage() != nil && m.GetCoverImage().GetLarge() != nil {
mediaImage = *m.GetCoverImage().GetLarge()
if m.CoverImage != nil && m.CoverImage.Large != nil {
mediaImage = *m.CoverImage.Large
}
break
}

2
internal/plugin/ui/context.go
View File

@@ -1373,7 +1373,7 @@ func (c *Context) flushEventBatch() {
c.eventBatchTimer.Stop()
// Create a copy of the pending events
allEvents := make([]*ServerPluginEvent, len(c.pendingClientEvents))
allEvents := make([]*ServerPluginEvent, 0, len(c.pendingClientEvents))
copy(allEvents, c.pendingClientEvents)
// Clear the pending events

13
internal/torrents/analyzer/analyzer.go
View File

@@ -213,7 +213,7 @@ func (a *Analyzer) scanFiles() error {
allMedia := tree.Values()
mc := scanner.NewMediaContainer(&scanner.MediaContainerOptions{
AllMedia: allMedia,
AllMedia: scanner.NormalizedMediaFromAnilistComplete(allMedia),
})
//scanLogger, _ := scanner.NewScanLogger("./logs")
@@ -223,12 +223,11 @@ func (a *Analyzer) scanFiles() error {
// +---------------------+
matcher := &scanner.Matcher{
LocalFiles: lfs,
MediaContainer: mc,
CompleteAnimeCache: completeAnimeCache,
Logger: util.NewLogger(),
ScanLogger: nil,
ScanSummaryLogger: nil,
LocalFiles: lfs,
MediaContainer: mc,
Logger: util.NewLogger(),
ScanLogger: nil,
ScanSummaryLogger: nil,
}
err := matcher.MatchLocalFilesWithMedia()

152
internal/util/comparison/filtering.go
View File

@@ -6,6 +6,54 @@ import (
"strings"
)
var (
// ValueContainsSeason regex
seasonOrdinalRegex = regexp.MustCompile(`\d(st|nd|rd|th) [Ss].*`)
// ExtractSeasonNumber regexes
seasonExplicitRegex = regexp.MustCompile(`season\s*(\d+)`)
seasonFormatRegex = regexp.MustCompile(`\bs0?(\d{1,2})(?:e\d|$|\s|\.)`)
seasonOrdinalNumRegex = regexp.MustCompile(`(\d+)(?:st|nd|rd|th)\s+season`)
romanPattern1Regex = regexp.MustCompile(`[\s.](i{1,3}|iv|vi?i?i?|ix|x)(?:\s|$|[:,.]|['\'])`)
romanPattern2Regex = regexp.MustCompile(`[\s.](i{1,3}|iv|vi?i?i?|ix|x)[.\s]*(?:s\d|e\d|part)`)
seasonTrailingNumRe = regexp.MustCompile(`(?:^|\s)(\d{1,2})\s*$`)
seasonPartCourRegex = regexp.MustCompile(`(?:part|cour)\s*\d{1,2}\s*$`)
seasonJapaneseRegex = regexp.MustCompile(`(?:第)?(\d+)\s*期`)
// ValueContainsSpecial regexes
specialRegex1 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)(SP|OAV|OVA|OAD|ONA) ?(?P<ep>\d{1,2})(-(?P<ep2>[0-9]{1,3}))? ?(?P<title>.*)$`)
specialRegex2 = regexp.MustCompile(`(?i)[-._( ](OVA|ONA)[-._) ]`)
specialRegex3 = regexp.MustCompile(`(?i)[-._ ](S|SP)(?P<season>(0|00))([Ee]\d)`)
specialRegex4 = regexp.MustCompile(`[-._({[ ]?(OVA|ONA|OAV|OAD)[])}\-._ ]?`)
// ValueContainsIgnoredKeywords regex
ignoredKeywordsRegex = regexp.MustCompile(`(?i)^\s?[({\[]?\s?(EXTRAS?|OVAS?|OTHERS?|SPECIALS|MOVIES|SEASONS|NC)\s?[])}]?\s?$`)
// ValueContainsBatchKeywords regex
batchKeywordsRegex = regexp.MustCompile(`(?i)[({\[]?\s?(EXTRAS|OVAS|OTHERS|SPECIALS|MOVIES|SEASONS|BATCH|COMPLETE|COMPLETE SERIES)\s?[])}]?\s?`)
// ValueContainsNC regexes
ncRegex1 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(OP|NCOP|OPED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`)
ncRegex2 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(ED|NCED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`)
ncRegex3 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(TRAILER|PROMO|PV)\b ?(?P<ep>\d{1,2}) ?([ _.\-)]+(?P<title>.*))?`)
ncRegex4 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(OTHERS?)\b(?P<ep>\d{1,2}) ?[ _.\-)]+(?P<title>.*)`)
ncRegex5 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(CM|COMMERCIAL|AD)\b ?(?P<ep>\d{1,2}) ?([ _.\-)]+(?P<title>.*))?`)
ncRegex6 = regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(CREDITLESS|NCOP|NCED|OP|ED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`)
ncRegex7 = regexp.MustCompile(`(?i)- ?(Opening|Ending)`)
// Roman numeral mapping
romanToNum = map[string]int{
"ii": 2, "iii": 3, "iv": 4, "v": 5,
"vi": 6, "vii": 7, "viii": 8, "ix": 9,
}
IgnoredFilenames = map[string]struct{}{
"extras": {}, "ova": {}, "ovas": {}, "ona": {}, "onas": {}, "oad": {}, "oads": {}, "others": {}, "specials": {}, "movies": {}, "seasons": {}, "batch": {},
"complete": {}, "complete series": {}, "nc": {}, "music": {}, "mv": {}, "trailer": {}, "promo": {}, "pv": {}, "commercial": {}, "ad": {}, "opening": {}, "ending": {},
"op": {}, "ed": {}, "ncop": {}, "nced": {}, "creditless": {},
}
)
func ValueContainsSeason(val string) bool {
val = strings.ToLower(val)
@@ -20,8 +68,7 @@ func ValueContainsSeason(val string) bool {
return true
}
re := regexp.MustCompile(`\d(st|nd|rd|th) [Ss].*`)
if re.MatchString(val) {
if seasonOrdinalRegex.MatchString(val) {
return true
}
@@ -31,9 +78,8 @@ func ValueContainsSeason(val string) bool {
func ExtractSeasonNumber(val string) int {
val = strings.ToLower(val)
// Check for the word "season" followed by a number
re := regexp.MustCompile(`season (\d+)`)
matches := re.FindStringSubmatch(val)
// "season X" pattern
matches := seasonExplicitRegex.FindStringSubmatch(val)
if len(matches) > 1 {
season, err := strconv.Atoi(matches[1])
if err == nil {
@@ -41,9 +87,51 @@ func ExtractSeasonNumber(val string) int {
}
}
// Check for a number followed by "st", "nd", "rd", or "th", followed by "s" or "S"
re = regexp.MustCompile(`(\d+)(st|nd|rd|th) [sS]`)
matches = re.FindStringSubmatch(val)
// "SXX" or "S0X" format
matches = seasonFormatRegex.FindStringSubmatch(val)
if len(matches) > 1 {
season, err := strconv.Atoi(matches[1])
if err == nil && season > 0 && season < 20 {
return season
}
}
// Ordinal + season (e.g., "2nd season")
matches = seasonOrdinalNumRegex.FindStringSubmatch(val)
if len(matches) > 1 {
season, err := strconv.Atoi(matches[1])
if err == nil {
return season
}
}
// Roman numerals at end of title or before common markers (e.g., "Overlord II", "Title III")
romanPatterns := []*regexp.Regexp{romanPattern1Regex, romanPattern2Regex}
for _, re := range romanPatterns {
matches = re.FindStringSubmatch(val)
if len(matches) > 1 {
romanNum := strings.ToLower(matches[1])
if num, ok := romanToNum[romanNum]; ok {
return num
}
}
}
// Number at the end of title (e.g., "Konosuba 2", only 2-10 range)
// Exclude numbers preceded by "part" or "cour" as those indicate parts, not seasons
matches = seasonTrailingNumRe.FindStringSubmatch(val)
if len(matches) > 1 {
// check if preceded by "part" or "cour"
if !seasonPartCourRegex.MatchString(val) {
season, err := strconv.Atoi(matches[1])
if err == nil && season >= 2 && season <= 10 {
return season
}
}
}
// Japanese season indicators (e.g., "2期")
matches = seasonJapaneseRegex.FindStringSubmatch(val)
if len(matches) > 1 {
season, err := strconv.Atoi(matches[1])
if err == nil {
@@ -51,16 +139,15 @@ func ExtractSeasonNumber(val string) int {
}
}
// No season number found
return -1
}
func ValueContainsSpecial(val string) bool {
regexes := []*regexp.Regexp{
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)(SP|OAV|OVA|OAD|ONA) ?(?P<ep>\d{1,2})(-(?P<ep2>[0-9]{1,3}))? ?(?P<title>.*)$`),
regexp.MustCompile(`(?i)[-._( ](OVA|ONA)[-._) ]`),
regexp.MustCompile(`(?i)[-._ ](S|SP)(?P<season>(0|00))([Ee]\d)`),
regexp.MustCompile(`[-._({\[ ]?(OVA|ONA|OAV|OAD)[])}\-._ ]?`),
specialRegex1,
specialRegex2,
specialRegex3,
specialRegex4,
}
for _, regex := range regexes {
@@ -73,41 +160,22 @@ func ValueContainsSpecial(val string) bool {
}
func ValueContainsIgnoredKeywords(val string) bool {
regexes := []*regexp.Regexp{
regexp.MustCompile(`(?i)^\s?[({\[]?\s?(EXTRAS?|OVAS?|OTHERS?|SPECIALS|MOVIES|SEASONS|NC)\s?[])}]?\s?$`),
}
for _, regex := range regexes {
if regex.MatchString(val) {
return true
}
}
return false
return ignoredKeywordsRegex.MatchString(val)
}
func ValueContainsBatchKeywords(val string) bool {
regexes := []*regexp.Regexp{
regexp.MustCompile(`(?i)[({\[]?\s?(EXTRAS|OVAS|OTHERS|SPECIALS|MOVIES|SEASONS|BATCH|COMPLETE|COMPLETE SERIES)\s?[])}]?\s?`),
}
for _, regex := range regexes {
if regex.MatchString(val) {
return true
}
}
return false
return batchKeywordsRegex.MatchString(val)
}
func ValueContainsNC(val string) bool {
regexes := []*regexp.Regexp{
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(OP|NCOP|OPED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`),
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(ED|NCED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`),
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(TRAILER|PROMO|PV)\b ?(?P<ep>\d{1,2}) ?([ _.\-)]+(?P<title>.*))?`),
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(OTHERS?)\b(?P<ep>\d{1,2}) ?[ _.\-)]+(?P<title>.*)`),
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(CM|COMMERCIAL|AD)\b ?(?P<ep>\d{1,2}) ?([ _.\-)]+(?P<title>.*))?`),
regexp.MustCompile(`(?i)(^|(?P<show>.*?)[ _.\-(]+)\b(CREDITLESS|NCOP|NCED|OP|ED)\b ?(?P<ep>\d{1,2}[a-z]?)? ?([ _.\-)]+(?P<title>.*))?`),
regexp.MustCompile(`(?i)- ?(Opening|Ending)`),
ncRegex1,
ncRegex2,
ncRegex3,
ncRegex4,
ncRegex5,
ncRegex6,
ncRegex7,
}
for _, regex := range regexes {

16
internal/util/comparison/filtering_test.go
View File

@@ -1,6 +1,7 @@
package comparison
import (
"seanime/internal/util"
"testing"
)
@@ -64,12 +65,17 @@ func TestExtractSeasonNumber(t *testing.T) {
expected: 2,
},
{
name: "Contains a number followed by 'st', 'nd', 'rd', or 'th', followed by 's' or 'S'",
input: "Spy x Family 2nd S",
name: "Ordinal",
input: "Spy x Family 2nd Season",
expected: 2,
},
{
name: "Does not contain 'season' or '1st S'",
name: "Roman Numerals",
input: "Overlord III",
expected: 3,
},
{
name: "Does not contain season",
input: "This is a test",
expected: -1,
},
@@ -130,7 +136,7 @@ func TestExtractResolutionInt(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := ExtractResolutionInt(test.input)
result := util.ExtractResolutionInt(test.input)
if result != test.expected {
t.Errorf("ExtractResolutionInt() with args %v, expected %v, but got %v.", test.input, test.expected, result)
}
@@ -309,7 +315,7 @@ func TestNormalizeResolution(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := NormalizeResolution(tt.input); got != tt.expected {
if got := util.NormalizeResolution(tt.input); got != tt.expected {
t.Errorf("NormalizeResolution() = %v, want %v", got, tt.expected)
}
})

3
internal/util/limiter/limiter.go
View File

@@ -9,7 +9,8 @@ import (
func NewAnilistLimiter() *Limiter {
//return NewLimiter(15*time.Second, 18)
return NewLimiter(6*time.Second, 8)
//return NewLimiter(6*time.Second, 8)
return NewLimiter(10*time.Second, 5)
}
//----------------------------------------------------------------------------------------------------------------------

5
internal/util/round_tripper.go
View File

@@ -65,8 +65,7 @@ func (ug *cloudFlareRoundTripper) RoundTrip(r *http.Request) (*http.Response, er
for header, value := range ug.options.Headers {
if _, ok := r.Header[header]; !ok {
if header == "User-Agent" {
// Generate new random user agent for each attempt
r.Header.Set(header, GetRandomUserAgent())
r.Header.Set(header, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36")
} else {
r.Header.Set(header, value)
}
@@ -120,7 +119,7 @@ func GetDefaultOptions() Options {
Headers: map[string]string{
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"User-Agent": GetRandomUserAgent(),
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
},
}
}

22
internal/util/user_agent.go
View File

@@ -7,8 +7,6 @@ import (
"net/http"
"sync"
"time"
"github.com/rs/zerolog/log"
)
var (
@@ -16,26 +14,6 @@ var (
uaMu sync.RWMutex
)
func init() {
go func() {
defer func() {
if r := recover(); r != nil {
log.Warn().Msgf("util: Failed to get online user agents: %v", r)
}
}()
agents, err := getOnlineUserAgents()
if err != nil {
log.Warn().Err(err).Msg("util: Failed to get online user agents")
return
}
uaMu.Lock()
userAgentList = agents
uaMu.Unlock()
}()
}
func getOnlineUserAgents() ([]string, error) {
link := "https://raw.githubusercontent.com/fake-useragent/fake-useragent/refs/heads/main/src/fake_useragent/data/browsers.jsonl"

1
seanime-web/src/api/generated/endpoint.types.ts
View File

@@ -1757,6 +1757,7 @@ export type SaveIssueReport_Variables = {
*/
export type ScanLocalFiles_Variables = {
enhanced: boolean
enhanceWithOfflineDatabase: boolean
skipLockedFiles: boolean
skipIgnoredFiles: boolean
}

1
seanime-web/src/api/generated/types.ts
View File

@@ -3804,6 +3804,7 @@ export type Models_LibrarySettings = {
autoSyncToLocalAccount: boolean
autoSaveCurrentMediaOffline: boolean
useFallbackMetadataProvider: boolean
scannerUseLegacyMatching: boolean
}
/**

33
seanime-web/src/app/(main)/(library)/_containers/scanner-modal.tsx
View File

@@ -7,6 +7,7 @@ import { AppLayoutStack } from "@/components/ui/app-layout"
import { Button } from "@/components/ui/button"
import { cn } from "@/components/ui/core/styling.ts"
import { Modal } from "@/components/ui/modal"
import { RadioGroup } from "@/components/ui/radio-group"
import { Separator } from "@/components/ui/separator"
import { Switch } from "@/components/ui/switch"
import { useBoolean } from "@/hooks/use-disclosure"
@@ -29,6 +30,7 @@ export function ScannerModal() {
const anilistDataOnly = useBoolean(true)
const skipLockedFiles = useBoolean(true)
const skipIgnoredFiles = useBoolean(true)
const enhanceWithOfflineDatabase = useBoolean(true)
const { mutate: scanLibrary, isPending: isScanning } = useScanLocalFiles(() => {
setOpen(false)
@@ -48,6 +50,7 @@ export function ScannerModal() {
enhanced: !anilistDataOnly.active,
skipLockedFiles: skipLockedFiles.active,
skipIgnoredFiles: skipIgnoredFiles.active,
enhanceWithOfflineDatabase: enhanceWithOfflineDatabase.active,
})
setOpen(false)
}
@@ -150,21 +153,33 @@ export function ScannerModal() {
<h5 className="text-[--muted]">Matching data</h5>
<Switch
side="right"
label="Use my AniList lists only"
moreHelp="Disabling this will cause Seanime to send more API requests which may lead to rate limits and slower scanning"
// label="Enhanced scanning"
label="My AniList Collection only"
moreHelp="This is faster but generally less accurate if your collection does not contain all anime in the library."
help={anilistDataOnly.active ? "Matches local files against your AniList collection." : ""}
value={anilistDataOnly.active}
onValueChange={v => anilistDataOnly.set(v as boolean)}
// className="data-[state=checked]:bg-amber-700 dark:data-[state=checked]:bg-amber-700"
// size="lg"
help={!anilistDataOnly.active
? <span><span className="text-[--orange]">Slower for large libraries</span>. For faster scanning, add the anime
entries present in your library to your
lists and re-enable this before
scanning.</span>
: ""}
disabled={!userMedia?.length}
/>
{!anilistDataOnly.active && <RadioGroup
label="Enhanced matching method"
options={[
{ value: "database", label: "Use Anime Offline Database" },
{ value: "anilist", label: "Use AniList API" },
]}
size="lg"
stackClass="space-y-2 py-1"
value={enhanceWithOfflineDatabase.active ? "database" : "anilist"}
onValueChange={v => enhanceWithOfflineDatabase.set(v === "database")}
help={enhanceWithOfflineDatabase.active
? <span>Matches local files against the entire AniList catalog. Scanning will be slower.</span>
: <span><span className="text-[--orange]">Slower for large libraries</span>. Seanime will send an API request for
each anime title found in the library,
which may lead to rate limits and
slower scanning.</span>}
/>}
</AppLayoutStack>
</AppLayoutStack>

21
seanime-web/src/app/(main)/settings/_containers/anime-library-settings.tsx
View File

@@ -5,6 +5,8 @@ import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from "@/
import { Field } from "@/components/ui/form"
import { Separator } from "@/components/ui/separator"
import React from "react"
import { useWatch } from "react-hook-form"
import { useFormContext } from "react-hook-form"
import { FcFolder } from "react-icons/fc"
type LibrarySettingsProps = {
@@ -18,6 +20,11 @@ export function AnimeLibrarySettings(props: LibrarySettingsProps) {
...rest
} = props
const { watch } = useFormContext()
const useLegacyMatching = useWatch({ name: "scannerUseLegacyMatching" })
const useLegacyEnhancedMatching = useWatch({ name: "scannerUseLegacyEnhancedMatching" })
return (
<div className="space-y-4">
@@ -66,14 +73,22 @@ export function AnimeLibrarySettings(props: LibrarySettingsProps) {
className="border rounded-[--radius-md]"
triggerClass="dark:bg-[--paper]"
contentClass="!pt-2 dark:bg-[--paper]"
defaultValue={(useLegacyMatching || useLegacyEnhancedMatching) ? "more" : undefined}
>
<AccordionItem value="more">
<AccordionTrigger className="bg-gray-900 rounded-[--radius-md]">
Advanced
</AccordionTrigger>
<AccordionContent className="space-y-4">
<div className="flex flex-col md:flex-row gap-3">
<>
<Field.Switch
name="scannerUseLegacyMatching"
label="Use legacy matching algorithm"
help="Enable to use the legacy matching algorithms. (Versions 3.4 and below)"
moreHelp="The legacy matching algorithm uses simpler methods which may be less accurate."
/>
</>
{useLegacyMatching && <div className="flex flex-col md:flex-row gap-3">
<Field.Select
options={[
{ value: "-", label: "Levenshtein + Sorensen-Dice (Default)" },
@@ -96,7 +111,7 @@ export function AnimeLibrarySettings(props: LibrarySettingsProps) {
max={1.0}
step={0.1}
/>
</div>
</div>}
<Separator />

2
seanime-web/src/app/(main)/settings/page.tsx
View File

@@ -326,6 +326,7 @@ export default function Page() {
autoSyncToLocalAccount: data.autoSyncToLocalAccount ?? false,
autoSaveCurrentMediaOffline: data.autoSaveCurrentMediaOffline ?? false,
useFallbackMetadataProvider: data.useFallbackMetadataProvider ?? false,
scannerUseLegacyMatching: data.scannerUseLegacyMatching ?? false,
},
nakama: {
enabled: data.nakamaEnabled ?? false,
@@ -491,6 +492,7 @@ export default function Page() {
vcTranslateApiKey: status?.settings?.mediaPlayer?.vcTranslateApiKey ?? "",
vcTranslateProvider: status?.settings?.mediaPlayer?.vcTranslateProvider ?? "",
vcTranslateTargetLanguage: status?.settings?.mediaPlayer?.vcTranslateTargetLanguage ?? "",
scannerUseLegacyMatching: status?.settings?.library?.scannerUseLegacyMatching ?? false,
}}
stackClass="space-y-0 relative"
>

2
seanime-web/src/lib/server/settings.ts
View File

@@ -116,6 +116,7 @@ export const settingsSchema = z.object({
vcTranslateApiKey: z.string().optional().default(""),
vcTranslateProvider: z.string().optional().default(""),
vcTranslateTargetLanguage: z.string().optional().default(""),
scannerUseLegacyMatching: z.boolean().optional().default(false),
})
export const gettingStartedSchema = _gettingStartedSchema.extend(settingsSchema.shape)
@@ -145,6 +146,7 @@ export const getDefaultSettings = (data: z.infer<typeof gettingStartedSchema>):
autoSyncToLocalAccount: false,
autoSaveCurrentMediaOffline: false,
useFallbackMetadataProvider: false,
scannerUseLegacyMatching: false,
},
nakama: {
enabled: false,