Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
b9bd7e6eb0 | ||
|
|
17a865043c | ||
|
|
a93f4c2b17 | ||
|
|
db5c0aec4e | ||
|
|
119a25a59c | ||
|
|
c83847e16a | ||
|
|
4568302187 | ||
|
|
e5a9f05e76 | ||
|
|
08e5ebbd3c | ||
|
|
9ffdcadd81 |
6
.github/workflows/tests.yml
vendored
6
.github/workflows/tests.yml
vendored
@@ -4,18 +4,20 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- 2.0
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- 2.0
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "pypy3"]
|
||||
python-version: ["3.8", "3.9", "3.10", "pypy3"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
@@ -16,7 +16,7 @@ and powerful `filenaming capabilities <Formatting_>`_.
|
||||
Dependencies
|
||||
============
|
||||
|
||||
- Python_ 3.4+
|
||||
- Python_ 3.8+
|
||||
- Requests_
|
||||
|
||||
Optional
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"extractor":
|
||||
{
|
||||
"general": {
|
||||
"base-directory": "~/gallery-dl/",
|
||||
|
||||
"#": "set global archive file for all extractors",
|
||||
@@ -32,265 +31,306 @@
|
||||
"mode": "tags",
|
||||
"whitelist": ["danbooru", "moebooru", "sankaku"]
|
||||
}
|
||||
],
|
||||
]
|
||||
},
|
||||
|
||||
"pixiv":
|
||||
{
|
||||
"#": "override global archive setting for pixiv",
|
||||
"archive": "~/gallery-dl/archive-pixiv.sqlite3",
|
||||
"postprocessor": {
|
||||
"#": "predefine several post processors, so can later be used by name",
|
||||
|
||||
"#": "set custom directory and filename format strings for all pixiv downloads",
|
||||
"filename": "{id}{num}.{extension}",
|
||||
"directory": ["Pixiv", "Works", "{user[id]}"],
|
||||
"refresh-token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ01234567890-FedC9",
|
||||
|
||||
"#": "transform ugoira into lossless MKVs",
|
||||
"ugoira": true,
|
||||
"postprocessors": ["ugoira-copy"],
|
||||
|
||||
"#": "use special settings for favorites and bookmarks",
|
||||
"favorite":
|
||||
{
|
||||
"directory": ["Pixiv", "Favorites", "{user[id]}"]
|
||||
},
|
||||
"bookmark":
|
||||
{
|
||||
"directory": ["Pixiv", "My Bookmarks"],
|
||||
"refresh-token": "01234567890aBcDeFgHiJkLmNoPqRsTuVwXyZ-ZyxW1"
|
||||
}
|
||||
"#": "write 'content' metadata into separate files",
|
||||
"content": {
|
||||
"name": "metadata",
|
||||
"#": "write only the values for 'content' or 'description'",
|
||||
"event": "post",
|
||||
"filename": "{post_id|tweet_id|id}.txt",
|
||||
"mode": "custom",
|
||||
"format": "{content|description}\n"
|
||||
},
|
||||
|
||||
"danbooru":
|
||||
{
|
||||
"ugoira": true,
|
||||
"postprocessors": ["ugoira-webm"]
|
||||
"#": "put files into a '.cbz' archive",
|
||||
"cbz": {
|
||||
"name": "zip",
|
||||
"extension": "cbz"
|
||||
},
|
||||
|
||||
"exhentai":
|
||||
{
|
||||
"#": "use cookies instead of logging in with username and password",
|
||||
"cookies":
|
||||
{
|
||||
"ipb_member_id": "12345",
|
||||
"ipb_pass_hash": "1234567890abcdef",
|
||||
"igneous" : "123456789",
|
||||
"hath_perks" : "m1.m2.m3.a-123456789a",
|
||||
"sk" : "n4m34tv3574m2c4e22c35zgeehiw",
|
||||
"sl" : "dm_2"
|
||||
},
|
||||
|
||||
"#": "wait 2 to 4.8 seconds between HTTP requests",
|
||||
"sleep-request": [2.0, 4.8],
|
||||
|
||||
"filename": "{num:>04}_{name}.{extension}",
|
||||
"directory": ["{category!c}", "{title}"]
|
||||
},
|
||||
|
||||
"sankaku":
|
||||
{
|
||||
"#": "authentication with cookies is not possible for sankaku",
|
||||
"username": "user",
|
||||
"password": "#secret#"
|
||||
},
|
||||
|
||||
"furaffinity": {
|
||||
"#": "authentication with username and password is not possible due to CAPTCHA",
|
||||
"cookies": {
|
||||
"a": "01234567-89ab-cdef-fedc-ba9876543210",
|
||||
"b": "fedcba98-7654-3210-0123-456789abcdef"
|
||||
},
|
||||
|
||||
"descriptions": "html",
|
||||
"postprocessors": ["content"]
|
||||
},
|
||||
|
||||
"deviantart":
|
||||
{
|
||||
"#": "download 'gallery' and 'scraps' images for user profile URLs",
|
||||
"include": "gallery,scraps",
|
||||
|
||||
"#": "use custom API credentials to avoid 429 errors",
|
||||
"client-id": "98765",
|
||||
"client-secret": "0123456789abcdef0123456789abcdef",
|
||||
"refresh-token": "0123456789abcdef0123456789abcdef01234567",
|
||||
|
||||
"#": "put description texts into a separate directory",
|
||||
"metadata": true,
|
||||
"postprocessors": [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "custom",
|
||||
"directory" : "Descriptions",
|
||||
"content-format" : "{description}\n",
|
||||
"extension-format": "descr.txt"
|
||||
}
|
||||
"#": "various ugoira post processor configurations to create different file formats",
|
||||
"ugoira-webm": {
|
||||
"name": "ugoira",
|
||||
"extension": "webm",
|
||||
"ffmpeg-twopass": true,
|
||||
"ffmpeg-args": [
|
||||
"-c:v", "libvpx-vp9",
|
||||
"-b:v", "0",
|
||||
"-crf", "30"
|
||||
]
|
||||
},
|
||||
|
||||
"flickr":
|
||||
{
|
||||
"access-token": "1234567890-abcdef",
|
||||
"access-token-secret": "1234567890abcdef",
|
||||
"size-max": 1920
|
||||
"ugoira-mp4": {
|
||||
"name": "ugoira",
|
||||
"extension": "mp4",
|
||||
"ffmpeg-twopass": true,
|
||||
"ffmpeg-args": [
|
||||
"-c:v", "libx264",
|
||||
"-b:v", "4M",
|
||||
"-preset", "veryslow"
|
||||
],
|
||||
"libx264-prevent-odd": true
|
||||
},
|
||||
|
||||
"mangadex":
|
||||
{
|
||||
"#": "only download safe/suggestive chapters translated to English",
|
||||
"lang": "en",
|
||||
"ratings": ["safe", "suggestive"],
|
||||
|
||||
"#": "put chapters into '.cbz' archives",
|
||||
"postprocessors": ["cbz"]
|
||||
"ugoira-gif": {
|
||||
"name": "ugoira",
|
||||
"extension": "gif",
|
||||
"ffmpeg-args": [
|
||||
"-filter_complex",
|
||||
"[0:v] split [a][b];[a] palettegen [p];[b][p] paletteuse"
|
||||
]
|
||||
},
|
||||
|
||||
"reddit":
|
||||
{
|
||||
"#": "only spawn child extractors for links to specific sites",
|
||||
"whitelist": ["imgur", "redgifs", "gfycat"],
|
||||
|
||||
"#": "put files from child extractors into the reddit directory",
|
||||
"parent-directory": true,
|
||||
|
||||
"#": "transfer metadata to any child extractor as '_reddit'",
|
||||
"parent-metadata": "_reddit"
|
||||
},
|
||||
|
||||
"imgur":
|
||||
{
|
||||
"#": "use different directory and filename formats when coming from a reddit post",
|
||||
"directory":
|
||||
{
|
||||
"'_reddit' in locals()": []
|
||||
},
|
||||
"filename":
|
||||
{
|
||||
"'_reddit' in locals()": "{_reddit[id]} {id}.{extension}",
|
||||
"" : "{id}.{extension}"
|
||||
}
|
||||
},
|
||||
|
||||
"tumblr":
|
||||
{
|
||||
"posts" : "all",
|
||||
"external": false,
|
||||
"reblogs" : false,
|
||||
"inline" : true,
|
||||
|
||||
"#": "use special settings when downloading liked posts",
|
||||
"likes":
|
||||
{
|
||||
"posts" : "video,photo,link",
|
||||
"external": true,
|
||||
"reblogs" : true
|
||||
}
|
||||
},
|
||||
|
||||
"twitter":
|
||||
{
|
||||
"#": "write text content for *all* tweets",
|
||||
"postprocessors": ["content"],
|
||||
"text-tweets": true
|
||||
},
|
||||
|
||||
"mastodon":
|
||||
{
|
||||
"#": "add 'tabletop.social' as recognized mastodon instance",
|
||||
"#": "(run 'gallery-dl oauth:mastodon:tabletop.social to get an access token')",
|
||||
"tabletop.social":
|
||||
{
|
||||
"root": "https://tabletop.social",
|
||||
"access-token": "513a36c6..."
|
||||
},
|
||||
|
||||
"#": "set filename format strings for all 'mastodon' instances",
|
||||
"directory": ["mastodon", "{instance}", "{account[username]!l}"],
|
||||
"filename" : "{id}_{media[id]}.{extension}"
|
||||
},
|
||||
|
||||
"foolslide": {
|
||||
"#": "add two more foolslide instances",
|
||||
"otscans" : {"root": "https://otscans.com/foolslide"},
|
||||
"helvetica": {"root": "https://helveticascans.com/r" }
|
||||
},
|
||||
|
||||
"foolfuuka": {
|
||||
"#": "add two other foolfuuka 4chan archives",
|
||||
"fireden-onion": {"root": "http://ydt6jy2ng3s3xg2e.onion"},
|
||||
"scalearchive" : {"root": "https://archive.scaled.team" }
|
||||
},
|
||||
|
||||
"gelbooru_v01":
|
||||
{
|
||||
"#": "add a custom gelbooru_v01 instance",
|
||||
"#": "this is just an example, this specific instance is already included!",
|
||||
"allgirlbooru": {"root": "https://allgirl.booru.org"},
|
||||
|
||||
"#": "the following options are used for all gelbooru_v01 instances",
|
||||
"tag":
|
||||
{
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Booru", "AllGirlBooru", "Tags", "{bkey}", "{ckey}", "{search_tags}"],
|
||||
"" : ["Booru", "AllGirlBooru", "Tags", "_Unsorted", "{search_tags}"]
|
||||
}
|
||||
},
|
||||
"post":
|
||||
{
|
||||
"directory": ["Booru", "AllGirlBooru", "Posts"]
|
||||
},
|
||||
"archive": "~/gallery-dl/custom-archive-file-for-gelbooru_v01_instances.db",
|
||||
"filename": "{tags}_{id}_{md5}.{extension}",
|
||||
"sleep-request": [0, 1.2]
|
||||
},
|
||||
|
||||
"gelbooru_v02":
|
||||
{
|
||||
"#": "add a custom gelbooru_v02 instance",
|
||||
"#": "this is just an example, this specific instance is already included!",
|
||||
"tbib":
|
||||
{
|
||||
"root": "https://tbib.org",
|
||||
"#": "some sites have different domains for API access",
|
||||
"#": "use the 'api_root' option in addition to the 'root' setting here"
|
||||
}
|
||||
},
|
||||
|
||||
"tbib": {
|
||||
"#": "the following options are only used for TBIB",
|
||||
"#": "gelbooru_v02 has four subcategories at the moment, use custom directory settings for all of these",
|
||||
"tag":
|
||||
{
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Tags", "{bkey}", "{ckey}", "{search_tags}"],
|
||||
"" : ["Other Boorus", "TBIB", "Tags", "_Unsorted", "{search_tags}"]
|
||||
}
|
||||
},
|
||||
"pool":
|
||||
{
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Pools", "{bkey}", "{ckey}", "{pool}"],
|
||||
"" : ["Other Boorus", "TBIB", "Pools", "_Unsorted", "{pool}"]
|
||||
}
|
||||
},
|
||||
"favorite":
|
||||
{
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Favorites", "{bkey}", "{ckey}", "{favorite_id}"],
|
||||
"" : ["Other Boorus", "TBIB", "Favorites", "_Unsorted", "{favorite_id}"]
|
||||
}
|
||||
},
|
||||
"post":
|
||||
{
|
||||
"directory": ["Other Boorus", "TBIB", "Posts"]
|
||||
},
|
||||
"archive": "~/gallery-dl/custom-archive-file-for-TBIB.db",
|
||||
"filename": "{id}_{md5}.{extension}",
|
||||
"sleep-request": [0, 1.2]
|
||||
"ugoira-copy": {
|
||||
"name": "ugoira",
|
||||
"extension": "mkv",
|
||||
"ffmpeg-args": [
|
||||
"-c", "copy"
|
||||
],
|
||||
"libx264-prevent-odd": false,
|
||||
"repeat-last-frame": false
|
||||
}
|
||||
},
|
||||
|
||||
"downloader":
|
||||
"pixiv": {
|
||||
"#": "override global archive setting for pixiv",
|
||||
"archive": "~/gallery-dl/archive-pixiv.sqlite3",
|
||||
|
||||
"#": "set custom directory and filename format strings for all pixiv downloads",
|
||||
"filename": "{id}{num}.{extension}",
|
||||
"directory": ["Pixiv", "Works", "{user[id]}"],
|
||||
"refresh-token": "aBcDeFgHiJkLmNoPqRsTuVwXyZ01234567890-FedC9",
|
||||
|
||||
"#": "transform ugoira into lossless MKVs",
|
||||
"ugoira": true,
|
||||
"postprocessors": ["ugoira-copy"]
|
||||
},
|
||||
"#": "use special settings for pixiv favorites and bookmarks",
|
||||
"pixiv:favorite": {
|
||||
"directory": ["Pixiv", "Favorites", "{user[id]}"]
|
||||
},
|
||||
"pixiv:bookmark": {
|
||||
"directory": ["Pixiv", "My Bookmarks"],
|
||||
"refresh-token": "01234567890aBcDeFgHiJkLmNoPqRsTuVwXyZ-ZyxW1"
|
||||
},
|
||||
|
||||
"danbooru": {
|
||||
"ugoira": true,
|
||||
"postprocessors": ["ugoira-webm"]
|
||||
},
|
||||
|
||||
"exhentai": {
|
||||
"#": "use cookies instead of logging in with username and password",
|
||||
"cookies":
|
||||
{
|
||||
"ipb_member_id": "12345",
|
||||
"ipb_pass_hash": "1234567890abcdef",
|
||||
"igneous" : "123456789",
|
||||
"hath_perks" : "m1.m2.m3.a-123456789a",
|
||||
"sk" : "n4m34tv3574m2c4e22c35zgeehiw",
|
||||
"sl" : "dm_2"
|
||||
},
|
||||
|
||||
"#": "wait 2 to 4.8 seconds between HTTP requests",
|
||||
"sleep-request": [2.0, 4.8],
|
||||
|
||||
"filename": "{num:>04}_{name}.{extension}",
|
||||
"directory": ["{category!c}", "{title}"]
|
||||
},
|
||||
|
||||
"sankaku": {
|
||||
"#": "authentication with cookies is not possible for sankaku",
|
||||
"username": "user",
|
||||
"password": "#secret#"
|
||||
},
|
||||
|
||||
"furaffinity": {
|
||||
"#": "authentication with username and password is not possible due to CAPTCHA",
|
||||
"cookies": {
|
||||
"a": "01234567-89ab-cdef-fedc-ba9876543210",
|
||||
"b": "fedcba98-7654-3210-0123-456789abcdef"
|
||||
},
|
||||
"descriptions": "html",
|
||||
"postprocessors": [
|
||||
"content"
|
||||
]
|
||||
},
|
||||
|
||||
"deviantart": {
|
||||
"#": "download 'gallery' and 'scraps' images for user profile URLs",
|
||||
"include": "gallery,scraps",
|
||||
|
||||
"#": "use custom API credentials to avoid 429 errors",
|
||||
"client-id": "98765",
|
||||
"client-secret": "0123456789abcdef0123456789abcdef",
|
||||
"refresh-token": "0123456789abcdef0123456789abcdef01234567",
|
||||
|
||||
"#": "put description texts into a separate directory",
|
||||
"metadata": true,
|
||||
"postprocessors": [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "custom",
|
||||
"directory" : "Descriptions",
|
||||
"content-format" : "{description}\n",
|
||||
"extension-format": "descr.txt"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"flickr": {
|
||||
"access-token": "1234567890-abcdef",
|
||||
"access-token-secret": "1234567890abcdef",
|
||||
"size-max": 1920
|
||||
},
|
||||
|
||||
"mangadex": {
|
||||
"#": "only download safe/suggestive chapters translated to English",
|
||||
"lang": "en",
|
||||
"ratings": ["safe", "suggestive"],
|
||||
|
||||
"#": "put chapters into '.cbz' archives",
|
||||
"postprocessors": ["cbz"]
|
||||
},
|
||||
|
||||
"reddit":
|
||||
{
|
||||
"#": "only spawn child extractors for links to specific sites",
|
||||
"whitelist": ["imgur", "redgifs", "gfycat"],
|
||||
|
||||
"#": "put files from child extractors into the reddit directory",
|
||||
"parent-directory": true,
|
||||
|
||||
"#": "transfer metadata to any child extractor as '_reddit'",
|
||||
"parent-metadata": "_reddit"
|
||||
},
|
||||
|
||||
"imgur":
|
||||
{
|
||||
"#": "use different directory and filename formats when coming from a reddit post",
|
||||
"directory":
|
||||
{
|
||||
"'_reddit' in locals()": []
|
||||
},
|
||||
"filename":
|
||||
{
|
||||
"'_reddit' in locals()": "{_reddit[id]} {id}.{extension}",
|
||||
"" : "{id}.{extension}"
|
||||
}
|
||||
},
|
||||
|
||||
"tumblr": {
|
||||
"posts" : "all",
|
||||
"external": false,
|
||||
"reblogs" : false,
|
||||
"inline" : true
|
||||
},
|
||||
"#": "use special settings when downloading liked tumblr posts",
|
||||
"tumblr:likes": {
|
||||
"posts" : "video,photo,link",
|
||||
"external": true,
|
||||
"reblogs" : true
|
||||
},
|
||||
|
||||
"twitter": {
|
||||
"#": "write text content for *all* tweets",
|
||||
"postprocessors": ["content"],
|
||||
"text-tweets": true
|
||||
},
|
||||
|
||||
"mastodon": {
|
||||
"#": "set filename format strings for all 'mastodon' instances",
|
||||
"directory": ["mastodon", "{instance}", "{account[username]!l}"],
|
||||
"filename" : "{id}_{media[id]}.{extension}"
|
||||
},
|
||||
|
||||
"mastodon:instances": {
|
||||
"#": "add 'tabletop.social' as recognized mastodon instance",
|
||||
"#": "(run 'gallery-dl oauth:mastodon:tabletop.social to get an access token')",
|
||||
"tabletop.social":
|
||||
{
|
||||
"root": "https://tabletop.social",
|
||||
"access-token": "513a36c6..."
|
||||
}
|
||||
},
|
||||
|
||||
"foolslide:instances": {
|
||||
"#": "add two more foolslide instances",
|
||||
"otscans" : {"root": "https://otscans.com/foolslide"},
|
||||
"helvetica": {"root": "https://helveticascans.com/r" }
|
||||
},
|
||||
|
||||
"foolfuuka:instances": {
|
||||
"#": "add two other foolfuuka 4chan archives",
|
||||
"fireden-onion": {"root": "http://ydt6jy2ng3s3xg2e.onion"},
|
||||
"scalearchive" : {"root": "https://archive.scaled.team" }
|
||||
},
|
||||
|
||||
"gelbooru_v01:instances": {
|
||||
"#": "add a custom gelbooru_v01 instance",
|
||||
"#": "this is just an example, this specific instance is already included!",
|
||||
"allgirlbooru": {"root": "https://allgirl.booru.org"}
|
||||
},
|
||||
|
||||
"gelbooru_v01": {
|
||||
"#": "the following options are used for all gelbooru_v01 instances",
|
||||
"archive": "~/gallery-dl/custom-archive-file-for-gelbooru_v01_instances.db",
|
||||
"filename": "{tags}_{id}_{md5}.{extension}",
|
||||
"sleep-request": [0, 1.2]
|
||||
},
|
||||
"gelbooru_v01:tag": {
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Booru", "AllGirlBooru", "Tags", "{bkey}", "{ckey}", "{search_tags}"],
|
||||
"" : ["Booru", "AllGirlBooru", "Tags", "_Unsorted", "{search_tags}"]
|
||||
}
|
||||
},
|
||||
"gelbooru_v01:post": {
|
||||
"directory": ["Booru", "AllGirlBooru", "Posts"]
|
||||
},
|
||||
|
||||
"gelbooru_v02:instances": {
|
||||
"#": "add a custom gelbooru_v02 instance",
|
||||
"#": "this is just an example, this specific instance is already included!",
|
||||
"tbib":
|
||||
{
|
||||
"root": "https://tbib.org",
|
||||
"#": "some sites have different domains for API access",
|
||||
"#": "use the 'api_root' option in addition to the 'root' setting here"
|
||||
}
|
||||
},
|
||||
|
||||
"tbib": {
|
||||
"#": "the following options are only used for TBIB",
|
||||
"#": "gelbooru_v02 has four subcategories at the moment, use custom directory settings for all of these",
|
||||
"archive": "~/gallery-dl/custom-archive-file-for-TBIB.db",
|
||||
"filename": "{id}_{md5}.{extension}",
|
||||
"sleep-request": [0, 1.2]
|
||||
},
|
||||
"tbib:tag": {
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Tags", "{bkey}", "{ckey}", "{search_tags}"],
|
||||
"" : ["Other Boorus", "TBIB", "Tags", "_Unsorted", "{search_tags}"]
|
||||
}
|
||||
},
|
||||
"tbib:pool": {
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Pools", "{bkey}", "{ckey}", "{pool}"],
|
||||
"" : ["Other Boorus", "TBIB", "Pools", "_Unsorted", "{pool}"]
|
||||
}
|
||||
},
|
||||
"tbib:favorite": {
|
||||
"directory": {
|
||||
"locals().get('bkey')": ["Other Boorus", "TBIB", "Favorites", "{bkey}", "{ckey}", "{favorite_id}"],
|
||||
"" : ["Other Boorus", "TBIB", "Favorites", "_Unsorted", "{favorite_id}"]
|
||||
}
|
||||
},
|
||||
"tbib:post": {
|
||||
"directory": ["Other Boorus", "TBIB", "Posts"]
|
||||
},
|
||||
|
||||
"downloader": {
|
||||
"#": "restrict download speed to 1 MB/s",
|
||||
"rate": "1M",
|
||||
|
||||
@@ -307,17 +347,14 @@
|
||||
"part-directory": "/tmp/.download/",
|
||||
|
||||
"#": "do not update file modification times",
|
||||
"mtime": false,
|
||||
|
||||
"ytdl":
|
||||
{
|
||||
"#": "use yt-dlp instead of youtube-dl",
|
||||
"module": "yt_dlp"
|
||||
}
|
||||
"mtime": false
|
||||
},
|
||||
"downloader:ytdl": {
|
||||
"#": "use yt-dlp instead of youtube-dl",
|
||||
"module": "yt_dlp"
|
||||
},
|
||||
|
||||
"output":
|
||||
{
|
||||
"output": {
|
||||
"log": {
|
||||
"level": "info",
|
||||
|
||||
@@ -350,61 +387,6 @@
|
||||
}
|
||||
},
|
||||
|
||||
"postprocessor":
|
||||
{
|
||||
"#": "write 'content' metadata into separate files",
|
||||
"content":
|
||||
{
|
||||
"name" : "metadata",
|
||||
|
||||
"#": "write data for every post instead of each individual file",
|
||||
"event": "post",
|
||||
"filename": "{post_id|tweet_id|id}.txt",
|
||||
|
||||
"#": "write only the values for 'content' or 'description'",
|
||||
"mode" : "custom",
|
||||
"format": "{content|description}\n"
|
||||
},
|
||||
|
||||
"#": "put files into a '.cbz' archive",
|
||||
"cbz":
|
||||
{
|
||||
"name": "zip",
|
||||
"extension": "cbz"
|
||||
},
|
||||
|
||||
"#": "various ugoira post processor configurations to create different file formats",
|
||||
"ugoira-webm":
|
||||
{
|
||||
"name": "ugoira",
|
||||
"extension": "webm",
|
||||
"ffmpeg-args": ["-c:v", "libvpx-vp9", "-an", "-b:v", "0", "-crf", "30"],
|
||||
"ffmpeg-twopass": true,
|
||||
"ffmpeg-demuxer": "image2"
|
||||
},
|
||||
"ugoira-mp4":
|
||||
{
|
||||
"name": "ugoira",
|
||||
"extension": "mp4",
|
||||
"ffmpeg-args": ["-c:v", "libx264", "-an", "-b:v", "4M", "-preset", "veryslow"],
|
||||
"ffmpeg-twopass": true,
|
||||
"libx264-prevent-odd": true
|
||||
},
|
||||
"ugoira-gif":
|
||||
{
|
||||
"name": "ugoira",
|
||||
"extension": "gif",
|
||||
"ffmpeg-args": ["-filter_complex", "[0:v] split [a][b];[a] palettegen [p];[b][p] paletteuse"]
|
||||
},
|
||||
"ugoira-copy": {
|
||||
"name": "ugoira",
|
||||
"extension": "mkv",
|
||||
"ffmpeg-args": ["-c", "copy"],
|
||||
"libx264-prevent-odd": false,
|
||||
"repeat-last-frame": false
|
||||
}
|
||||
},
|
||||
|
||||
"#": "use a custom cache file location",
|
||||
"cache": {
|
||||
"file": "~/gallery-dl/cache.sqlite3"
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"extractor":
|
||||
{
|
||||
"general": {
|
||||
"base-directory": "./gallery-dl/",
|
||||
"parent-directory": false,
|
||||
"postprocessors": null,
|
||||
@@ -9,319 +8,276 @@
|
||||
"cookies-update": true,
|
||||
"proxy": null,
|
||||
"skip": true,
|
||||
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
|
||||
"retries": 4,
|
||||
"timeout": 30.0,
|
||||
"verify": true,
|
||||
"fallback": true,
|
||||
|
||||
"sleep": 0,
|
||||
"sleep-request": 0,
|
||||
"sleep-extractor": 0,
|
||||
|
||||
"path-restrict": "auto",
|
||||
"path-replace": "_",
|
||||
"path-remove": "\\u0000-\\u001f\\u007f",
|
||||
"path-strip": "auto",
|
||||
"extension-map": {
|
||||
"jpeg": "jpg",
|
||||
"jpe" : "jpg",
|
||||
"jpe": "jpg",
|
||||
"jfif": "jpg",
|
||||
"jif" : "jpg",
|
||||
"jfi" : "jpg"
|
||||
"jif": "jpg",
|
||||
"jfi": "jpg"
|
||||
},
|
||||
|
||||
|
||||
"artstation":
|
||||
{
|
||||
"external": false
|
||||
},
|
||||
"aryion":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"recursive": true
|
||||
},
|
||||
"bbc": {
|
||||
"width": 1920
|
||||
},
|
||||
"blogger":
|
||||
{
|
||||
"videos": true
|
||||
},
|
||||
"cyberdrop":
|
||||
{
|
||||
"domain": "auto"
|
||||
},
|
||||
"danbooru":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"external": false,
|
||||
"metadata": false,
|
||||
"ugoira": false
|
||||
},
|
||||
"derpibooru":
|
||||
{
|
||||
"api-key": null,
|
||||
"filter": 56027
|
||||
},
|
||||
"deviantart":
|
||||
{
|
||||
"client-id": null,
|
||||
"client-secret": null,
|
||||
"comments": false,
|
||||
"extra": false,
|
||||
"flat": true,
|
||||
"folders": false,
|
||||
"include": "gallery",
|
||||
"journals": "html",
|
||||
"mature": true,
|
||||
"metadata": false,
|
||||
"original": true,
|
||||
"wait-min": 0
|
||||
},
|
||||
"e621":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"exhentai":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"domain": "auto",
|
||||
"limits": true,
|
||||
"metadata": false,
|
||||
"original": true,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"flickr":
|
||||
{
|
||||
"videos": true,
|
||||
"size-max": null
|
||||
},
|
||||
"furaffinity":
|
||||
{
|
||||
"descriptions": "text",
|
||||
"external": false,
|
||||
"include": "gallery",
|
||||
"layout": "auto"
|
||||
},
|
||||
"gfycat":
|
||||
{
|
||||
"format": ["mp4", "webm", "mobile", "gif"]
|
||||
},
|
||||
"hentaifoundry":
|
||||
{
|
||||
"include": "pictures"
|
||||
},
|
||||
"hitomi":
|
||||
{
|
||||
"format": "webp",
|
||||
"metadata": false
|
||||
},
|
||||
"idolcomplex":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"imgbb":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"imgur":
|
||||
{
|
||||
"mp4": true
|
||||
},
|
||||
"inkbunny":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"orderby": "create_datetime"
|
||||
},
|
||||
"instagram":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"include": "posts",
|
||||
"sleep-request": 8.0,
|
||||
"videos": true
|
||||
},
|
||||
"khinsider":
|
||||
{
|
||||
"format": "mp3"
|
||||
},
|
||||
"luscious":
|
||||
{
|
||||
"gif": false
|
||||
},
|
||||
"mangadex":
|
||||
{
|
||||
"api-server": "https://api.mangadex.org",
|
||||
"api-parameters": null,
|
||||
"lang": null,
|
||||
"ratings": ["safe", "suggestive", "erotica", "pornographic"]
|
||||
},
|
||||
"mangoxo":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"newgrounds":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"flash": true,
|
||||
"format": "original",
|
||||
"include": "art"
|
||||
},
|
||||
"nijie":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"include": "illustration,doujin"
|
||||
},
|
||||
"oauth":
|
||||
{
|
||||
"browser": true,
|
||||
"cache": true,
|
||||
"port": 6414
|
||||
},
|
||||
"pillowfort":
|
||||
{
|
||||
"external": false,
|
||||
"inline": true,
|
||||
"reblogs": false
|
||||
},
|
||||
"pinterest":
|
||||
{
|
||||
"sections": true,
|
||||
"videos": true
|
||||
},
|
||||
"pixiv":
|
||||
{
|
||||
"refresh-token": null,
|
||||
"include": "artworks",
|
||||
"tags": "japanese",
|
||||
"ugoira": true
|
||||
},
|
||||
"reactor":
|
||||
{
|
||||
"gif": false,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"reddit":
|
||||
{
|
||||
"comments": 0,
|
||||
"morecomments": false,
|
||||
"date-min": 0,
|
||||
"date-max": 253402210800,
|
||||
"date-format": "%Y-%m-%dT%H:%M:%S",
|
||||
"id-min": "0",
|
||||
"id-max": "zik0zj",
|
||||
"recursion": 0,
|
||||
"videos": true
|
||||
},
|
||||
"redgifs":
|
||||
{
|
||||
"format": ["hd", "sd", "gif"]
|
||||
},
|
||||
"sankakucomplex":
|
||||
{
|
||||
"embeds": false,
|
||||
"videos": true
|
||||
},
|
||||
"sankaku":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"smugmug":
|
||||
{
|
||||
"videos": true
|
||||
},
|
||||
"seiga":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"subscribestar":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"tsumino":
|
||||
{
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"tumblr":
|
||||
{
|
||||
"avatar": false,
|
||||
"external": false,
|
||||
"inline": true,
|
||||
"posts": "all",
|
||||
"reblogs": true
|
||||
},
|
||||
"twitter":
|
||||
{
|
||||
"username": null,
|
||||
"password": null,
|
||||
"cards": false,
|
||||
"conversations": false,
|
||||
"pinned": false,
|
||||
"quoted": false,
|
||||
"replies": true,
|
||||
"retweets": false,
|
||||
"text-tweets": false,
|
||||
"twitpic": false,
|
||||
"users": "timeline",
|
||||
"videos": true
|
||||
},
|
||||
"unsplash":
|
||||
{
|
||||
"format": "raw"
|
||||
},
|
||||
"vsco":
|
||||
{
|
||||
"videos": true
|
||||
},
|
||||
"wallhaven":
|
||||
{
|
||||
"api-key": null
|
||||
},
|
||||
"weasyl":
|
||||
{
|
||||
"api-key": null,
|
||||
"metadata": false
|
||||
},
|
||||
"weibo":
|
||||
{
|
||||
"retweets": true,
|
||||
"videos": true
|
||||
},
|
||||
"ytdl":
|
||||
{
|
||||
"enabled": false,
|
||||
"format": null,
|
||||
"generic": true,
|
||||
"logging": true,
|
||||
"module": null,
|
||||
"raw-options": null
|
||||
},
|
||||
"booru":
|
||||
{
|
||||
"tags": false,
|
||||
"notes": false
|
||||
}
|
||||
"netrc": false
|
||||
},
|
||||
|
||||
"downloader":
|
||||
{
|
||||
"artstation": {
|
||||
"external": false
|
||||
},
|
||||
"aryion": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"recursive": true
|
||||
},
|
||||
"bbc": {
|
||||
"width": 1920
|
||||
},
|
||||
"blogger": {
|
||||
"videos": true
|
||||
},
|
||||
"danbooru": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"external": false,
|
||||
"metadata": false,
|
||||
"ugoira": false
|
||||
},
|
||||
"derpibooru": {
|
||||
"api-key": null,
|
||||
"filter": 56027
|
||||
},
|
||||
"deviantart": {
|
||||
"client-id": null,
|
||||
"client-secret": null,
|
||||
"comments": false,
|
||||
"extra": false,
|
||||
"flat": true,
|
||||
"folders": false,
|
||||
"include": "gallery",
|
||||
"journals": "html",
|
||||
"mature": true,
|
||||
"metadata": false,
|
||||
"original": true,
|
||||
"wait-min": 0
|
||||
},
|
||||
"e621": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"exhentai": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"domain": "auto",
|
||||
"limits": true,
|
||||
"metadata": false,
|
||||
"original": true,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"flickr": {
|
||||
"videos": true,
|
||||
"size-max": null
|
||||
},
|
||||
"furaffinity": {
|
||||
"descriptions": "text",
|
||||
"external": false,
|
||||
"include": "gallery",
|
||||
"layout": "auto"
|
||||
},
|
||||
"gfycat": {
|
||||
"format": [
|
||||
"mp4",
|
||||
"webm",
|
||||
"mobile",
|
||||
"gif"
|
||||
]
|
||||
},
|
||||
"hentaifoundry": {
|
||||
"include": "pictures"
|
||||
},
|
||||
"hitomi": {
|
||||
"format": "webp",
|
||||
"metadata": false
|
||||
},
|
||||
"idolcomplex": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"imgbb": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"imgur": {
|
||||
"mp4": true
|
||||
},
|
||||
"inkbunny": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"orderby": "create_datetime"
|
||||
},
|
||||
"instagram": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"include": "posts",
|
||||
"sleep-request": 8.0,
|
||||
"videos": true
|
||||
},
|
||||
"khinsider": {
|
||||
"format": "mp3"
|
||||
},
|
||||
"luscious": {
|
||||
"gif": false
|
||||
},
|
||||
"mangadex": {
|
||||
"api-server": "https://api.mangadex.org",
|
||||
"api-parameters": null,
|
||||
"lang": null,
|
||||
"ratings": [
|
||||
"safe",
|
||||
"suggestive",
|
||||
"erotica",
|
||||
"pornographic"
|
||||
]
|
||||
},
|
||||
"mangoxo": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"newgrounds": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"flash": true,
|
||||
"format": "original",
|
||||
"include": "art"
|
||||
},
|
||||
"nijie": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"include": "illustration,doujin"
|
||||
},
|
||||
"oauth": {
|
||||
"browser": true,
|
||||
"cache": true,
|
||||
"port": 6414
|
||||
},
|
||||
"pillowfort": {
|
||||
"external": false,
|
||||
"inline": true,
|
||||
"reblogs": false
|
||||
},
|
||||
"pinterest": {
|
||||
"sections": true,
|
||||
"videos": true
|
||||
},
|
||||
"pixiv": {
|
||||
"refresh-token": null,
|
||||
"avatar": false,
|
||||
"tags": "japanese",
|
||||
"ugoira": true
|
||||
},
|
||||
"reactor": {
|
||||
"gif": false,
|
||||
"sleep-request": 5.0
|
||||
},
|
||||
"reddit": {
|
||||
"comments": 0,
|
||||
"morecomments": false,
|
||||
"date-min": 0,
|
||||
"date-max": 253402210800,
|
||||
"date-format": "%Y-%m-%dT%H:%M:%S",
|
||||
"id-min": "0",
|
||||
"id-max": "zik0zj",
|
||||
"recursion": 0,
|
||||
"videos": true
|
||||
},
|
||||
"redgifs": {
|
||||
"format": [
|
||||
"hd",
|
||||
"sd",
|
||||
"gif"
|
||||
]
|
||||
},
|
||||
"sankakucomplex": {
|
||||
"embeds": false,
|
||||
"videos": true
|
||||
},
|
||||
"sankaku": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"smugmug": {
|
||||
"videos": true
|
||||
},
|
||||
"seiga": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"subscribestar": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"tsumino": {
|
||||
"username": null,
|
||||
"password": null
|
||||
},
|
||||
"tumblr": {
|
||||
"avatar": false,
|
||||
"external": false,
|
||||
"inline": true,
|
||||
"posts": "all",
|
||||
"reblogs": true
|
||||
},
|
||||
"twitter": {
|
||||
"username": null,
|
||||
"password": null,
|
||||
"cards": true,
|
||||
"conversations": false,
|
||||
"pinned": false,
|
||||
"quoted": false,
|
||||
"replies": true,
|
||||
"retweets": false,
|
||||
"text-tweets": false,
|
||||
"twitpic": false,
|
||||
"users": "timeline",
|
||||
"videos": true
|
||||
},
|
||||
"unsplash": {
|
||||
"format": "raw"
|
||||
},
|
||||
"vsco": {
|
||||
"videos": true
|
||||
},
|
||||
"wallhaven": {
|
||||
"api-key": null
|
||||
},
|
||||
"weasyl": {
|
||||
"api-key": null
|
||||
},
|
||||
"weibo": {
|
||||
"retweets": true,
|
||||
"videos": true
|
||||
},
|
||||
"ytdl": {
|
||||
"enabled": false,
|
||||
"format": null,
|
||||
"generic": true,
|
||||
"logging": true,
|
||||
"module": null,
|
||||
"raw-options": null
|
||||
},
|
||||
"booru": {
|
||||
"tags": false,
|
||||
"notes": false
|
||||
},
|
||||
"downloader": {
|
||||
"filesize-min": null,
|
||||
"filesize-max": null,
|
||||
"mtime": true,
|
||||
@@ -331,27 +287,21 @@
|
||||
"rate": null,
|
||||
"retries": 4,
|
||||
"timeout": 30.0,
|
||||
"verify": true,
|
||||
|
||||
"http":
|
||||
{
|
||||
"adjust-extensions": true,
|
||||
"headers": null
|
||||
},
|
||||
|
||||
"ytdl":
|
||||
{
|
||||
"format": null,
|
||||
"forward-cookies": false,
|
||||
"logging": true,
|
||||
"module": null,
|
||||
"outtmpl": null,
|
||||
"raw-options": null
|
||||
}
|
||||
"verify": true
|
||||
},
|
||||
|
||||
"output":
|
||||
{
|
||||
"downloader:http": {
|
||||
"adjust-extensions": true,
|
||||
"headers": null
|
||||
},
|
||||
"downloader:ytdl": {
|
||||
"format": null,
|
||||
"forward-cookies": false,
|
||||
"logging": true,
|
||||
"module": null,
|
||||
"outtmpl": null,
|
||||
"raw-options": null
|
||||
},
|
||||
"output": {
|
||||
"mode": "auto",
|
||||
"progress": true,
|
||||
"shorten": true,
|
||||
@@ -363,7 +313,5 @@
|
||||
"log": "[{name}][{levelname}] {message}",
|
||||
"logfile": null,
|
||||
"unsupportedfile": null
|
||||
},
|
||||
|
||||
"netrc": false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,8 +89,8 @@ def parse_inputfile(file, log):
|
||||
log.warning("input file: unable to parse '%s': %s", value, exc)
|
||||
continue
|
||||
|
||||
key = key.strip().split(".")
|
||||
conf.append((key[:-1], key[-1], value))
|
||||
section, sep, key = key.strip().rpartition(":")
|
||||
conf.append((section if sep else "__global__", key, value))
|
||||
|
||||
else:
|
||||
# url
|
||||
@@ -111,6 +111,9 @@ def main():
|
||||
args = parser.parse_args()
|
||||
log = output.initialize_logging(args.loglevel)
|
||||
|
||||
if args.config:
|
||||
config._warn_legacy = False
|
||||
|
||||
# configuration
|
||||
if args.load_config:
|
||||
config.load()
|
||||
@@ -124,25 +127,25 @@ def main():
|
||||
filename = "{filename}.{extension}"
|
||||
elif filename.startswith("\\f"):
|
||||
filename = "\f" + filename[2:]
|
||||
config.set((), "filename", filename)
|
||||
config.set("__global__", "filename", filename)
|
||||
if args.directory:
|
||||
config.set((), "base-directory", args.directory)
|
||||
config.set((), "directory", ())
|
||||
config.set("__global__", "base-directory", args.directory)
|
||||
config.set("__global__", "directory", ())
|
||||
if args.postprocessors:
|
||||
config.set((), "postprocessors", args.postprocessors)
|
||||
config.set("__global__", "postprocessors", args.postprocessors)
|
||||
if args.abort:
|
||||
config.set((), "skip", "abort:" + str(args.abort))
|
||||
config.set("__global__", "skip", f"abort:{args.abort}")
|
||||
if args.terminate:
|
||||
config.set((), "skip", "terminate:" + str(args.terminate))
|
||||
config.set("__global__", "skip", f"terminate:{args.terminate}")
|
||||
if args.cookies_from_browser:
|
||||
browser, _, profile = args.cookies_from_browser.partition(":")
|
||||
browser, _, keyring = browser.partition("+")
|
||||
config.set((), "cookies", (browser, profile, keyring))
|
||||
config.set("__global__", "cookies", (browser, profile, keyring))
|
||||
for opts in args.options:
|
||||
config.set(*opts)
|
||||
|
||||
# signals
|
||||
signals = config.get((), "signals-ignore")
|
||||
signals = config.interpolate(("general",), "signals-ignore")
|
||||
if signals:
|
||||
import signal
|
||||
if isinstance(signals, str):
|
||||
@@ -155,7 +158,7 @@ def main():
|
||||
signal.signal(signal_num, signal.SIG_IGN)
|
||||
|
||||
# extractor modules
|
||||
modules = config.get(("extractor",), "modules")
|
||||
modules = config.interpolate(("general", "extractor"), "modules")
|
||||
if modules is not None:
|
||||
if isinstance(modules, str):
|
||||
modules = modules.split(",")
|
||||
@@ -228,6 +231,19 @@ def main():
|
||||
"Deleted %d %s from '%s'",
|
||||
cnt, "entry" if cnt == 1 else "entries", cache._path(),
|
||||
)
|
||||
elif args.config:
|
||||
if not args.load_config:
|
||||
del config._default_configs[:]
|
||||
if args.cfgfiles:
|
||||
config._default_configs.extend(args.cfgfiles)
|
||||
if args.config == "init":
|
||||
return config.config_init()
|
||||
if args.config == "open":
|
||||
return config.config_open()
|
||||
if args.config == "status":
|
||||
return config.config_status()
|
||||
if args.config == "update":
|
||||
return config.config_update()
|
||||
else:
|
||||
if not args.urls and not args.inputfiles:
|
||||
parser.error(
|
||||
@@ -237,7 +253,7 @@ def main():
|
||||
if args.list_urls:
|
||||
jobtype = job.UrlJob
|
||||
jobtype.maxdepth = args.list_urls
|
||||
if config.get(("output",), "fallback", True):
|
||||
if config.get("output", "fallback", True):
|
||||
jobtype.handle_url = \
|
||||
staticmethod(jobtype.handle_url_fallback)
|
||||
else:
|
||||
@@ -251,7 +267,8 @@ def main():
|
||||
if sys.stdin:
|
||||
urls += parse_inputfile(sys.stdin, log)
|
||||
else:
|
||||
log.warning("input file: stdin is not readable")
|
||||
log.warning(
|
||||
"input file: stdin is not readable")
|
||||
else:
|
||||
with open(inputfile, encoding="utf-8") as file:
|
||||
urls += parse_inputfile(file, log)
|
||||
@@ -267,7 +284,7 @@ def main():
|
||||
ulog.propagate = False
|
||||
job.Job.ulog = ulog
|
||||
|
||||
pformat = config.get(("output",), "progress", True)
|
||||
pformat = config.get("output", "progress", True)
|
||||
if pformat and len(urls) > 1 and args.loglevel < logging.ERROR:
|
||||
urls = progress(urls, pformat)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2015-2021 Mike Fährmann
|
||||
# Copyright 2015-2022 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
@@ -8,9 +8,9 @@
|
||||
|
||||
"""Global configuration module"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import os.path
|
||||
import logging
|
||||
from . import util
|
||||
|
||||
@@ -20,7 +20,8 @@ log = logging.getLogger("config")
|
||||
# --------------------------------------------------------------------
|
||||
# internals
|
||||
|
||||
_config = {}
|
||||
_config = {"__global__": {}}
|
||||
_warn_legacy = True
|
||||
|
||||
if util.WINDOWS:
|
||||
_default_configs = [
|
||||
@@ -49,148 +50,47 @@ if getattr(sys, "frozen", False):
|
||||
# --------------------------------------------------------------------
|
||||
# public interface
|
||||
|
||||
def load(files=None, strict=False, fmt="json"):
|
||||
"""Load JSON configuration files"""
|
||||
if fmt == "yaml":
|
||||
try:
|
||||
import yaml
|
||||
parsefunc = yaml.safe_load
|
||||
except ImportError:
|
||||
log.error("Could not import 'yaml' module")
|
||||
return
|
||||
else:
|
||||
parsefunc = json.load
|
||||
|
||||
for path in files or _default_configs:
|
||||
path = util.expand_path(path)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as file:
|
||||
confdict = parsefunc(file)
|
||||
except OSError as exc:
|
||||
if strict:
|
||||
log.error(exc)
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
log.warning("Could not parse '%s': %s", path, exc)
|
||||
if strict:
|
||||
sys.exit(2)
|
||||
else:
|
||||
if not _config:
|
||||
_config.update(confdict)
|
||||
else:
|
||||
util.combine_dict(_config, confdict)
|
||||
|
||||
|
||||
def clear():
|
||||
"""Reset configuration to an empty state"""
|
||||
_config.clear()
|
||||
|
||||
|
||||
def get(path, key, default=None, *, conf=_config):
|
||||
def get(section, key, default=None, *, conf=_config):
|
||||
"""Get the value of property 'key' or a default value"""
|
||||
try:
|
||||
for p in path:
|
||||
conf = conf[p]
|
||||
return conf[key]
|
||||
return conf[section][key]
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def interpolate(path, key, default=None, *, conf=_config):
|
||||
"""Interpolate the value of 'key'"""
|
||||
if key in conf:
|
||||
return conf[key]
|
||||
try:
|
||||
for p in path:
|
||||
conf = conf[p]
|
||||
if key in conf:
|
||||
default = conf[key]
|
||||
except Exception:
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def interpolate_common(common, paths, key, default=None, *, conf=_config):
|
||||
"""Interpolate the value of 'key'
|
||||
using multiple 'paths' along a 'common' ancestor
|
||||
"""
|
||||
if key in conf:
|
||||
return conf[key]
|
||||
|
||||
# follow the common path
|
||||
try:
|
||||
for p in common:
|
||||
conf = conf[p]
|
||||
if key in conf:
|
||||
default = conf[key]
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
# try all paths until a value is found
|
||||
value = util.SENTINEL
|
||||
for path in paths:
|
||||
c = conf
|
||||
try:
|
||||
for p in path:
|
||||
c = c[p]
|
||||
if key in c:
|
||||
value = c[key]
|
||||
except Exception:
|
||||
pass
|
||||
if value is not util.SENTINEL:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def accumulate(path, key, *, conf=_config):
|
||||
"""Accumulate the values of 'key' along 'path'"""
|
||||
result = []
|
||||
try:
|
||||
if key in conf:
|
||||
value = conf[key]
|
||||
if value:
|
||||
result.extend(value)
|
||||
for p in path:
|
||||
conf = conf[p]
|
||||
if key in conf:
|
||||
value = conf[key]
|
||||
if value:
|
||||
result[:0] = value
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def set(path, key, value, *, conf=_config):
|
||||
def set(section, key, value, *, conf=_config):
|
||||
"""Set the value of property 'key' for this session"""
|
||||
for p in path:
|
||||
try:
|
||||
conf = conf[p]
|
||||
except KeyError:
|
||||
conf[p] = conf = {}
|
||||
conf[key] = value
|
||||
try:
|
||||
conf[section][key] = value
|
||||
except KeyError:
|
||||
conf[section] = {key: value}
|
||||
|
||||
|
||||
def setdefault(path, key, value, *, conf=_config):
|
||||
def setdefault(section, key, value, *, conf=_config):
|
||||
"""Set the value of property 'key' if it doesn't exist"""
|
||||
for p in path:
|
||||
try:
|
||||
conf = conf[p]
|
||||
except KeyError:
|
||||
conf[p] = conf = {}
|
||||
return conf.setdefault(key, value)
|
||||
try:
|
||||
conf[section].setdefault(key, value)
|
||||
except KeyError:
|
||||
conf[section] = {key: value}
|
||||
|
||||
|
||||
def unset(path, key, *, conf=_config):
|
||||
def unset(section, key, *, conf=_config):
|
||||
"""Unset the value of property 'key'"""
|
||||
try:
|
||||
for p in path:
|
||||
conf = conf[p]
|
||||
del conf[key]
|
||||
del conf[section][key]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def interpolate(sections, key, default=None, *, conf=_config):
|
||||
if key in conf["__global__"]:
|
||||
return conf["__global__"][key]
|
||||
for section in sections:
|
||||
if section in conf and key in conf[section]:
|
||||
default = conf[section][key]
|
||||
return default
|
||||
|
||||
|
||||
class apply():
|
||||
"""Context Manager: apply a collection of key-value pairs"""
|
||||
|
||||
@@ -209,3 +109,279 @@ class apply():
|
||||
unset(path, key)
|
||||
else:
|
||||
set(path, key, value)
|
||||
|
||||
|
||||
def load(files=None, strict=False, fmt="json"):
|
||||
"""Load JSON configuration files"""
|
||||
if fmt == "yaml":
|
||||
try:
|
||||
import yaml
|
||||
load = yaml.safe_load
|
||||
except ImportError:
|
||||
log.error("Could not import 'yaml' module")
|
||||
return
|
||||
else:
|
||||
load = json.load
|
||||
|
||||
for path in files or _default_configs:
|
||||
path = util.expand_path(path)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fp:
|
||||
config_dict = load(fp)
|
||||
except OSError as exc:
|
||||
if strict:
|
||||
log.error(exc)
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
log.warning("Could not parse '%s': %s", path, exc)
|
||||
if strict:
|
||||
sys.exit(2)
|
||||
else:
|
||||
if "extractor" in config_dict:
|
||||
if _warn_legacy:
|
||||
log.warning("Legacy config file found at %s", path)
|
||||
log.warning("Run 'gallery-dl --config-update' or follow "
|
||||
"<link> to update to the new format")
|
||||
config_dict = update_config_dict(config_dict)
|
||||
|
||||
if not _config:
|
||||
_config.update(config_dict)
|
||||
else:
|
||||
util.combine_dict(_config, config_dict)
|
||||
|
||||
|
||||
def clear():
|
||||
"""Reset configuration to an empty state"""
|
||||
_config.clear()
|
||||
_config["__global__"] = {}
|
||||
|
||||
|
||||
def build_options_dict(keys, conf=_config):
|
||||
opts = {}
|
||||
update = opts.update
|
||||
for key in keys:
|
||||
if key in conf:
|
||||
update(conf[key])
|
||||
if "__global__" in conf:
|
||||
update(conf["__global__"])
|
||||
return opts
|
||||
|
||||
|
||||
def build_extractor_options_dict(extr):
|
||||
ccat = extr.category
|
||||
bcat = extr.basecategory
|
||||
subcat = extr.subcategory
|
||||
|
||||
if bcat:
|
||||
keys = (
|
||||
"general",
|
||||
bcat,
|
||||
bcat + ":" + subcat,
|
||||
ccat,
|
||||
ccat + ":" + subcat,
|
||||
)
|
||||
else:
|
||||
keys = (
|
||||
"general",
|
||||
ccat,
|
||||
ccat + ":" + subcat,
|
||||
)
|
||||
|
||||
return build_options_dict(keys)
|
||||
|
||||
|
||||
def build_module_options_dict(extr, package, module, conf=_config):
|
||||
ccat = extr.category
|
||||
bcat = extr.basecategory
|
||||
subcat = extr.subcategory
|
||||
module = package + ":" + module
|
||||
|
||||
if bcat:
|
||||
keys = (
|
||||
package,
|
||||
module,
|
||||
bcat + ":" + package,
|
||||
bcat + ":" + module,
|
||||
bcat + ":" + subcat + ":" + package,
|
||||
bcat + ":" + subcat + ":" + module,
|
||||
ccat + ":" + package,
|
||||
ccat + ":" + module,
|
||||
ccat + ":" + subcat + ":" + package,
|
||||
ccat + ":" + subcat + ":" + module,
|
||||
)
|
||||
else:
|
||||
keys = (
|
||||
package,
|
||||
module,
|
||||
ccat + ":" + package,
|
||||
ccat + ":" + module,
|
||||
ccat + ":" + subcat + ":" + package,
|
||||
ccat + ":" + subcat + ":" + module,
|
||||
)
|
||||
|
||||
return build_options_dict(keys)
|
||||
|
||||
|
||||
def config_init():
|
||||
paths = [
|
||||
util.expand_path(path)
|
||||
for path in _default_configs
|
||||
]
|
||||
|
||||
for path in paths:
|
||||
if os.access(path, os.R_OK):
|
||||
log.error("There is already a configuration file at %s", path)
|
||||
return 1
|
||||
|
||||
for path in paths:
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as fp:
|
||||
fp.write("""\
|
||||
{
|
||||
"general": {
|
||||
|
||||
},
|
||||
"downloader": {
|
||||
|
||||
},
|
||||
"output": {
|
||||
|
||||
}
|
||||
}
|
||||
""")
|
||||
break
|
||||
except OSError as exc:
|
||||
log.debug(exc)
|
||||
else:
|
||||
log.error("Unable to create a new configuration file "
|
||||
"at any of the default paths")
|
||||
return 1
|
||||
|
||||
log.info("Created a basic configuration file at %s", path)
|
||||
|
||||
|
||||
def config_open():
|
||||
for path in _default_configs:
|
||||
path = util.expand_path(path)
|
||||
if os.access(path, os.R_OK | os.W_OK):
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
openers = (("explorer", "notepad")
|
||||
if util.WINDOWS else
|
||||
("xdg-open", "open", os.environ.get("EDITOR", "")))
|
||||
for opener in openers:
|
||||
if opener := shutil.which(opener):
|
||||
break
|
||||
else:
|
||||
log.warning("Unable to find a program to open '%s' with", path)
|
||||
return 1
|
||||
|
||||
log.info("Running '%s %s'", opener, path)
|
||||
return subprocess.Popen((opener, path)).wait()
|
||||
|
||||
log.warning("Unable to find any writable configuration file")
|
||||
return 1
|
||||
|
||||
|
||||
def config_status():
|
||||
for path in _default_configs:
|
||||
path = util.expand_path(path)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fp:
|
||||
config_dict = json.load(fp)
|
||||
except FileNotFoundError:
|
||||
status = "NOT PRESENT"
|
||||
except ValueError:
|
||||
status = "INVALID JSON"
|
||||
except Exception as exc:
|
||||
log.debug(exc)
|
||||
status = "UNKNOWN"
|
||||
else:
|
||||
status = "OK"
|
||||
if "extractor" in config_dict:
|
||||
status += " (legacy)"
|
||||
print(f"{path}: {status}")
|
||||
|
||||
|
||||
def config_update():
|
||||
for path in _default_configs:
|
||||
path = util.expand_path(path)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fp:
|
||||
config_content = fp.read()
|
||||
config_dict = json.loads(config_content)
|
||||
except Exception as exc:
|
||||
log.debug(exc)
|
||||
else:
|
||||
if "extractor" in config_dict:
|
||||
config_dict = update_config_dict(config_dict)
|
||||
|
||||
# write backup
|
||||
with open(path + ".bak", "w", encoding="utf-8") as fp:
|
||||
fp.write(config_content)
|
||||
|
||||
# overwrite with updated JSON
|
||||
with open(path, "w", encoding="utf-8") as fp:
|
||||
json.dump(
|
||||
config_dict, fp,
|
||||
indent=4,
|
||||
ensure_ascii=get("output", "ascii"),
|
||||
)
|
||||
|
||||
log.info("Updated %s", path)
|
||||
log.info("Backup at %s", path + ".bak")
|
||||
|
||||
|
||||
def update_config_dict(config_legacy):
|
||||
# option names that could be a dict
|
||||
optnames = {"filename", "directory", "path-restrict", "cookies",
|
||||
"extension-map", "keywords", "keywords-default", "proxy"}
|
||||
|
||||
config = {"general": {}}
|
||||
|
||||
if extractor := config_legacy.pop("extractor", None):
|
||||
for key, value in extractor.items():
|
||||
if isinstance(value, dict) and key not in optnames:
|
||||
config[key] = value
|
||||
|
||||
delete = []
|
||||
instances = None
|
||||
|
||||
for skey, sval in value.items():
|
||||
if isinstance(sval, dict):
|
||||
|
||||
# basecategory instance
|
||||
if "root" in sval:
|
||||
if instances is None:
|
||||
config[key + ":instances"] = instances = {}
|
||||
instances[skey] = sval
|
||||
delete.append(skey)
|
||||
|
||||
# subcategory options
|
||||
elif skey not in optnames:
|
||||
config[f"{key}:{skey}"] = value[skey]
|
||||
delete.append(skey)
|
||||
|
||||
for skey in delete:
|
||||
del value[skey]
|
||||
|
||||
if not value:
|
||||
del config[key]
|
||||
else:
|
||||
config["general"][key] = value
|
||||
|
||||
if downloader := config_legacy.pop("downloader", None):
|
||||
config["downloader"] = downloader
|
||||
for module in ("http", "ytdl", "text"):
|
||||
if opts := downloader.pop(module, None):
|
||||
config["downloader:" + module] = opts
|
||||
|
||||
for section_name in ("output", "postprocessor", "cache"):
|
||||
if section := config_legacy.pop(section_name, None):
|
||||
config[section_name] = section
|
||||
|
||||
for key, value in config_legacy.items():
|
||||
config["general"][key] = value
|
||||
|
||||
return config
|
||||
|
||||
@@ -17,25 +17,27 @@ class DownloaderBase():
|
||||
scheme = ""
|
||||
|
||||
def __init__(self, job):
|
||||
extr = job.extractor
|
||||
|
||||
self.options = options = config.build_module_options_dict(
|
||||
extr, "downloader", self.scheme)
|
||||
self.config = cfg = options.get
|
||||
|
||||
self.out = job.out
|
||||
self.session = job.extractor.session
|
||||
self.part = self.config("part", True)
|
||||
self.partdir = self.config("part-directory")
|
||||
self.log = job.get_logger("downloader." + self.scheme)
|
||||
self.session = extr.session
|
||||
|
||||
if self.partdir:
|
||||
self.partdir = util.expand_path(self.partdir)
|
||||
self.part = cfg("part", True)
|
||||
if partdir := cfg("part-directory"):
|
||||
self.partdir = util.expand_path(partdir)
|
||||
os.makedirs(self.partdir, exist_ok=True)
|
||||
else:
|
||||
self.partdir = None
|
||||
|
||||
proxies = self.config("proxy", util.SENTINEL)
|
||||
if proxies is util.SENTINEL:
|
||||
if (proxies := cfg("proxy")) is None:
|
||||
self.proxies = job.extractor._proxies
|
||||
else:
|
||||
self.proxies = util.build_proxy_map(proxies, self.log)
|
||||
|
||||
def config(self, key, default=None):
|
||||
"""Interpolate downloader config value for 'key'"""
|
||||
return config.interpolate(("downloader", self.scheme), key, default)
|
||||
|
||||
def download(self, url, pathfmt):
|
||||
"""Write data from 'url' into the file specified by 'pathfmt'"""
|
||||
|
||||
@@ -45,19 +45,17 @@ class Extractor():
|
||||
self.url = match.string
|
||||
self.finalize = None
|
||||
|
||||
if self.basecategory:
|
||||
self.config = self._config_shared
|
||||
self.config_accumulate = self._config_shared_accumulate
|
||||
self._cfgpath = ("extractor", self.category, self.subcategory)
|
||||
self._parentdir = ""
|
||||
self.options = options = config.build_extractor_options_dict(self)
|
||||
self.config = cfg = options.get
|
||||
|
||||
self._write_pages = self.config("write-pages", False)
|
||||
self._retries = self.config("retries", 4)
|
||||
self._timeout = self.config("timeout", 30)
|
||||
self._verify = self.config("verify", True)
|
||||
self._proxies = util.build_proxy_map(self.config("proxy"), self.log)
|
||||
self._parentdir = ""
|
||||
self._write_pages = cfg("write-pages", False)
|
||||
self._retries = cfg("retries", 4)
|
||||
self._timeout = cfg("timeout", 30)
|
||||
self._verify = cfg("verify", True)
|
||||
self._proxies = util.build_proxy_map(cfg("proxy"), self.log)
|
||||
self._interval = util.build_duration_func(
|
||||
self.config("sleep-request", self.request_interval),
|
||||
cfg("sleep-request", self.request_interval),
|
||||
self.request_interval_min,
|
||||
)
|
||||
|
||||
@@ -83,25 +81,6 @@ class Extractor():
|
||||
def skip(self, num):
|
||||
return 0
|
||||
|
||||
def config(self, key, default=None):
|
||||
return config.interpolate(self._cfgpath, key, default)
|
||||
|
||||
def config_accumulate(self, key):
|
||||
return config.accumulate(self._cfgpath, key)
|
||||
|
||||
def _config_shared(self, key, default=None):
|
||||
return config.interpolate_common(("extractor",), (
|
||||
(self.category, self.subcategory),
|
||||
(self.basecategory, self.subcategory),
|
||||
), key, default)
|
||||
|
||||
def _config_shared_accumulate(self, key):
|
||||
values = config.accumulate(self._cfgpath, key)
|
||||
conf = config.get(("extractor",), self.basecategory)
|
||||
if conf:
|
||||
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
|
||||
return values
|
||||
|
||||
def request(self, url, *, method="GET", session=None, retries=None,
|
||||
encoding=None, fatal=True, notfound=None, **kwargs):
|
||||
if session is None:
|
||||
@@ -195,8 +174,8 @@ class Extractor():
|
||||
return
|
||||
|
||||
if reason:
|
||||
t = datetime.datetime.fromtimestamp(until).time()
|
||||
isotime = "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second)
|
||||
isotime = datetime.datetime.fromtimestamp(
|
||||
until).time().isoformat("seconds")
|
||||
self.log.info("Waiting until %s for %s.", isotime, reason)
|
||||
time.sleep(seconds)
|
||||
|
||||
@@ -626,7 +605,7 @@ class BaseExtractor(Extractor):
|
||||
|
||||
@classmethod
|
||||
def update(cls, instances):
|
||||
extra_instances = config.get(("extractor",), cls.basecategory)
|
||||
extra_instances = config._config.get(cls.basecategory + ":instances")
|
||||
if extra_instances:
|
||||
for category, info in extra_instances.items():
|
||||
if isinstance(info, dict) and "root" in info:
|
||||
@@ -763,7 +742,7 @@ SSL_CIPHERS = {
|
||||
|
||||
|
||||
# Undo automatic pyOpenSSL injection by requests
|
||||
pyopenssl = config.get((), "pyopenssl", False)
|
||||
pyopenssl = config.interpolate(("general",), "pyopenssl", False)
|
||||
if not pyopenssl:
|
||||
try:
|
||||
from requests.packages.urllib3.contrib import pyopenssl # noqa
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"""Extractors for https://e-hentai.org/ and https://exhentai.org/"""
|
||||
|
||||
from .common import Extractor, Message
|
||||
from .. import text, util, exception
|
||||
from .. import text, util, config, exception
|
||||
from ..cache import cache
|
||||
import itertools
|
||||
import math
|
||||
@@ -32,11 +32,9 @@ class ExhentaiExtractor(Extractor):
|
||||
LIMIT = False
|
||||
|
||||
def __init__(self, match):
|
||||
# allow calling 'self.config()' before 'Extractor.__init__()'
|
||||
self._cfgpath = ("extractor", self.category, self.subcategory)
|
||||
|
||||
version = match.group(1)
|
||||
domain = self.config("domain", "auto")
|
||||
|
||||
domain = config.interpolate(("exhentai"), "domain", "auto")
|
||||
if domain == "auto":
|
||||
domain = ("ex" if version == "ex" else "e-") + "hentai.org"
|
||||
self.root = "https://" + domain
|
||||
|
||||
@@ -19,7 +19,7 @@ class GenericExtractor(Extractor):
|
||||
# and the "g(eneric):" prefix in url is required.
|
||||
# If the extractor is enabled, make the prefix optional
|
||||
pattern = r"(?ix)(?P<generic>g(?:eneric)?:)"
|
||||
if config.get(("extractor", "generic"), "enabled"):
|
||||
if config.get("generic", "enabled"):
|
||||
pattern += r"?"
|
||||
|
||||
# The generic extractor pattern should match (almost) any valid url
|
||||
|
||||
@@ -29,11 +29,10 @@ class OAuthBase(Extractor):
|
||||
def __init__(self, match):
|
||||
Extractor.__init__(self, match)
|
||||
self.client = None
|
||||
self.cache = config.get(("extractor", self.category), "cache", True)
|
||||
self.cache = self.config("cache", True)
|
||||
|
||||
def oauth_config(self, key, default=None):
|
||||
value = config.interpolate(("extractor", self.subcategory), key)
|
||||
return value if value is not None else default
|
||||
self.oauth_config = config.build_options_dict((
|
||||
"general", self.subcategory)).get
|
||||
|
||||
def recv(self):
|
||||
"""Open local HTTP server and recv callback parameters"""
|
||||
|
||||
@@ -397,8 +397,7 @@ class TumblrAPI(oauth.OAuth1API):
|
||||
# daily rate limit
|
||||
if response.headers.get("x-ratelimit-perday-remaining") == "0":
|
||||
reset = response.headers.get("x-ratelimit-perday-reset")
|
||||
t = (datetime.now() + timedelta(seconds=float(reset))).time()
|
||||
|
||||
until = datetime.now() + timedelta(seconds=float(reset))
|
||||
self.log.error("Daily API rate limit exceeded")
|
||||
|
||||
api_key = self.api_key or self.session.auth.consumer_key
|
||||
@@ -411,7 +410,7 @@ class TumblrAPI(oauth.OAuth1API):
|
||||
|
||||
raise exception.StopExtraction(
|
||||
"Aborting - Rate limit will reset at %s",
|
||||
"{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second))
|
||||
until.time().isoformat("seconds"))
|
||||
|
||||
# hourly rate limit
|
||||
reset = response.headers.get("x-ratelimit-perhour-reset")
|
||||
|
||||
@@ -23,13 +23,12 @@ class YoutubeDLExtractor(Extractor):
|
||||
|
||||
def __init__(self, match):
|
||||
# import main youtube_dl module
|
||||
ytdl_module = ytdl.import_module(config.get(
|
||||
("extractor", "ytdl"), "module"))
|
||||
ytdl_module = ytdl.import_module(config.get("ytdl", "module"))
|
||||
self.ytdl_module_name = ytdl_module.__name__
|
||||
|
||||
# find suitable youtube_dl extractor
|
||||
self.ytdl_url = url = match.group(1)
|
||||
generic = config.interpolate(("extractor", "ytdl"), "generic", True)
|
||||
generic = config.interpolate(("ytdl",), "generic", True)
|
||||
if generic == "force":
|
||||
self.ytdl_ie_key = "Generic"
|
||||
self.force_generic_extractor = True
|
||||
@@ -49,7 +48,7 @@ class YoutubeDLExtractor(Extractor):
|
||||
def items(self):
|
||||
# import subcategory module
|
||||
ytdl_module = ytdl.import_module(
|
||||
config.get(("extractor", "ytdl", self.subcategory), "module") or
|
||||
config.get("ytdl:" + self.subcategory, "module") or
|
||||
self.ytdl_module_name)
|
||||
self.log.debug("Using %s", ytdl_module)
|
||||
|
||||
@@ -135,6 +134,6 @@ class YoutubeDLExtractor(Extractor):
|
||||
yield entry
|
||||
|
||||
|
||||
if config.get(("extractor", "ytdl"), "enabled"):
|
||||
if config.get("ytdl", "enabled"):
|
||||
# make 'ytdl:' prefix optional
|
||||
YoutubeDLExtractor.pattern = r"(?:ytdl:)?(.*)"
|
||||
|
||||
@@ -50,7 +50,8 @@ class Job():
|
||||
|
||||
# transfer (sub)category
|
||||
if pextr.config("category-transfer", pextr.categorytransfer):
|
||||
extr._cfgpath = pextr._cfgpath
|
||||
extr.config = pextr.config
|
||||
extr.options = pextr.options
|
||||
extr.category = pextr.category
|
||||
extr.subcategory = pextr.subcategory
|
||||
|
||||
@@ -439,13 +440,13 @@ class DownloadJob(Job):
|
||||
if self.archive:
|
||||
self.archive.check = pathfmt.exists
|
||||
|
||||
postprocessors = extr.config_accumulate("postprocessors")
|
||||
postprocessors = cfg("postprocessors")
|
||||
if postprocessors:
|
||||
self.hooks = collections.defaultdict(list)
|
||||
pp_log = self.get_logger("postprocessor")
|
||||
pp_list = []
|
||||
|
||||
pp_conf = config.get((), "postprocessor") or {}
|
||||
pp_conf = config._config.get("postprocessor") or {}
|
||||
for pp_dict in postprocessors:
|
||||
if isinstance(pp_dict, str):
|
||||
pp_dict = pp_conf.get(pp_dict) or {"name": pp_dict}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2017-2021 Mike Fährmann
|
||||
# Copyright 2017-2022 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
@@ -18,13 +18,13 @@ from . import job, version
|
||||
class ConfigAction(argparse.Action):
|
||||
"""Set argparse results as config values"""
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
namespace.options.append(((), self.dest, values))
|
||||
namespace.options.append(("__global__", self.dest, values))
|
||||
|
||||
|
||||
class ConfigConstAction(argparse.Action):
|
||||
"""Set argparse const values as config values"""
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
namespace.options.append(((), self.dest, self.const))
|
||||
namespace.options.append(("__global__", self.dest, self.const))
|
||||
|
||||
|
||||
class AppendCommandAction(argparse.Action):
|
||||
@@ -48,13 +48,16 @@ class DeprecatedConfigConstAction(argparse.Action):
|
||||
class ParseAction(argparse.Action):
|
||||
"""Parse <key>=<value> options and set them as config values"""
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
key, _, value = values.partition("=")
|
||||
key, sep, value = values.partition("=")
|
||||
section, sep, key = key.rpartition(":")
|
||||
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except ValueError:
|
||||
pass
|
||||
key = key.split(".") # splitting an empty string becomes [""]
|
||||
namespace.options.append((key[:-1], key[-1], value))
|
||||
|
||||
namespace.options.append(
|
||||
(section if sep else "__global__", key, value))
|
||||
|
||||
|
||||
class Formatter(argparse.HelpFormatter):
|
||||
@@ -295,20 +298,45 @@ def build_parser():
|
||||
dest="cfgfiles", metavar="FILE", action="append",
|
||||
help="Additional configuration files",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-yaml",
|
||||
dest="yamlfiles", metavar="FILE", action="append",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
configuration.add_argument(
|
||||
"-o", "--option",
|
||||
dest="options", metavar="OPT", action=ParseAction, default=[],
|
||||
help="Additional '<key>=<value>' option values",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-yaml",
|
||||
dest="yamlfiles", metavar="FILE", action="append",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-init",
|
||||
dest="config", action="store_const", const="init",
|
||||
help="Create a basic, initial configuration file",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-open",
|
||||
dest="config", action="store_const", const="open",
|
||||
help="Open a configuration file in the user's preferred application",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-status",
|
||||
dest="config", action="store_const", const="status",
|
||||
help="Show configuration file status",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-update",
|
||||
dest="config", action="store_const", const="update",
|
||||
help="Convert legacy configuration files",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--config-ignore",
|
||||
dest="load_config", action="store_false",
|
||||
help="Do not load any default configuration files",
|
||||
)
|
||||
configuration.add_argument(
|
||||
"--ignore-config",
|
||||
dest="load_config", action="store_false",
|
||||
help="Do not read the default configuration files",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
|
||||
authentication = parser.add_argument_group("Authentication Options")
|
||||
|
||||
@@ -147,7 +147,7 @@ def configure_logging(loglevel):
|
||||
|
||||
# stream logging handler
|
||||
handler = root.handlers[0]
|
||||
opts = config.interpolate(("output",), "log")
|
||||
opts = config.get("output", "log")
|
||||
if opts:
|
||||
if isinstance(opts, str):
|
||||
opts = {"format": opts}
|
||||
@@ -173,7 +173,7 @@ def configure_logging(loglevel):
|
||||
|
||||
def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
|
||||
"""Setup a new logging handler"""
|
||||
opts = config.interpolate(("output",), key)
|
||||
opts = config.get("output", key)
|
||||
if not opts:
|
||||
return None
|
||||
if not isinstance(opts, dict):
|
||||
@@ -255,7 +255,7 @@ def select():
|
||||
"color": ColorOutput,
|
||||
"null": NullOutput,
|
||||
}
|
||||
omode = config.get(("output",), "mode", "auto").lower()
|
||||
omode = config.get("output", "mode", "auto").lower()
|
||||
if omode in pdict:
|
||||
output = pdict[omode]()
|
||||
elif omode == "auto":
|
||||
@@ -266,7 +266,7 @@ def select():
|
||||
else:
|
||||
raise Exception("invalid output mode: " + omode)
|
||||
|
||||
if not config.get(("output",), "skip", True):
|
||||
if not config.get("output", "skip", True):
|
||||
output.skip = util.identity
|
||||
return output
|
||||
|
||||
@@ -298,7 +298,7 @@ class PipeOutput(NullOutput):
|
||||
class TerminalOutput(NullOutput):
|
||||
|
||||
def __init__(self):
|
||||
shorten = config.get(("output",), "shorten", True)
|
||||
shorten = config.get("output", "shorten", True)
|
||||
if shorten:
|
||||
func = shorten_string_eaw if shorten == "eaw" else shorten_string
|
||||
limit = shutil.get_terminal_size().columns - OFFSET
|
||||
|
||||
@@ -15,7 +15,6 @@ import json
|
||||
import time
|
||||
import random
|
||||
import sqlite3
|
||||
import binascii
|
||||
import datetime
|
||||
import functools
|
||||
import itertools
|
||||
@@ -113,8 +112,7 @@ def noop():
|
||||
|
||||
def generate_token(size=16):
|
||||
"""Generate a random token with hexadecimal digits"""
|
||||
data = random.getrandbits(size * 8).to_bytes(size, "big")
|
||||
return binascii.hexlify(data).decode()
|
||||
return random.getrandbits(size * 8).to_bytes(size, "big").hex()
|
||||
|
||||
|
||||
def format_value(value, suffixes="kMGTPEZY"):
|
||||
|
||||
@@ -6,4 +6,4 @@
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
__version__ = "1.22.0"
|
||||
__version__ = "2.0.0-dev"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[flake8]
|
||||
exclude = gallery_dl/__init__.py,gallery_dl/__main__.py,setup.py,build,scripts,archive
|
||||
exclude = build,scripts,archive
|
||||
ignore = E203,E226,W504
|
||||
per-file-ignores =
|
||||
gallery_dl/extractor/500px.py: E501
|
||||
|
||||
18
setup.py
18
setup.py
@@ -13,6 +13,7 @@ def read(fname):
|
||||
with open(path, encoding="utf-8") as file:
|
||||
return file.read()
|
||||
|
||||
|
||||
def check_file(fname):
|
||||
path = os.path.join(os.path.dirname(__file__), fname)
|
||||
if os.path.exists(path):
|
||||
@@ -33,11 +34,12 @@ VERSION = re.search(
|
||||
FILES = [
|
||||
(path, [f for f in files if check_file(f)])
|
||||
for (path, files) in [
|
||||
("share/bash-completion/completions", ["data/completion/gallery-dl"]),
|
||||
("share/zsh/site-functions" , ["data/completion/_gallery-dl"]),
|
||||
("share/fish/vendor_completions.d" , ["data/completion/gallery-dl.fish"]),
|
||||
("share/man/man1" , ["data/man/gallery-dl.1"]),
|
||||
("share/man/man5" , ["data/man/gallery-dl.conf.5"]),
|
||||
("share/bash-completion/completions", ["data/completion/gallery-dl"]),
|
||||
("share/zsh/site-functions" , ["data/completion/_gallery-dl"]),
|
||||
("share/fish/vendor_completions.d" , ["data/completion/"
|
||||
"gallery-dl.fish"]),
|
||||
]
|
||||
]
|
||||
|
||||
@@ -48,7 +50,7 @@ LONG_DESCRIPTION = read("README.rst")
|
||||
|
||||
if "py2exe" in sys.argv:
|
||||
try:
|
||||
import py2exe
|
||||
import py2exe # noqa E401
|
||||
except ImportError:
|
||||
sys.exit("Error importing 'py2exe'")
|
||||
|
||||
@@ -93,7 +95,7 @@ setup(
|
||||
maintainer="Mike Fährmann",
|
||||
maintainer_email="mike_faehrmann@web.de",
|
||||
license="GPLv2",
|
||||
python_requires=">=3.4",
|
||||
python_requires=">=3.8",
|
||||
install_requires=[
|
||||
"requests>=2.11.0",
|
||||
],
|
||||
@@ -123,13 +125,9 @@ setup(
|
||||
"Operating System :: Microsoft :: Windows",
|
||||
"Operating System :: POSIX",
|
||||
"Operating System :: MacOS",
|
||||
"Programming Language :: Python :: 3.4",
|
||||
"Programming Language :: Python :: 3.5",
|
||||
"Programming Language :: Python :: 3.6",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Multimedia :: Graphics",
|
||||
"Topic :: Utilities",
|
||||
|
||||
@@ -68,7 +68,7 @@ class TestCookiejar(unittest.TestCase):
|
||||
with mock.patch.object(log, "warning") as mock_warning:
|
||||
cookies = extractor.find("test:").session.cookies
|
||||
self.assertEqual(len(cookies), 0)
|
||||
self.assertEqual(mock_warning.call_count, 1)
|
||||
mock_warning.assert_called_once()
|
||||
self.assertEqual(mock_warning.call_args[0][0], "cookies: %s")
|
||||
self.assertIsInstance(mock_warning.call_args[0][1], exc)
|
||||
|
||||
|
||||
@@ -311,12 +311,12 @@ class TestDataJob(TestJob):
|
||||
|
||||
with patch("gallery_dl.util.number_to_string") as nts:
|
||||
tjob.run()
|
||||
self.assertEqual(len(nts.call_args_list), 0)
|
||||
nts.assert_not_called()
|
||||
|
||||
config.set(("output",), "num-to-str", True)
|
||||
with patch("gallery_dl.util.number_to_string") as nts:
|
||||
tjob.run()
|
||||
self.assertEqual(len(nts.call_args_list), 52)
|
||||
self.assertEqual(nts.call_count, 52)
|
||||
|
||||
tjob.run()
|
||||
self.assertEqual(tjob.data[-1][0], Message.Url)
|
||||
|
||||
@@ -138,7 +138,7 @@ class ClassifyTest(BasePostprocessorTest):
|
||||
|
||||
with patch("os.makedirs") as mkdirs:
|
||||
self._trigger()
|
||||
self.assertEqual(mkdirs.call_count, 0)
|
||||
mkdirs.assert_not_called()
|
||||
|
||||
def test_classify_custom(self):
|
||||
pp = self._create({"mapping": {
|
||||
|
||||
Reference in New Issue
Block a user