mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Merge branch 'master' into bugfix/gis-map-template0
This commit is contained in:
commit
f5474ea4ce
59 changed files with 1673 additions and 496 deletions
|
|
@ -0,0 +1,26 @@
|
|||
"""Lower FileGroup.primary_path
|
||||
|
||||
Revision ID: 158973fda20f
|
||||
Revises: 69e9754d1a63
|
||||
Create Date: 2025-08-16 10:54:58.874094
|
||||
|
||||
"""
|
||||
import os
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '158973fda20f'
|
||||
down_revision = '69e9754d1a63'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute('CREATE INDEX file_group_primary_path_lower_idx ON file_group (LOWER(primary_path))')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute('DROP INDEX IF EXISTS file_group_primary_path_lower_idx')
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
"""file_group.effective_date trigger
|
||||
|
||||
Revision ID: 4f03b9548f6e
|
||||
Revises: 54c19aa847da
|
||||
Create Date: 2025-08-17 10:49:35.754548
|
||||
|
||||
"""
|
||||
import os
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '4f03b9548f6e'
|
||||
down_revision = '54c19aa847da'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Create trigger function
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_effective_datetime()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
NEW.effective_datetime = COALESCE(NEW.published_datetime, NEW.download_datetime);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
|
||||
# Create trigger
|
||||
op.execute("""
|
||||
CREATE TRIGGER file_group_effective_datetime_trigger
|
||||
BEFORE INSERT OR UPDATE OF published_datetime, download_datetime
|
||||
ON file_group
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_effective_datetime();
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
# Drop trigger
|
||||
op.execute("DROP TRIGGER IF EXISTS file_group_effective_datetime_trigger ON file_group")
|
||||
|
||||
# Drop trigger function
|
||||
op.execute("DROP FUNCTION IF EXISTS update_effective_datetime")
|
||||
145
alembic/versions/54c19aa847da_channel_summary_indexes.py
Normal file
145
alembic/versions/54c19aa847da_channel_summary_indexes.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""Channel summary columns, triggers and indexes.
|
||||
|
||||
Revision ID: 54c19aa847da
|
||||
Revises: 158973fda20f
|
||||
Create Date: 2025-08-16 11:45:48.190427
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '54c19aa847da'
|
||||
down_revision = '158973fda20f'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Add columns
|
||||
op.execute("""
|
||||
ALTER TABLE channel
|
||||
ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL,
|
||||
ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL,
|
||||
ADD COLUMN minimum_frequency INTEGER;
|
||||
""")
|
||||
|
||||
# Populate columns
|
||||
op.execute("""
|
||||
UPDATE channel
|
||||
SET video_count = (SELECT COUNT(*)
|
||||
FROM video
|
||||
WHERE channel_id = channel.id),
|
||||
total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
|
||||
FROM video v
|
||||
LEFT JOIN file_group fg ON v.file_group_id = fg.id
|
||||
WHERE v.channel_id = channel.id),
|
||||
minimum_frequency = (SELECT MIN(frequency)
|
||||
FROM download
|
||||
WHERE channel_id = channel.id);
|
||||
""")
|
||||
|
||||
# Create trigger functions
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_video_count()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id)
|
||||
WHERE id = NEW.channel_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_size()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
|
||||
FROM video v
|
||||
LEFT JOIN file_group fg ON v.file_group_id = fg.id
|
||||
WHERE v.channel_id = NEW.id)
|
||||
WHERE id = NEW.id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_channel_minimum_frequency()
|
||||
RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
UPDATE channel
|
||||
SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id)
|
||||
WHERE id = NEW.channel_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
|
||||
# Create triggers
|
||||
op.execute("""
|
||||
CREATE TRIGGER video_count_trigger
|
||||
AFTER INSERT OR DELETE
|
||||
ON video
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_video_count();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER video_size_trigger
|
||||
AFTER INSERT OR DELETE
|
||||
ON video
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_size();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER file_group_size_trigger
|
||||
AFTER UPDATE OF size
|
||||
ON file_group
|
||||
FOR EACH ROW
|
||||
WHEN (OLD.size IS DISTINCT FROM NEW.size)
|
||||
EXECUTE FUNCTION update_channel_size();
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER download_frequency_trigger
|
||||
AFTER INSERT OR UPDATE OF frequency OR DELETE
|
||||
ON download
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_channel_minimum_frequency();
|
||||
""")
|
||||
|
||||
# Create indexes
|
||||
op.execute("""
|
||||
CREATE INDEX channel_video_count_idx ON channel (video_count);
|
||||
CREATE INDEX channel_total_size_idx ON channel (total_size);
|
||||
CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency);
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
# Drop triggers
|
||||
op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;")
|
||||
op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;")
|
||||
op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;")
|
||||
op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;")
|
||||
|
||||
# Drop trigger functions
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;")
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_size;")
|
||||
op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;")
|
||||
|
||||
# Drop indexes
|
||||
op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;")
|
||||
op.execute("DROP INDEX IF EXISTS channel_total_size_idx;")
|
||||
op.execute("DROP INDEX IF EXISTS channel_video_count_idx;")
|
||||
|
||||
# Drop columns
|
||||
op.execute("""
|
||||
ALTER TABLE channel
|
||||
DROP COLUMN IF EXISTS minimum_frequency,
|
||||
DROP COLUMN IF EXISTS total_size,
|
||||
DROP COLUMN IF EXISTS video_count;
|
||||
""")
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""FileGroup.effective_datetime
|
||||
|
||||
Revision ID: 69e9754d1a63
|
||||
Revises: f7229a67e333
|
||||
Create Date: 2025-08-16 10:28:42.661108
|
||||
|
||||
"""
|
||||
import os
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '69e9754d1a63'
|
||||
down_revision = 'f7229a67e333'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute('ALTER TABLE file_group ADD COLUMN IF NOT EXISTS effective_datetime TIMESTAMPTZ')
|
||||
|
||||
# Populating effective_datetime for existing rows
|
||||
op.execute("""
|
||||
UPDATE file_group
|
||||
SET effective_datetime = COALESCE(published_datetime, download_datetime)
|
||||
WHERE published_datetime IS NOT NULL
|
||||
OR download_datetime IS NOT NULL;
|
||||
""")
|
||||
|
||||
op.execute('CREATE INDEX IF NOT EXISTS file_group_effective_datetime_idx ON file_group(effective_datetime)')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute('DROP INDEX IF EXISTS file_group_effective_datetime_idx')
|
||||
op.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS effective_datetime')
|
||||
274
app/package-lock.json
generated
274
app/package-lock.json
generated
|
|
@ -2236,12 +2236,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@cypress/request/node_modules/form-data": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
|
||||
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
|
||||
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
|
|
@ -5574,6 +5576,18 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/call-bind-apply-helpers": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
|
||||
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
"function-bind": "^1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
|
|
@ -7164,6 +7178,19 @@
|
|||
"resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.6.tgz",
|
||||
"integrity": "sha512-+3NaRjWktb5r61ZFoDejlykPEFKT5N/LkbXsaddlw6xNSXBanUYpFc2AXXpbJDilPHazcSreU/DpQIaxfX0NfQ=="
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
|
||||
"dependencies": {
|
||||
"call-bind-apply-helpers": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"gopd": "^1.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/duplexer": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
|
||||
|
|
@ -7342,12 +7369,9 @@
|
|||
"integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA=="
|
||||
},
|
||||
"node_modules/es-define-property": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
|
||||
"integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
|
||||
"dependencies": {
|
||||
"get-intrinsic": "^1.2.4"
|
||||
},
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
|
|
@ -7384,14 +7408,26 @@
|
|||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
|
||||
"integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
|
||||
},
|
||||
"node_modules/es-set-tostringtag": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
|
||||
"integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
|
||||
"node_modules/es-object-atoms": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
|
||||
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
|
||||
"dependencies": {
|
||||
"get-intrinsic": "^1.1.3",
|
||||
"has": "^1.0.3",
|
||||
"has-tostringtag": "^1.0.0"
|
||||
"es-errors": "^1.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/es-set-tostringtag": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
"get-intrinsic": "^1.2.6",
|
||||
"has-tostringtag": "^1.0.2",
|
||||
"hasown": "^2.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
|
|
@ -8866,13 +8902,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz",
|
||||
"integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"mime-types": "^2.1.35"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
|
|
@ -8992,15 +9030,20 @@
|
|||
}
|
||||
},
|
||||
"node_modules/get-intrinsic": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
|
||||
"integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
|
||||
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
|
||||
"dependencies": {
|
||||
"call-bind-apply-helpers": "^1.0.2",
|
||||
"es-define-property": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"es-object-atoms": "^1.1.1",
|
||||
"function-bind": "^1.1.2",
|
||||
"has-proto": "^1.0.1",
|
||||
"has-symbols": "^1.0.3",
|
||||
"hasown": "^2.0.0"
|
||||
"get-proto": "^1.0.1",
|
||||
"gopd": "^1.2.0",
|
||||
"has-symbols": "^1.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"math-intrinsics": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
|
|
@ -9022,6 +9065,18 @@
|
|||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/get-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
|
||||
"dependencies": {
|
||||
"dunder-proto": "^1.0.1",
|
||||
"es-object-atoms": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-stream": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
|
||||
|
|
@ -9198,11 +9253,11 @@
|
|||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
|
||||
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
|
||||
"dependencies": {
|
||||
"get-intrinsic": "^1.1.3"
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
|
|
@ -9292,9 +9347,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/has-symbols": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
|
||||
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
|
||||
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
|
|
@ -9303,11 +9358,11 @@
|
|||
}
|
||||
},
|
||||
"node_modules/has-tostringtag": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
|
||||
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"dependencies": {
|
||||
"has-symbols": "^1.0.2"
|
||||
"has-symbols": "^1.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
|
|
@ -12841,6 +12896,14 @@
|
|||
"tmpl": "1.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/math-intrinsics": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
|
||||
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/mathjs": {
|
||||
"version": "14.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.0.1.tgz",
|
||||
|
|
@ -17367,9 +17430,9 @@
|
|||
"integrity": "sha512-XHjoxak8SFQnHnmYHb3PcnW5TZ+9ErLZemZei3azuIRhQLw4IExsVbL3VZJdHcLeNaXq6NqawgpDPpjBOg4B5g=="
|
||||
},
|
||||
"node_modules/tmp": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
|
||||
"integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.4.tgz",
|
||||
"integrity": "sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ==",
|
||||
"engines": {
|
||||
"node": ">=14.14"
|
||||
}
|
||||
|
|
@ -20256,12 +20319,14 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"form-data": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
|
||||
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
|
||||
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
|
|
@ -22700,6 +22765,15 @@
|
|||
"set-function-length": "^1.2.1"
|
||||
}
|
||||
},
|
||||
"call-bind-apply-helpers": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
|
||||
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
|
||||
"requires": {
|
||||
"es-errors": "^1.3.0",
|
||||
"function-bind": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
|
|
@ -23831,6 +23905,16 @@
|
|||
"resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.6.tgz",
|
||||
"integrity": "sha512-+3NaRjWktb5r61ZFoDejlykPEFKT5N/LkbXsaddlw6xNSXBanUYpFc2AXXpbJDilPHazcSreU/DpQIaxfX0NfQ=="
|
||||
},
|
||||
"dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
|
||||
"requires": {
|
||||
"call-bind-apply-helpers": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"gopd": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"duplexer": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
|
||||
|
|
@ -23976,12 +24060,9 @@
|
|||
"integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA=="
|
||||
},
|
||||
"es-define-property": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
|
||||
"integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
|
||||
"requires": {
|
||||
"get-intrinsic": "^1.2.4"
|
||||
}
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="
|
||||
},
|
||||
"es-errors": {
|
||||
"version": "1.3.0",
|
||||
|
|
@ -24009,14 +24090,23 @@
|
|||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
|
||||
"integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
|
||||
},
|
||||
"es-set-tostringtag": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
|
||||
"integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
|
||||
"es-object-atoms": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
|
||||
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
|
||||
"requires": {
|
||||
"get-intrinsic": "^1.1.3",
|
||||
"has": "^1.0.3",
|
||||
"has-tostringtag": "^1.0.0"
|
||||
"es-errors": "^1.3.0"
|
||||
}
|
||||
},
|
||||
"es-set-tostringtag": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"requires": {
|
||||
"es-errors": "^1.3.0",
|
||||
"get-intrinsic": "^1.2.6",
|
||||
"has-tostringtag": "^1.0.2",
|
||||
"hasown": "^2.0.2"
|
||||
}
|
||||
},
|
||||
"es-shim-unscopables": {
|
||||
|
|
@ -25082,13 +25172,15 @@
|
|||
}
|
||||
},
|
||||
"form-data": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz",
|
||||
"integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==",
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"mime-types": "^2.1.35"
|
||||
}
|
||||
},
|
||||
"forwarded": {
|
||||
|
|
@ -25164,15 +25256,20 @@
|
|||
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
|
||||
},
|
||||
"get-intrinsic": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
|
||||
"integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
|
||||
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
|
||||
"requires": {
|
||||
"call-bind-apply-helpers": "^1.0.2",
|
||||
"es-define-property": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"es-object-atoms": "^1.1.1",
|
||||
"function-bind": "^1.1.2",
|
||||
"has-proto": "^1.0.1",
|
||||
"has-symbols": "^1.0.3",
|
||||
"hasown": "^2.0.0"
|
||||
"get-proto": "^1.0.1",
|
||||
"gopd": "^1.2.0",
|
||||
"has-symbols": "^1.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"math-intrinsics": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"get-own-enumerable-property-symbols": {
|
||||
|
|
@ -25185,6 +25282,15 @@
|
|||
"resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
|
||||
"integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q=="
|
||||
},
|
||||
"get-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
|
||||
"requires": {
|
||||
"dunder-proto": "^1.0.1",
|
||||
"es-object-atoms": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"get-stream": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
|
||||
|
|
@ -25311,12 +25417,9 @@
|
|||
}
|
||||
},
|
||||
"gopd": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
|
||||
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
|
||||
"requires": {
|
||||
"get-intrinsic": "^1.1.3"
|
||||
}
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.11",
|
||||
|
|
@ -25378,16 +25481,16 @@
|
|||
"integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg=="
|
||||
},
|
||||
"has-symbols": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
|
||||
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
|
||||
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="
|
||||
},
|
||||
"has-tostringtag": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
|
||||
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"requires": {
|
||||
"has-symbols": "^1.0.2"
|
||||
"has-symbols": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"hasown": {
|
||||
|
|
@ -27942,6 +28045,11 @@
|
|||
"tmpl": "1.0.5"
|
||||
}
|
||||
},
|
||||
"math-intrinsics": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
|
||||
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="
|
||||
},
|
||||
"mathjs": {
|
||||
"version": "14.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.0.1.tgz",
|
||||
|
|
@ -31052,9 +31160,9 @@
|
|||
"integrity": "sha512-XHjoxak8SFQnHnmYHb3PcnW5TZ+9ErLZemZei3azuIRhQLw4IExsVbL3VZJdHcLeNaXq6NqawgpDPpjBOg4B5g=="
|
||||
},
|
||||
"tmp": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
|
||||
"integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w=="
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.4.tgz",
|
||||
"integrity": "sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ=="
|
||||
},
|
||||
"tmpl": {
|
||||
"version": "1.0.5",
|
||||
|
|
|
|||
|
|
@ -101,6 +101,10 @@ function handleEvents(events) {
|
|||
() => window.open(url, '_self'));
|
||||
}
|
||||
|
||||
if (event === 'upload_archive_failed') {
|
||||
eventToast('Archive Upload Failed!', message, 'error', 5000);
|
||||
}
|
||||
|
||||
if (subject) {
|
||||
newestEvents[subject] = dt;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -271,8 +271,13 @@ function VideoDownloadCountLimit({form, name = 'video_count_limit', path = 'sett
|
|||
/>
|
||||
}
|
||||
|
||||
export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions'}) {
|
||||
const [inputProps, inputAttrs] = form.getSelectionProps({name, path, type: 'array'});
|
||||
export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions', onChange}) {
|
||||
const [inputProps, inputAttrs] = form.getSelectionProps({
|
||||
name,
|
||||
path,
|
||||
type: 'array',
|
||||
afterChange: onChange
|
||||
});
|
||||
|
||||
return <>
|
||||
<InfoHeader
|
||||
|
|
@ -382,13 +387,17 @@ export function UseBrowserProfile({form}) {
|
|||
/>
|
||||
}
|
||||
|
||||
export function VideosDownloadForm({singleDownload = true, onCancel}) {
|
||||
export function VideosDownloadForm({singleDownload = true, onCancel, onSuccess: propOnSuccess, download, submitter: propSubmitter, actions}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
|
||||
const [config, setConfig] = React.useState(null);
|
||||
|
||||
// Keep video settings in session to help user start downloads consistently.
|
||||
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
|
||||
// Keep video format in session to help user start downloads consistently.
|
||||
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
|
||||
|
||||
// Use config video resolutions if available, otherwise use default
|
||||
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
|
||||
|
||||
const defaultFormData = {
|
||||
urls: '', // Textarea, one URL per line.
|
||||
destination: '',
|
||||
|
|
@ -397,11 +406,11 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
|
|||
settings: {
|
||||
use_browser_profile: false,
|
||||
video_format: defaultVideoFormat,
|
||||
video_resolutions: defaultVideoResolutions,
|
||||
video_resolutions: configResolutions,
|
||||
}
|
||||
}
|
||||
|
||||
const submitter = async (formData) => {
|
||||
const submitter = propSubmitter || (async (formData) => {
|
||||
const downloadData = {
|
||||
destination: formData.destination,
|
||||
downloader: formData.downloader,
|
||||
|
|
@ -410,24 +419,45 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
|
|||
urls: formData.urls.split(/r?\n/),
|
||||
}
|
||||
await postDownload(downloadData);
|
||||
}
|
||||
});
|
||||
|
||||
const onSuccess = () => {
|
||||
setShowMessage(true);
|
||||
form.reset();
|
||||
if (propOnSuccess) {
|
||||
propOnSuccess();
|
||||
} else {
|
||||
form.reset();
|
||||
}
|
||||
}
|
||||
|
||||
const form = useForm({
|
||||
submitter,
|
||||
defaultFormData,
|
||||
defaultFormData: download ? mergeDeep(defaultFormData, download) : defaultFormData,
|
||||
onSuccess,
|
||||
});
|
||||
|
||||
// Fetch video downloader config when the modal opens
|
||||
React.useEffect(() => {
|
||||
const {video_resolutions, video_format} = form.formData.settings;
|
||||
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
|
||||
setDefaultVideoResolutions(video_resolutions);
|
||||
}
|
||||
const fetchConfig = async () => {
|
||||
try {
|
||||
const result = await fetchVideoDownloaderConfig();
|
||||
setConfig(result);
|
||||
|
||||
// Update form with video resolutions from config only if user hasn't changed them
|
||||
// and if no download object was provided (this is a new download, not an edit)
|
||||
if (result && result.video_resolutions && !userChangedResolutions && !download) {
|
||||
form.setValue('settings.video_resolutions', result.video_resolutions);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch video downloader config:', error);
|
||||
}
|
||||
};
|
||||
|
||||
fetchConfig();
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
const {video_format} = form.formData.settings;
|
||||
if (video_format && video_format !== defaultVideoFormat) {
|
||||
setDefaultVideoFormat(video_format);
|
||||
}
|
||||
|
|
@ -473,7 +503,10 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
|
|||
</Grid.Row>
|
||||
<Grid.Row columns={2}>
|
||||
<Grid.Column width={11}>
|
||||
<VideoResolutionSelectorForm form={form}/>
|
||||
<VideoResolutionSelectorForm
|
||||
form={form}
|
||||
onChange={() => setUserChangedResolutions(true)}
|
||||
/>
|
||||
</Grid.Column>
|
||||
<Grid.Column width={4}>
|
||||
<VideoFormatSelectorForm form={form}/>
|
||||
|
|
@ -492,7 +525,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
|
|||
{showMessage && <SuccessfulDownloadSubmitMessage/>}
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<DownloadFormButtons onCancel={localOnCancel} form={form}/>
|
||||
{actions ? actions({onCancel: localOnCancel, form}) : <DownloadFormButtons onCancel={localOnCancel} form={form}/>}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
|
|
@ -511,6 +544,40 @@ export function VideoMinimumDurationForm({form}) {
|
|||
/>
|
||||
}
|
||||
|
||||
export function EditVideosDownloadForm({
|
||||
download,
|
||||
onCancel,
|
||||
onSuccess,
|
||||
onDelete,
|
||||
actions = EditDownloadFormButtons,
|
||||
}) {
|
||||
const submitter = async (formData) => {
|
||||
// Create a copy of settings without channel_url and channel_id
|
||||
const settings = {...formData.settings};
|
||||
// These settings are created when downloading the video, we can't submit them again.
|
||||
delete settings.channel_url;
|
||||
delete settings.channel_id;
|
||||
|
||||
const downloadData = {
|
||||
destination: formData.destination,
|
||||
downloader: formData.downloader,
|
||||
settings: settings,
|
||||
tag_names: formData.tag_names,
|
||||
urls: formData.urls.split(/r?\n/),
|
||||
}
|
||||
await putDownload(download.id, downloadData);
|
||||
}
|
||||
|
||||
return <VideosDownloadForm
|
||||
singleDownload={true}
|
||||
onCancel={onCancel}
|
||||
onSuccess={onSuccess}
|
||||
download={download}
|
||||
submitter={submitter}
|
||||
actions={(props) => actions({...props, onDelete})}
|
||||
/>
|
||||
}
|
||||
|
||||
export function VideoMaximumDurationForm({form}) {
|
||||
return <VideoDurationLimit
|
||||
form={form}
|
||||
|
|
@ -535,6 +602,9 @@ export function ChannelDownloadForm({
|
|||
channel_id = null,
|
||||
}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
const [config, setConfig] = React.useState(null);
|
||||
const [isConfigLoaded, setIsConfigLoaded] = React.useState(false);
|
||||
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
|
||||
|
||||
// May have received submitter from EditChannelDownloadForm.
|
||||
submitter = submitter || (async (formData) => {
|
||||
|
|
@ -557,10 +627,12 @@ export function ChannelDownloadForm({
|
|||
}
|
||||
}
|
||||
|
||||
// Keep video settings in session to help user start downloads consistently.
|
||||
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
|
||||
// Keep video format in session to help user start downloads consistently.
|
||||
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
|
||||
|
||||
// Use config video resolutions if available, otherwise use default
|
||||
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
|
||||
|
||||
const emptyFormData = {
|
||||
destination: '',
|
||||
downloader: Downloaders.VideoChannel,
|
||||
|
|
@ -577,7 +649,7 @@ export function ChannelDownloadForm({
|
|||
use_browser_profile: false,
|
||||
video_count_limit: null,
|
||||
video_format: defaultVideoFormat,
|
||||
video_resolutions: defaultVideoResolutions,
|
||||
video_resolutions: configResolutions,
|
||||
},
|
||||
sub_downloader: Downloaders.Video,
|
||||
tag_names: [],
|
||||
|
|
@ -591,11 +663,30 @@ export function ChannelDownloadForm({
|
|||
clearOnSuccess,
|
||||
});
|
||||
|
||||
// Fetch video downloader config when the modal opens
|
||||
React.useEffect(() => {
|
||||
const {video_resolutions, video_format} = form.formData.settings;
|
||||
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
|
||||
setDefaultVideoResolutions(video_resolutions);
|
||||
}
|
||||
const fetchConfig = async () => {
|
||||
try {
|
||||
const result = await fetchVideoDownloaderConfig();
|
||||
setConfig(result);
|
||||
setIsConfigLoaded(true);
|
||||
|
||||
// Update form with video resolutions from config only if user hasn't changed them
|
||||
// and if no download object was provided (this is a new download, not an edit)
|
||||
if (result && result.video_resolutions && !userChangedResolutions && !download) {
|
||||
form.setValue('settings.video_resolutions', result.video_resolutions);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch video downloader config:', error);
|
||||
setIsConfigLoaded(true);
|
||||
}
|
||||
};
|
||||
|
||||
fetchConfig();
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
const {video_format} = form.formData.settings;
|
||||
if (video_format && video_format !== defaultVideoFormat) {
|
||||
setDefaultVideoFormat(video_format);
|
||||
}
|
||||
|
|
@ -656,7 +747,10 @@ export function ChannelDownloadForm({
|
|||
</Grid.Row>
|
||||
<Grid.Row columns={2}>
|
||||
<Grid.Column width={11}>
|
||||
<VideoResolutionSelectorForm form={form}/>
|
||||
<VideoResolutionSelectorForm
|
||||
form={form}
|
||||
onChange={() => setUserChangedResolutions(true)}
|
||||
/>
|
||||
</Grid.Column>
|
||||
<Grid.Column width={4}>
|
||||
<VideoFormatSelectorForm form={form}/>
|
||||
|
|
@ -730,17 +824,42 @@ function SuccessfulDownloadSubmitMessage() {
|
|||
</Grid.Row>
|
||||
}
|
||||
|
||||
export function ArchiveDownloadForm({download, onCancel}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
|
||||
export function EditArchiveDownloadForm({
|
||||
download,
|
||||
onCancel,
|
||||
onSuccess,
|
||||
onDelete,
|
||||
actions = EditDownloadFormButtons,
|
||||
}) {
|
||||
const submitter = async (formData) => {
|
||||
const downloadData = {
|
||||
downloader: formData.downloader,
|
||||
tag_names: formData.tag_names,
|
||||
urls: formData.urls.split(/\r?\n/),
|
||||
}
|
||||
await putDownload(download.id, downloadData);
|
||||
}
|
||||
|
||||
return <ArchiveDownloadForm
|
||||
download={download}
|
||||
onCancel={onCancel}
|
||||
onSuccess={onSuccess}
|
||||
submitter={submitter}
|
||||
actions={(props) => actions({...props, onDelete})}
|
||||
/>
|
||||
}
|
||||
|
||||
export function ArchiveDownloadForm({download, onCancel, onSuccess: propOnSuccess, submitter: propSubmitter, actions}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
|
||||
const submitter = propSubmitter || (async (formData) => {
|
||||
const downloadData = {
|
||||
downloader: formData.downloader,
|
||||
tag_names: formData.tag_names,
|
||||
urls: formData.urls.split(/\r?\n/),
|
||||
}
|
||||
await postDownload(downloadData);
|
||||
};
|
||||
});
|
||||
|
||||
const emptyFormData = {
|
||||
downloader: Downloaders.Archive,
|
||||
|
|
@ -748,14 +867,28 @@ export function ArchiveDownloadForm({download, onCancel}) {
|
|||
tag_names: [],
|
||||
};
|
||||
|
||||
const onSuccess = () => {
|
||||
setShowMessage(true);
|
||||
if (propOnSuccess) {
|
||||
propOnSuccess();
|
||||
}
|
||||
}
|
||||
|
||||
const form = useForm({
|
||||
submitter,
|
||||
defaultFormData: mergeDeep(emptyFormData, download),
|
||||
emptyFormData,
|
||||
clearOnSuccess: true,
|
||||
onSuccess: async () => setShowMessage(true),
|
||||
clearOnSuccess: !propOnSuccess,
|
||||
onSuccess,
|
||||
});
|
||||
|
||||
const localOnCancel = (e) => {
|
||||
if (e) e.preventDefault();
|
||||
if (onCancel) {
|
||||
onCancel();
|
||||
}
|
||||
}
|
||||
|
||||
return <Form>
|
||||
<Header as='h3'><Icon name='file text' color='green'/> Archives</Header>
|
||||
<p>Create a Singlefile Archive for each of the URLs provided below.</p>
|
||||
|
|
@ -774,7 +907,7 @@ export function ArchiveDownloadForm({download, onCancel}) {
|
|||
{showMessage && <SuccessfulDownloadSubmitMessage/>}
|
||||
<Grid.Row>
|
||||
<Grid.Column textAlign='right'>
|
||||
<DownloadFormButtons onCancel={onCancel} form={form}/>
|
||||
{actions ? actions({onCancel: localOnCancel, form}) : <DownloadFormButtons onCancel={localOnCancel} form={form}/>}
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
|
|
@ -783,8 +916,10 @@ export function ArchiveDownloadForm({download, onCancel}) {
|
|||
|
||||
export function RSSDownloadForm({download, submitter, onDelete, onCancel, actions, clearOnSuccess = true}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
const [config, setConfig] = React.useState(null);
|
||||
const [isConfigLoaded, setIsConfigLoaded] = React.useState(false);
|
||||
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
|
||||
|
||||
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
|
||||
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
|
||||
|
||||
submitter = submitter || (async (formData) => {
|
||||
|
|
@ -800,6 +935,9 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
await postDownload(downloadData);
|
||||
});
|
||||
|
||||
// Use config video resolutions if available, otherwise use default
|
||||
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
|
||||
|
||||
const emptyFormData = {
|
||||
destination: null,
|
||||
downloader: Downloaders.RSS,
|
||||
|
|
@ -809,7 +947,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
excluded_urls: null,
|
||||
title_exclude: null,
|
||||
title_include: null,
|
||||
video_resolutions: defaultVideoResolutions,
|
||||
video_resolutions: configResolutions,
|
||||
video_format: defaultVideoFormat,
|
||||
},
|
||||
tag_names: [],
|
||||
|
|
@ -824,11 +962,30 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
onSuccess: async () => setShowMessage(true),
|
||||
});
|
||||
|
||||
// Fetch video downloader config when the modal opens
|
||||
React.useEffect(() => {
|
||||
const {video_resolutions, video_format} = form.formData.settings;
|
||||
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
|
||||
setDefaultVideoResolutions(video_resolutions);
|
||||
}
|
||||
const fetchConfig = async () => {
|
||||
try {
|
||||
const result = await fetchVideoDownloaderConfig();
|
||||
setConfig(result);
|
||||
setIsConfigLoaded(true);
|
||||
|
||||
// Update form with video resolutions from config only if user hasn't changed them
|
||||
// and if no download object was provided (this is a new download, not an edit)
|
||||
if (result && result.video_resolutions && !userChangedResolutions && !download) {
|
||||
form.setValue('settings.video_resolutions', result.video_resolutions);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch video downloader config:', error);
|
||||
setIsConfigLoaded(true);
|
||||
}
|
||||
};
|
||||
|
||||
fetchConfig();
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
const {video_format} = form.formData.settings;
|
||||
if (video_format && video_format !== defaultVideoFormat) {
|
||||
setDefaultVideoFormat(video_format);
|
||||
}
|
||||
|
|
@ -859,7 +1016,10 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
</Grid.Row>
|
||||
<Grid.Row columns={2}>
|
||||
<Grid.Column mobile={16} computer={11}>
|
||||
<VideoResolutionSelectorForm form={form}/>
|
||||
<VideoResolutionSelectorForm
|
||||
form={form}
|
||||
onChange={() => setUserChangedResolutions(true)}
|
||||
/>
|
||||
</Grid.Column>
|
||||
<Grid.Column mobile={8} computer={3}>
|
||||
<VideoFormatSelectorForm form={form}/>
|
||||
|
|
|
|||
|
|
@ -193,6 +193,11 @@ export function FilePreviewProvider({children}) {
|
|||
// Navigated to a page which has a file preview active, fetch the information about the file, then display it.
|
||||
if (previewQuery && !previewFile) {
|
||||
initPreviewFile();
|
||||
} else if (!previewQuery && (previewFile || previewModal)) {
|
||||
// User navigated back/forward and preview query is gone, close the preview.
|
||||
// Only close if there's actually a preview open (previewFile or previewModal exists).
|
||||
setPreviewFile(null);
|
||||
setPreviewModal(null);
|
||||
}
|
||||
}, [previewQuery]);
|
||||
|
||||
|
|
|
|||
|
|
@ -208,6 +208,10 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
// The results summarized.
|
||||
const [suggestionsSums, setSuggestionsSums] = useState({});
|
||||
|
||||
// Disable selecting a result briefly after new suggestions arrive.
|
||||
const [selectDisabled, setSelectDisabled] = React.useState(false);
|
||||
const selectDisableTimerRef = React.useRef(null);
|
||||
|
||||
const noResults = [{title: 'No results'}];
|
||||
|
||||
const normalizeSuggestionsResults = (newSuggestions) => {
|
||||
|
|
@ -303,17 +307,40 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
|
|||
});
|
||||
}
|
||||
|
||||
const suggestionSelectDelay = 250;
|
||||
|
||||
React.useEffect(() => {
|
||||
setSuggestionsSums({});
|
||||
setSuggestionsResults({});
|
||||
|
||||
if (!_.isEmpty(suggestions)) {
|
||||
// Start a brief window where selecting results is disabled to prevent accidental clicks
|
||||
// when the dropdown re-renders with new suggestions.
|
||||
if (selectDisableTimerRef.current) {
|
||||
clearTimeout(selectDisableTimerRef.current);
|
||||
}
|
||||
setSelectDisabled(true);
|
||||
selectDisableTimerRef.current = setTimeout(() => setSelectDisabled(false), suggestionSelectDelay);
|
||||
|
||||
normalizeSuggestionsResults(suggestions);
|
||||
}
|
||||
}, [JSON.stringify(suggestions)]);
|
||||
|
||||
// Clear timer on unmount.
|
||||
React.useEffect(() => {
|
||||
return () => {
|
||||
if (selectDisableTimerRef.current) {
|
||||
clearTimeout(selectDisableTimerRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// User clicked on a result in the dropdown.
|
||||
const handleResultSelect = ({result}) => {
|
||||
if (selectDisabled) {
|
||||
console.debug('Selection temporarily disabled after suggestions update');
|
||||
return;
|
||||
}
|
||||
if (result.location) {
|
||||
console.info(`useSearchSuggestions Navigating: ${result.location}`)
|
||||
navigate(result.location);
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ function VideosPage() {
|
|||
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
|
||||
{value: 'length', text: 'Length'},
|
||||
{value: 'size', text: 'Size'},
|
||||
{value: 'size_to_duration', text: 'Size to Duration', short: 'S/Duration'},
|
||||
{value: 'view_count', text: 'Views'},
|
||||
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
|
||||
{value: 'download_datetime', text: 'Download Date', short: 'D.Date'},
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import {
|
|||
WROLModeMessage
|
||||
} from "../Common";
|
||||
import {
|
||||
Button as SButton,
|
||||
Button as SButton, ButtonGroup,
|
||||
Label,
|
||||
Loader,
|
||||
PlaceholderLine,
|
||||
|
|
@ -35,7 +35,9 @@ import {
|
|||
EditChannelDownloadForm,
|
||||
EditRSSDownloadForm,
|
||||
EditScrapeFilesDownloadForm,
|
||||
EditZimDownloadForm
|
||||
EditZimDownloadForm,
|
||||
EditVideosDownloadForm,
|
||||
EditArchiveDownloadForm
|
||||
} from "../Download";
|
||||
import {Downloaders} from "../Vars";
|
||||
|
||||
|
|
@ -246,6 +248,9 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
|
|||
class OnceDownloadRow extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
editModalOpen: false
|
||||
};
|
||||
}
|
||||
|
||||
handleDelete = async () => {
|
||||
|
|
@ -272,8 +277,21 @@ class OnceDownloadRow extends React.Component {
|
|||
}
|
||||
};
|
||||
|
||||
handleEditOpen = () => {
|
||||
this.setState({ editModalOpen: true });
|
||||
};
|
||||
|
||||
handleEditClose = () => {
|
||||
this.setState({ editModalOpen: false });
|
||||
};
|
||||
|
||||
handleEditSuccess = async () => {
|
||||
await this.props.fetchDownloads();
|
||||
this.handleEditClose();
|
||||
};
|
||||
|
||||
render() {
|
||||
let {url, last_successful_download, status, location, error} = this.props;
|
||||
let {url, last_successful_download, status, location, error, downloader, settings, id} = this.props;
|
||||
|
||||
// Open downloads (/download), or external links in an anchor.
|
||||
const link = location && !location.startsWith('/download') ?
|
||||
|
|
@ -297,6 +315,8 @@ class OnceDownloadRow extends React.Component {
|
|||
} else if (status === 'failed' || status === 'deferred') {
|
||||
buttonCell = (
|
||||
<TableCell>
|
||||
<ButtonGroup>
|
||||
|
||||
<APIButton
|
||||
color='red'
|
||||
icon='trash'
|
||||
|
|
@ -305,6 +325,13 @@ class OnceDownloadRow extends React.Component {
|
|||
confirmButton='Delete'
|
||||
obeyWROLMode={true}
|
||||
/>
|
||||
{(downloader === Downloaders.Video || downloader === Downloaders.Archive) && status !== 'pending' && (
|
||||
<Button
|
||||
icon='edit'
|
||||
color='blue'
|
||||
onClick={this.handleEditOpen}
|
||||
/>
|
||||
)}
|
||||
<APIButton
|
||||
color='green'
|
||||
icon='redo'
|
||||
|
|
@ -313,6 +340,7 @@ class OnceDownloadRow extends React.Component {
|
|||
onClick={this.handleRestart}
|
||||
obeyWROLMode={true}
|
||||
/>
|
||||
</ButtonGroup>
|
||||
</TableCell>
|
||||
);
|
||||
} else if (status === 'complete' && location) {
|
||||
|
|
@ -336,6 +364,55 @@ class OnceDownloadRow extends React.Component {
|
|||
)
|
||||
}
|
||||
|
||||
// Create edit modal for video or archive downloads
|
||||
let editModal = null;
|
||||
|
||||
if (downloader === Downloaders.Video) {
|
||||
editModal = (
|
||||
<Modal
|
||||
closeIcon
|
||||
open={this.state.editModalOpen}
|
||||
onClose={this.handleEditClose}
|
||||
>
|
||||
<ModalHeader>Edit Video Download</ModalHeader>
|
||||
<ModalContent>
|
||||
<EditVideosDownloadForm
|
||||
download={{
|
||||
urls: url,
|
||||
settings: settings || {},
|
||||
id: id
|
||||
}}
|
||||
onCancel={this.handleEditClose}
|
||||
onSuccess={this.handleEditSuccess}
|
||||
onDelete={this.handleDelete}
|
||||
/>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
);
|
||||
} else if (downloader === Downloaders.Archive) {
|
||||
editModal = (
|
||||
<Modal
|
||||
closeIcon
|
||||
open={this.state.editModalOpen}
|
||||
onClose={this.handleEditClose}
|
||||
>
|
||||
<ModalHeader>Edit Archive Download</ModalHeader>
|
||||
<ModalContent>
|
||||
<EditArchiveDownloadForm
|
||||
download={{
|
||||
urls: url,
|
||||
tag_names: settings?.tag_names || [],
|
||||
id: id
|
||||
}}
|
||||
onCancel={this.handleEditClose}
|
||||
onSuccess={this.handleEditSuccess}
|
||||
onDelete={this.handleDelete}
|
||||
/>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
return <TableRow>
|
||||
<TableCell className='column-ellipsis'>
|
||||
{link(url)}
|
||||
|
|
@ -345,6 +422,7 @@ class OnceDownloadRow extends React.Component {
|
|||
{status === 'pending' ? <Loader active inline size='tiny'/> : null}
|
||||
</TableCell>
|
||||
{buttonCell}
|
||||
{editModal}
|
||||
</TableRow>
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ export function useForm({
|
|||
patchFormData(_.set(formData, path, newValue));
|
||||
}
|
||||
|
||||
const getCustomProps = ({name, validator, type = 'text', path, required = false}) => {
|
||||
const getCustomProps = ({name, validator, type = 'text', path, required = false, afterChange}) => {
|
||||
// Props for any other elements.
|
||||
|
||||
path = path || name;
|
||||
|
|
@ -202,7 +202,12 @@ export function useForm({
|
|||
type,
|
||||
disabled,
|
||||
value,
|
||||
onChange: (newValue) => {setValue(path, newValue)},
|
||||
onChange: (newValue) => {
|
||||
setValue(path, newValue);
|
||||
if (afterChange) {
|
||||
afterChange(newValue);
|
||||
}
|
||||
},
|
||||
'data-path': path,
|
||||
}
|
||||
// Attributes about the input, but should not be passed as properties.
|
||||
|
|
@ -214,7 +219,15 @@ export function useForm({
|
|||
return [inputProps, inputAttrs]
|
||||
}
|
||||
|
||||
const getInputProps = ({name, validator, path, required = false, type = 'text', onChange = null}) => {
|
||||
const getInputProps = ({
|
||||
name,
|
||||
validator,
|
||||
path,
|
||||
required = false,
|
||||
type = 'text',
|
||||
onChange = null,
|
||||
afterChange = null
|
||||
}) => {
|
||||
// Props for <input/>
|
||||
path = path || name;
|
||||
|
||||
|
|
@ -223,7 +236,7 @@ export function useForm({
|
|||
validator = validURL;
|
||||
}
|
||||
|
||||
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required});
|
||||
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required, afterChange});
|
||||
|
||||
const localHandleInputEvent = async (e) => {
|
||||
if (e) e.preventDefault();
|
||||
|
|
@ -244,12 +257,15 @@ export function useForm({
|
|||
return [inputProps, inputAttrs]
|
||||
}
|
||||
|
||||
const getSelectionProps = ({name, validator, path, type, required = false}) => {
|
||||
const getSelectionProps = ({name, validator, path, type, required = false, afterChange = null}) => {
|
||||
// Props for Dropdowns.
|
||||
path = path || name;
|
||||
const [customProps, inputAttrs] = getCustomProps({name, validator, type, path, required});
|
||||
const [customProps, inputAttrs] = getCustomProps({name, validator, type, path, required, afterChange});
|
||||
customProps.onChange = (e, {value}) => {
|
||||
patchFormData(_.set(formData, path, value))
|
||||
patchFormData(_.set(formData, path, value));
|
||||
if (afterChange) {
|
||||
afterChange(value);
|
||||
}
|
||||
};
|
||||
return [customProps, inputAttrs]
|
||||
}
|
||||
|
|
@ -388,50 +404,49 @@ export function UrlInput({form, required = true, name = 'url', path = 'url', dis
|
|||
export function UrlsTextarea({name = 'urls', required, form}) {
|
||||
required = required !== undefined;
|
||||
|
||||
const validator = (value) => {
|
||||
if (!validURLs(value)) {
|
||||
return 'Invalid URLs';
|
||||
}
|
||||
}
|
||||
// Memoize input props to prevent recomputation during render
|
||||
const [inputProps, inputAttrs] = React.useMemo(() => {
|
||||
|
||||
const [inputProps, inputAttrs] = form.getInputProps({name, validator, required});
|
||||
const validator = (value) => {
|
||||
if (!validURLs(value)) {
|
||||
return 'Invalid URLs';
|
||||
}
|
||||
};
|
||||
|
||||
return form.getInputProps({name, validator, required});
|
||||
}, [form, name, required]);
|
||||
|
||||
const handleDrop = (e) => {
|
||||
if (e) e.preventDefault();
|
||||
// Handle user dropping multiple URLs into the textarea. Assume each drop is a URL. Ensure each URL
|
||||
// is on its own line.
|
||||
const droppedUrl = e.dataTransfer.getData('text');
|
||||
let urls = (inputProps.value || '').split('\n');
|
||||
urls = [...urls, droppedUrl];
|
||||
urls = urls.filter(i => !!i).join('\n');
|
||||
// Leave trailing newline for user to type.
|
||||
inputAttrs.setValue(`${urls}\n`)
|
||||
inputAttrs.setValue(`${urls}\n`);
|
||||
};
|
||||
|
||||
const handleKeyDown = (event) => {
|
||||
// Check if Ctrl (or Cmd for Mac) + Enter are pressed
|
||||
if ((event.ctrlKey || event.metaKey) && event.key === 'Enter') {
|
||||
event.preventDefault(); // Prevents the addition of a new line in textarea if necessary
|
||||
form.onSubmit(); // Call your submit handler
|
||||
event.preventDefault();
|
||||
form.onSubmit();
|
||||
}
|
||||
};
|
||||
|
||||
return <FormInput required
|
||||
error={inputProps.error}
|
||||
label='URLs'
|
||||
>
|
||||
<TextArea
|
||||
id='urls_textarea'
|
||||
placeholder={'Enter one URL per line'}
|
||||
name='urls'
|
||||
onDrop={handleDrop}
|
||||
onKeyDown={handleKeyDown}
|
||||
{...inputProps}
|
||||
/>
|
||||
</FormInput>
|
||||
return (
|
||||
<FormInput required error={inputProps.error} label="URLs">
|
||||
<TextArea
|
||||
id="urls_textarea"
|
||||
placeholder="Enter one URL per line"
|
||||
name="urls"
|
||||
onDrop={handleDrop}
|
||||
onKeyDown={handleKeyDown}
|
||||
{...inputProps}
|
||||
/>
|
||||
</FormInput>
|
||||
);
|
||||
}
|
||||
|
||||
export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled=false}) {
|
||||
export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled = false}) {
|
||||
const [inputProps, inputAttrs] = form.getCustomProps({name, path})
|
||||
|
||||
const iconElm = icon ? <Icon size={iconSize} name={icon}/> : null;
|
||||
|
|
|
|||
|
|
@ -60,11 +60,18 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi-*.service /etc/systemd/system/
|
|||
cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
|
||||
cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
|
||||
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd
|
||||
# Copy MOTD scripts, delete original.
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd/20-wrolpi.motd /etc/update-motd.d/20-wrolpi
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi-first.motd /etc/update-motd.d/30-wrolpi
|
||||
chmod +x /etc/update-motd.d/*
|
||||
truncate -s 0 /etc/motd
|
||||
|
||||
# NetworkManager for the hotspot.
|
||||
systemctl enable NetworkManager
|
||||
|
||||
# Enable iperf3 for speed testing.
|
||||
systemctl enable iperf3
|
||||
|
||||
set +x
|
||||
|
||||
echo
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ gpsd
|
|||
hostapd
|
||||
htop
|
||||
iotop
|
||||
iperf3
|
||||
kiwix
|
||||
kiwix-tools
|
||||
libagg-dev
|
||||
|
|
@ -101,4 +102,4 @@ vim
|
|||
vlc
|
||||
wget
|
||||
wpasupplicant
|
||||
zim-tools
|
||||
zim-tools
|
||||
|
|
|
|||
|
|
@ -86,6 +86,8 @@ services:
|
|||
map_https: # HTTPS for map tile server
|
||||
depends_on:
|
||||
- map
|
||||
environment:
|
||||
- UPSTREAM_HOST=map
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/https_proxy/Dockerfile
|
||||
|
|
@ -130,6 +132,8 @@ services:
|
|||
zim_https: # HTTPS for Zim service.
|
||||
depends_on:
|
||||
- zim
|
||||
environment:
|
||||
- UPSTREAM_HOST=zim
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/https_proxy/Dockerfile
|
||||
|
|
@ -154,6 +158,8 @@ services:
|
|||
help_https: # HTTPS for Help service.
|
||||
depends_on:
|
||||
- help
|
||||
environment:
|
||||
- UPSTREAM_HOST=help
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/https_proxy/Dockerfile
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
FROM python:3.11-buster
|
||||
FROM python:3.11-bookworm
|
||||
ENV DOCKER=true
|
||||
WORKDIR /opt/wrolpi
|
||||
|
||||
# Update dependencies for the services install.
|
||||
RUN apt update
|
||||
# ffmpeg for Videos
|
||||
RUN apt-get install -y ffmpeg catdoc sysstat aria2
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping
|
||||
RUN ffmpeg -version
|
||||
# yt-dlp needs cache directory
|
||||
RUN mkdir /.cache && chown -R 1000:1000 /.cache
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# This Dockerfile is used as a reverse proxy to serve a HTTPS nieve service with HTTPS.
|
||||
FROM nginx:latest as https_proxy
|
||||
FROM nginx:latest AS https_proxy
|
||||
|
||||
# Create a self-signed cert and key pair for HTTPS.
|
||||
RUN openssl genrsa -out /cert.key 2048
|
||||
|
|
@ -7,6 +7,10 @@ RUN openssl req -new -x509 -nodes -key /cert.key -out /cert.crt -days 3650 \
|
|||
-subj "/C=US/ST=State/L=City/O=Org/OU=WROLPi/CN=wrolpi.local"
|
||||
RUN chmod 777 /cert.key /cert.crt
|
||||
|
||||
# Copy Nginx configuration
|
||||
COPY docker/https_proxy/nginx.proxy.conf /etc/nginx/nginx.conf
|
||||
# Apply docker-compose environment variables to config
|
||||
ENV UPSTREAM_HOST=you-must-change-this-in-docker-compose
|
||||
ENV UPSTREAM_PORT=80
|
||||
COPY docker/https_proxy/nginx.proxy.conf.template /etc/nginx/nginx.conf
|
||||
RUN envsubst < /etc/nginx/nginx.conf > /etc/nginx/nginx.conf
|
||||
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ http {
|
|||
ssl_certificate_key /cert.key;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:80;
|
||||
proxy_pass http://${UPSTREAM_HOST}:${UPSTREAM_PORT};
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
The programs included with the Debian GNU/Linux system are free software;
|
||||
the exact distribution terms for each program are described in the
|
||||
individual files in /usr/share/doc/*/copyright.
|
||||
|
||||
Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
|
||||
permitted by applicable law.
|
||||
|
||||
██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗
|
||||
██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║
|
||||
██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██
|
||||
██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║
|
||||
╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║
|
||||
╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝
|
||||
|
||||
Welcome to your WROLPi!
|
||||
|
||||
Run the help script to find potential problems:
|
||||
/opt/wrolpi/help.sh
|
||||
|
||||
Help documentation is available: https://wrolpi.local:8086 or /opt/wrolpi-help/docs
|
||||
|
||||
16
etc/raspberrypios/motd/20-wrolpi.motd
Executable file
16
etc/raspberrypios/motd/20-wrolpi.motd
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/sh
|
||||
echo ""
|
||||
echo "The programs included with the Debian GNU/Linux system are free software;"
|
||||
echo "the exact distribution terms for each program are described in the"
|
||||
echo "individual files in /usr/share/doc/*/copyright."
|
||||
echo ""
|
||||
echo "Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent"
|
||||
echo "permitted by applicable law."
|
||||
echo ""
|
||||
echo "██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗"
|
||||
echo "██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║"
|
||||
echo "██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██"
|
||||
echo "██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║"
|
||||
echo "╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║"
|
||||
echo " ╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝"
|
||||
echo ""
|
||||
17
etc/raspberrypios/motd/30-wrolpi-first.motd
Executable file
17
etc/raspberrypios/motd/30-wrolpi-first.motd
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/sh
|
||||
|
||||
ip_address=$(ip addr show eth0 | grep -oP 'inet \K[\d.]+')
|
||||
hostname="$(hostname).local"
|
||||
|
||||
echo "Welcome to your new WROLPi! Follow the steps below to finish the setup:"
|
||||
echo ""
|
||||
echo " 1. If you have an external drive, mount it to /media/wrolpi"
|
||||
echo " 2. Run the repair script: /opt/wrolpi/repair.sh"
|
||||
echo " 3. Refresh your files in the UI at https://${hostname} or https://127.0.0.1"
|
||||
echo ""
|
||||
echo "Run the help script to find potential problems:"
|
||||
echo " /opt/wrolpi/help.sh"
|
||||
echo ""
|
||||
echo "Help documentation is available: https://${ip_address}:8086"
|
||||
echo " or in this directory /opt/wrolpi-help/docs"
|
||||
echo ""
|
||||
12
etc/raspberrypios/motd/30-wrolpi.motd
Executable file
12
etc/raspberrypios/motd/30-wrolpi.motd
Executable file
|
|
@ -0,0 +1,12 @@
|
|||
#!/bin/sh
|
||||
|
||||
ip_address=$(ip addr show eth0 | grep -oP 'inet \K[\d.]+')
|
||||
|
||||
echo " Welcome to your WROLPi!"
|
||||
echo ""
|
||||
echo " Run the help script to find potential problems:"
|
||||
echo " /opt/wrolpi/help.sh"
|
||||
echo ""
|
||||
echo " Help documentation is available: https://${ip_address}:8086"
|
||||
echo " or in this directory /opt/wrolpi-help/docs"
|
||||
echo ""
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
The programs included with the Debian GNU/Linux system are free software;
|
||||
the exact distribution terms for each program are described in the
|
||||
individual files in /usr/share/doc/*/copyright.
|
||||
|
||||
Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
|
||||
permitted by applicable law.
|
||||
|
||||
██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗
|
||||
██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║
|
||||
██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██
|
||||
██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║
|
||||
╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║
|
||||
╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝
|
||||
|
||||
Welcome to your new WROLPi! Follow the steps below to finish the setup:
|
||||
|
||||
1. If you have an external drive, mount it to /media/wrolpi
|
||||
2. Run the repair script: /opt/wrolpi/repair.sh
|
||||
3. Refresh your files in the UI at https://wrolpi.local or https://127.0.0.1
|
||||
|
||||
Run the help script to find potential problems:
|
||||
/opt/wrolpi/help.sh
|
||||
|
||||
Help documentation is available: https://wrolpi.local:8086 or /opt/wrolpi-help/docs
|
||||
|
||||
43
main.py
43
main.py
|
|
@ -224,24 +224,31 @@ async def start_single_tasks(app: Sanic):
|
|||
with suppress(Exception):
|
||||
get_wrolpi_config().import_config()
|
||||
logger.debug('wrolpi config imported')
|
||||
with suppress(Exception):
|
||||
tags.import_tags_config()
|
||||
logger.debug('tags config imported')
|
||||
with suppress(Exception):
|
||||
get_videos_downloader_config().import_config()
|
||||
logger.debug('videos downloader config imported')
|
||||
with suppress(Exception):
|
||||
await import_downloads_config()
|
||||
logger.debug('downloads config imported')
|
||||
# Channels uses both downloads and tags.
|
||||
with suppress(Exception):
|
||||
import_channels_config()
|
||||
logger.debug('channels config imported')
|
||||
with suppress(Exception):
|
||||
import_inventories_config()
|
||||
logger.debug('inventories config imported')
|
||||
with suppress(Exception):
|
||||
init_inventory()
|
||||
|
||||
wrolpi_config = get_wrolpi_config()
|
||||
if wrolpi_config.successful_import:
|
||||
# Only import other configs if WROLPi config was imported, and WROL mode is not enabled.
|
||||
if wrolpi_config.wrol_mode:
|
||||
logger.warning('Refusing to import other configs when WROL mode is enabled!')
|
||||
else:
|
||||
with suppress(Exception):
|
||||
tags.import_tags_config()
|
||||
logger.debug('tags config imported')
|
||||
with suppress(Exception):
|
||||
get_videos_downloader_config().import_config()
|
||||
logger.debug('videos downloader config imported')
|
||||
with suppress(Exception):
|
||||
await import_downloads_config()
|
||||
logger.debug('downloads config imported')
|
||||
# Channels uses both downloads and tags.
|
||||
with suppress(Exception):
|
||||
import_channels_config()
|
||||
logger.debug('channels config imported')
|
||||
with suppress(Exception):
|
||||
import_inventories_config()
|
||||
logger.debug('inventories config imported')
|
||||
with suppress(Exception):
|
||||
init_inventory()
|
||||
|
||||
from modules.zim.lib import flag_outdated_zim_files
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -107,13 +107,32 @@ archive_downloader = ArchiveDownloader()
|
|||
|
||||
|
||||
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
||||
"""
|
||||
Models an Archive from a given FileGroup.
|
||||
|
||||
This function takes in a FileGroup and attempts to create an Archive object.
|
||||
It does this by checking for the presence of HTML files within the FileGroup,
|
||||
determining if any of these are SingleFiles, and then extracting relevant data
|
||||
(title, contents) from either the JSON or text readability files.
|
||||
|
||||
Args:
|
||||
file_group: The FileGroup to model as an Archive.
|
||||
session: An optional database session for committing changes.
|
||||
|
||||
Returns:
|
||||
An Archive object representing the modeled archive.
|
||||
|
||||
Raises:
|
||||
InvalidArchive: If no HTML SingleFile is found in the FileGroup, or if
|
||||
any other error occurs during modeling.
|
||||
"""
|
||||
file_group_id = file_group.id
|
||||
if not file_group_id:
|
||||
session.flush([file_group])
|
||||
file_group_id = file_group.id
|
||||
|
||||
# All Archives have an HTML Singlefile.
|
||||
html_paths = file_group.my_paths('text/html')
|
||||
# All Archives have an HTML Singlefile. Singlefile may be text HTML, or bytes HTML.
|
||||
html_paths = file_group.my_html_paths()
|
||||
if not html_paths:
|
||||
logger.error('Query returned a group without an HTML file!')
|
||||
raise InvalidArchive('FileGroup does not contain any html files')
|
||||
|
|
@ -162,8 +181,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
if readability_txt_path:
|
||||
contents = get_article(readability_txt_path)
|
||||
|
||||
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
||||
session.add(archive)
|
||||
# Check if an Archive already exists for this FileGroup
|
||||
archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none()
|
||||
if not archive:
|
||||
# Create new Archive if it doesn't exist
|
||||
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
||||
session.add(archive)
|
||||
archive.validate()
|
||||
archive.flush()
|
||||
|
||||
|
|
@ -177,6 +200,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
|
|||
'readability_txt_path': archive.readability_txt_path,
|
||||
'screenshot_path': archive.screenshot_path,
|
||||
'singlefile_path': archive.singlefile_path,
|
||||
'info_json_path': archive.info_json_path,
|
||||
}
|
||||
|
||||
return archive
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import multiprocessing
|
||||
import queue
|
||||
from http import HTTPStatus
|
||||
|
||||
from sanic import response, Request, Blueprint
|
||||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.api_utils import json_response
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
|
||||
from wrolpi.api_utils import json_response, api_app
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
|
|
@ -84,20 +87,50 @@ async def get_upload_singlefile(request: Request):
|
|||
|
||||
|
||||
@register_switch_handler('singlefile_upload_switch_handler')
|
||||
async def singlefile_upload_switch_handler(singlefile):
|
||||
async def singlefile_upload_switch_handler(url=None):
|
||||
"""Used by `post_upload_singlefile` to upload a single file"""
|
||||
from wrolpi.downloader import download_manager, Download
|
||||
from wrolpi.downloader import Download
|
||||
from . import ArchiveDownloader
|
||||
|
||||
archive = await lib.singlefile_to_archive(singlefile)
|
||||
logger.info(f'Created Archive from upload: {archive}')
|
||||
q: multiprocessing.Queue = api_app.shared_ctx.archive_singlefiles
|
||||
|
||||
trace_enabled = logger.isEnabledFor(TRACE_LEVEL)
|
||||
if trace_enabled:
|
||||
logger.trace(f'singlefile_upload_switch_handler called for {url}')
|
||||
try:
|
||||
singlefile = q.get_nowait()
|
||||
except queue.Empty:
|
||||
if trace_enabled:
|
||||
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
|
||||
return
|
||||
|
||||
try:
|
||||
q_size = q.qsize()
|
||||
except NotImplementedError:
|
||||
# qsize() is not implemented on macOS
|
||||
q_size = '?'
|
||||
logger.info(f'singlefile_upload_switch_handler queue size: {q_size}')
|
||||
|
||||
try:
|
||||
archive = await lib.singlefile_to_archive(singlefile)
|
||||
except Exception as e:
|
||||
logger.error(f'singlefile_upload_switch_handler failed', exc_info=e)
|
||||
Events.send_upload_archive_failed(f'Failed to convert singlefile to archive: {e}')
|
||||
raise
|
||||
|
||||
name = archive.file_group.title or archive.file_group.url
|
||||
logger.info(f'Created Archive from upload ({q_size}): {archive}')
|
||||
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
|
||||
url = archive.file_group.url
|
||||
if url and (download := Download.get_by_url(url)):
|
||||
if (download.is_failed or download.is_deferred) and download.downloader == ArchiveDownloader.name:
|
||||
# Download was attempted and failed, user manually archived the URL.
|
||||
download_manager.delete_download(download.id)
|
||||
download.complete()
|
||||
download.location = archive.location
|
||||
Session.object_session(download).commit()
|
||||
|
||||
# Call this function again so any new singlefiles can be archived.
|
||||
singlefile_upload_switch_handler.activate_switch()
|
||||
|
||||
|
||||
singlefile_upload_switch_handler: ActivateSwitchMethod
|
||||
|
|
@ -112,10 +145,11 @@ async def post_upload_singlefile(request: Request):
|
|||
singlefile = request.files['singlefile_contents'][0].body
|
||||
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
|
||||
|
||||
# Get the URL to ensure that the singlefile is valid.
|
||||
# Extract the URL to ensure that the singlefile is valid.
|
||||
lib.get_url_from_singlefile(singlefile)
|
||||
|
||||
# Send processing to background task so the extension can continue.
|
||||
singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile))
|
||||
api_app.shared_ctx.archive_singlefiles.put(singlefile)
|
||||
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
|
||||
# Return empty json response because SingleFile extension expects a JSON response.
|
||||
return json_response(dict(), status=HTTPStatus.OK)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def archive_directory(test_directory) -> pathlib.Path:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def archive_factory(test_session, archive_directory, make_files_structure):
|
||||
def archive_factory(test_session, archive_directory, make_files_structure, image_bytes_factory):
|
||||
def time_generator():
|
||||
timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC)
|
||||
while True:
|
||||
|
|
@ -29,7 +29,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
now = time_generator()
|
||||
|
||||
def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None,
|
||||
singlefile_contents: str = None, tag_names: List[str] = None) -> Archive:
|
||||
singlefile_contents: str = None, tag_names: List[str] = None, screenshot: bool = True) -> Archive:
|
||||
if domain:
|
||||
assert '/' not in domain
|
||||
|
||||
|
|
@ -49,10 +49,9 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
if url:
|
||||
json_contents['url'] = url
|
||||
json_contents = json.dumps(json_contents)
|
||||
singlefile_path, screenshot_path, readability_path, readability_txt_path, readability_json_path = \
|
||||
singlefile_path, readability_path, readability_txt_path, readability_json_path = \
|
||||
make_files_structure({
|
||||
str(domain_dir / f'{timestamp}_{title}.html'): singlefile_contents or '<html></html>',
|
||||
str(domain_dir / f'{timestamp}_{title}.png'): None,
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.html'): '<html></html>',
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.txt'): contents,
|
||||
str(domain_dir / f'{timestamp}_{title}.readability.json'): json_contents,
|
||||
|
|
@ -61,6 +60,11 @@ def archive_factory(test_session, archive_directory, make_files_structure):
|
|||
if domain:
|
||||
domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
|
||||
|
||||
screenshot_path = None
|
||||
if screenshot:
|
||||
screenshot_path = domain_dir / f'{timestamp}_{title}.png'
|
||||
screenshot_path.write_bytes(image_bytes_factory())
|
||||
|
||||
if not domain and domain_dir.name != 'NA':
|
||||
domain = Domain(
|
||||
domain=domain_dir.name,
|
||||
|
|
|
|||
|
|
@ -319,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
|
|||
except json.decoder.JSONDecodeError:
|
||||
# Was not valid JSON.
|
||||
schema = None
|
||||
if isinstance(schema, dict) and '://schema.org' in schema.get('@context'):
|
||||
if isinstance(schema, dict) and (context := schema.get('@context')) and '://schema.org' in context:
|
||||
# Found https://schema.org/
|
||||
if headline := schema.get('headline'):
|
||||
metadata.title = metadata.title or headline
|
||||
|
|
@ -429,21 +429,22 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
|
|||
def get_domains():
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
|
||||
FROM domains
|
||||
LEFT JOIN archive a on domains.id = a.domain_id
|
||||
LEFT JOIN file_group fg on fg.id = a.file_group_id
|
||||
GROUP BY domains.domain
|
||||
ORDER BY domains.domain
|
||||
'''
|
||||
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
|
||||
FROM domains
|
||||
LEFT JOIN archive a on domains.id = a.domain_id
|
||||
LEFT JOIN file_group fg on fg.id = a.file_group_id
|
||||
GROUP BY domains.domain
|
||||
ORDER BY domains.domain \
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
domains = [dict(i) for i in curs.fetchall()]
|
||||
return domains
|
||||
|
||||
|
||||
ARCHIVE_ORDERS = {
|
||||
'published_datetime': (date := 'COALESCE(fg.published_datetime, fg.download_datetime) ASC NULLS LAST'),
|
||||
'-published_datetime': (_date := 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST'),
|
||||
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
|
||||
'published_datetime': (date := 'fg.effective_datetime ASC NULLS LAST'),
|
||||
'-published_datetime': (_date := 'fg.effective_datetime DESC NULLS LAST'),
|
||||
'published_modified_datetime': f'fg.published_modified_datetime ASC NULLS LAST, {date}',
|
||||
'-published_modified_datetime': f'fg.published_modified_datetime DESC NULLS LAST, {_date}',
|
||||
'download_datetime': 'fg.download_datetime ASC NULLS LAST',
|
||||
|
|
@ -456,8 +457,8 @@ ARCHIVE_ORDERS = {
|
|||
'-viewed': 'fg.viewed DESC',
|
||||
}
|
||||
ORDER_GROUP_BYS = {
|
||||
'published_datetime': 'fg.published_datetime, fg.download_datetime',
|
||||
'-published_datetime': 'fg.published_datetime, fg.download_datetime',
|
||||
'published_datetime': 'fg.effective_datetime',
|
||||
'-published_datetime': 'fg.effective_datetime',
|
||||
'published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
|
||||
'-published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
|
||||
'download_datetime': 'fg.download_datetime',
|
||||
|
|
@ -482,7 +483,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
-> Tuple[List[dict], int]:
|
||||
# Always filter FileGroups to Archives.
|
||||
wheres = []
|
||||
group_by = 'fg.published_datetime, fg.download_datetime, a.file_group_id'
|
||||
group_by = 'fg.effective_datetime, a.file_group_id'
|
||||
|
||||
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
|
||||
order_by = ARCHIVE_ORDERS['-published_datetime']
|
||||
|
|
|
|||
|
|
@ -141,8 +141,7 @@ class Archive(Base, ModelHelper):
|
|||
@property
|
||||
def singlefile_file(self) -> Optional[dict]:
|
||||
from modules.archive.lib import is_singlefile_file
|
||||
files = self.file_group.my_files('text/html')
|
||||
for file in files:
|
||||
for file in self.file_group.my_html_files():
|
||||
if is_singlefile_file(file['path']):
|
||||
return file
|
||||
|
||||
|
|
@ -198,6 +197,18 @@ class Archive(Base, ModelHelper):
|
|||
if screenshot_file := self.screenshot_file:
|
||||
return screenshot_file['path']
|
||||
|
||||
@property
|
||||
def info_json_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files()
|
||||
for file in files:
|
||||
if file['path'].name.endswith('.info.json'):
|
||||
return file
|
||||
|
||||
@property
|
||||
def info_json_path(self) -> Optional[pathlib.Path]:
|
||||
if info_json_file := self.info_json_file:
|
||||
return info_json_file['path']
|
||||
|
||||
def apply_readability_data(self):
|
||||
"""Read the Readability JSON file, apply its contents to this record."""
|
||||
readability_json_path = self.readability_json_path
|
||||
|
|
|
|||
|
|
@ -4,8 +4,11 @@ from http import HTTPStatus
|
|||
import pytest
|
||||
|
||||
from modules.archive import lib, Archive
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import skip_circleci, skip_macos
|
||||
|
||||
from wrolpi.common import get_relative_to_media_directory
|
||||
|
||||
|
||||
def check_results(test_client, data, ids):
|
||||
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
|
||||
|
|
@ -202,7 +205,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
|
|||
request, response = test_client.get(f'/api/archive/domains')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['domains'][0]['domain'] == 'example.com'
|
||||
assert response.json['domains'][0]['size'] == 116
|
||||
assert response.json['domains'][0]['size'] == 254
|
||||
|
||||
# Deleting works.
|
||||
request, response = test_client.delete(f'/api/archive/{archive1.id}')
|
||||
|
|
@ -257,3 +260,107 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
|
|||
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
|
||||
events = list(events_history)
|
||||
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
|
||||
await_switches, make_multipart_form, image_bytes_factory):
|
||||
"""Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup."""
|
||||
archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
# Verify Archive and FileGroup are created, but screenshot does not exist.
|
||||
file_group = archive.file_group
|
||||
assert not archive.screenshot_path, 'Screenshot should not exist yet.'
|
||||
assert file_group.files is not None, 'FileGroup should have files array'
|
||||
initial_file_count = len(file_group.files)
|
||||
|
||||
# Upload info.json via /api/files/upload
|
||||
# Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test")
|
||||
singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test"
|
||||
|
||||
# Create info.json content
|
||||
info_json_data = {
|
||||
'title': 'test',
|
||||
'url': 'https://example.com/test-url',
|
||||
'custom_field': 'value'
|
||||
}
|
||||
info_json_bytes = json.dumps(info_json_data).encode()
|
||||
|
||||
# Get relative destination directory
|
||||
destination = str(get_relative_to_media_directory(archive.singlefile_path.parent))
|
||||
filename = f'{singlefile_stem}.info.json'
|
||||
|
||||
# POST to /api/files/upload
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(info_json_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=info_json_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Step 4: Verify info.json is tracked
|
||||
# Refresh the session to get updated data
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
|
||||
# Assert info.json properties exist
|
||||
assert archive.info_json_path is not None, 'Archive.info_json_path should exist'
|
||||
assert archive.info_json_file is not None, 'Archive.info_json_file should exist'
|
||||
|
||||
# Assert FileGroup has the correct number of files.
|
||||
assert len(archive.file_group.files) == initial_file_count + 1, \
|
||||
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
|
||||
assert archive.file_group.data.get('info_json_path'), \
|
||||
'Archive readability info json should be in FileGroup.data'
|
||||
|
||||
# Upload an image file and verify it's added to the FileGroup.
|
||||
# Store file count before adding image
|
||||
file_count_before_image = len(archive.file_group.files)
|
||||
|
||||
# Create image using factory
|
||||
image_data = image_bytes_factory()
|
||||
|
||||
# Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png)
|
||||
image_filename = f'{singlefile_stem}.jpg'
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=image_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(image_data))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=image_data, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
|
||||
# Assert screenshot properties exist
|
||||
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
|
||||
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
|
||||
assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
|
||||
|
||||
# Assert image is in FileGroup.files
|
||||
file_paths = [f['path'] for f in archive.file_group.files]
|
||||
assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(archive.file_group.files) == file_count_before_image + 1, \
|
||||
f'FileGroup should have {file_count_before_image + 1} files after adding image'
|
||||
|
|
|
|||
|
|
@ -276,6 +276,7 @@ def test_get_domains(test_session, archive_factory):
|
|||
assert [i['domain'] for i in get_domains()] == []
|
||||
|
||||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_archive(test_session, test_directory, fake_now):
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
|
|
|
|||
|
|
@ -29,52 +29,24 @@ async def get_minimal_channels() -> List[dict]:
|
|||
# Get all channels, even if they don't have videos. Also get the minimum frequency download because this is the
|
||||
# one that will consume the most resources.
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
|
||||
COUNT(v.id) as video_count,
|
||||
SUM(fg.size)::BIGINT AS size,
|
||||
(
|
||||
select min(d.frequency)
|
||||
from download d
|
||||
where d.channel_id = c.id
|
||||
) AS minimum_frequency
|
||||
FROM
|
||||
channel AS c
|
||||
LEFT JOIN video v on c.id = v.channel_id
|
||||
LEFT JOIN file_group fg on fg.id = v.file_group_id
|
||||
LEFT JOIN tag t ON t.id = c.tag_id
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
'''
|
||||
SELECT c.id,
|
||||
c.name AS "name",
|
||||
c.directory,
|
||||
c.url,
|
||||
t.name AS "tag_name",
|
||||
c.video_count,
|
||||
c.total_size,
|
||||
c.minimum_frequency
|
||||
FROM channel AS c
|
||||
LEFT JOIN tag t ON t.id = c.tag_id
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
logger.debug(stmt)
|
||||
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
|
||||
|
||||
return channels
|
||||
|
||||
|
||||
async def get_channels_video_count() -> Dict[int, int]:
|
||||
"""
|
||||
Add video counts to all channels
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
stmt = 'SELECT id FROM channel'
|
||||
curs.execute(stmt)
|
||||
# Get all channel IDs, start them with a count of 0.
|
||||
video_counts = {int(i['id']): 0 for i in curs.fetchall()}
|
||||
|
||||
stmt = '''
|
||||
SELECT
|
||||
c.id, COUNT(v.id) AS video_count
|
||||
FROM
|
||||
channel AS c
|
||||
LEFT JOIN video AS v ON v.channel_id = c.id
|
||||
GROUP BY 1
|
||||
'''
|
||||
curs.execute(stmt)
|
||||
# Replace all the counts of those channels with videos.
|
||||
video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()})
|
||||
return video_counts
|
||||
|
||||
|
||||
COD = Union[dict, Channel]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ async def test_channel_download_requires_refresh(
|
|||
assert channel.refreshed == expected
|
||||
assert bool(channel.info_json) == expected
|
||||
|
||||
await async_client.sanic_app.dispatch('wrolpi.download.')
|
||||
await async_client.sanic_app.dispatch('wrolpi.download.download')
|
||||
|
||||
assert_refreshed(False)
|
||||
test_session.commit()
|
||||
|
|
@ -373,16 +373,14 @@ def test_search_videos_channel(test_client, test_session, video_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
|
||||
async def test_get_channel_videos_pagination(test_session, video_factory, assert_video_search, channel_factory):
|
||||
# Create two channels, one with 50 videos...
|
||||
channel1, channel2 = channel_factory(), channel_factory()
|
||||
for i in range(50):
|
||||
video_factory(channel_id=simple_channel.id)
|
||||
|
||||
channel2 = Channel(name='Bar')
|
||||
test_session.add(channel2)
|
||||
test_session.flush()
|
||||
test_session.refresh(channel2)
|
||||
video_factory(channel_id=channel1.id)
|
||||
# The other channel only has one video.
|
||||
video_factory(channel_id=channel2.id)
|
||||
video_factory(title='vid2', channel_id=channel2.id, with_video_file=True)
|
||||
video_factory(title='vid2', channel_id=channel2.id)
|
||||
test_session.commit()
|
||||
|
||||
# Get first, second, third, and empty pages of videos.
|
||||
|
|
@ -395,10 +393,11 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
|
|||
]
|
||||
last_ids = []
|
||||
for offset, video_count in tests:
|
||||
_, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime',
|
||||
_, response = await assert_video_search(channel_id=channel1.id, order_by='published_datetime',
|
||||
offset=offset,
|
||||
limit=20)
|
||||
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
|
||||
assert len(response.json['file_groups']) == video_count, \
|
||||
f'Returned videos does not match for ({offset}, {video_count})'
|
||||
current_ids = [i['id'] for i in response.json['file_groups']]
|
||||
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
|
||||
last_ids = current_ids
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
import os.path
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from datetime import timedelta
|
||||
|
|
@ -41,30 +42,47 @@ from .video.lib import download_video_info_json
|
|||
logger = logger.getChild(__name__)
|
||||
|
||||
VIDEO_RESOLUTION_MAP = {
|
||||
'360p': ['134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
|
||||
'hls-175+bestaudio', # 360p avc1 + best audio
|
||||
],
|
||||
'480p': ['135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480',
|
||||
'hls-312+bestaudio', # 480p avc1 + best audio
|
||||
],
|
||||
'360p': [
|
||||
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
|
||||
'hls-175+bestaudio', # 360p avc1 + best audio
|
||||
'396+bestaudio', '243+bestaudio', # Vertical
|
||||
'hls-655+bestaudio', 'hls-655',
|
||||
'bestvideo[height=360]+bestaudio/best[height=360]',
|
||||
],
|
||||
'480p': [
|
||||
'135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480',
|
||||
'hls-312+bestaudio', # 480p avc1 + best audio
|
||||
'788+bestaudio', '780+bestaudio', '397+bestaudio', # Vertical
|
||||
'bestvideo[height=480]+bestaudio/best[height=480]',
|
||||
],
|
||||
'720p': [
|
||||
'298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio
|
||||
'136+bestaudio', '22', '311+bestaudio', 'res:720',
|
||||
'298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio
|
||||
'hls-517+bestaudio', 'hls-637+bestaudio', # 720p avc1 + best audio
|
||||
'398+bestaudio', '247+bestaudio', # Vertical
|
||||
'hls-2890+bestaudio', 'hls-2890',
|
||||
'bestvideo[height=720]+bestaudio/best[height=720]',
|
||||
],
|
||||
'1080p': [
|
||||
'299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio
|
||||
'137+bestaudio', '614+bestaudio', 'res:1080',
|
||||
'299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio
|
||||
'hls-899+bestaudio', 'hls-1271+bestaudio', # 1080p avc1 + best audio
|
||||
'399+bestaudio', '616+bestaudio', # Vertical
|
||||
'hls-4026+bestaudio', 'hls-4026',
|
||||
'bestvideo[height=1080]+bestaudio/best[height=1080]',
|
||||
],
|
||||
'1440p': [
|
||||
'639+bestaudio', # vp9, best audio, hdr
|
||||
'623+bestaudio', # vp9, best audio, no hdr
|
||||
'620+bestaudio', 'res:1440'],
|
||||
'620+bestaudio', 'res:1440',
|
||||
'bestvideo[height=1440]+bestaudio/best[height=1440]',
|
||||
],
|
||||
'2160p': [
|
||||
'642+bestaudio', # vp9, best audio, hdr
|
||||
'628+bestaudio', # vp9, best audio, no hdr
|
||||
'625+bestaudio', 'res:2160'],
|
||||
'625+bestaudio', 'res:2160',
|
||||
'bestvideo[height=2160]+bestaudio/best[height=2160]',
|
||||
],
|
||||
'maximum': ['bestvideo*+bestaudio/best'],
|
||||
}
|
||||
|
||||
|
|
@ -272,9 +290,9 @@ class ChannelDownloader(Downloader, ABC):
|
|||
# Filter videos by their length.
|
||||
try:
|
||||
if minimum_duration := settings.get('minimum_duration'):
|
||||
downloads = [i for i in downloads if int(i['duration']) >= minimum_duration]
|
||||
downloads = [i for i in downloads if (d := i.get('duration')) and int(d) >= minimum_duration]
|
||||
if maximum_duration := settings.get('maximum_duration'):
|
||||
downloads = [i for i in downloads if int(i['duration']) <= maximum_duration]
|
||||
downloads = [i for i in downloads if (d := i.get('duration')) and int(d) <= maximum_duration]
|
||||
except KeyError as e:
|
||||
raise RuntimeError('Unable to filter videos because there is no duration') from e
|
||||
|
||||
|
|
|
|||
|
|
@ -100,8 +100,12 @@ class Video(ModelHelper, Base):
|
|||
|
||||
@staticmethod
|
||||
def do_model(file_group: FileGroup, session: Session) -> 'Video':
|
||||
video = Video(file_group_id=file_group.id, file_group=file_group)
|
||||
session.add(video)
|
||||
# Check if a Video already exists for this FileGroup
|
||||
video = session.query(Video).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not video:
|
||||
# Create new Video if it doesn't exist
|
||||
video = Video(file_group_id=file_group.id, file_group=file_group)
|
||||
session.add(video)
|
||||
video.validate()
|
||||
file_group.indexed = True
|
||||
video.flush()
|
||||
|
|
@ -152,25 +156,20 @@ class Video(ModelHelper, Base):
|
|||
if self.file_group.published_datetime:
|
||||
# Get videos next to this Video's upload date.
|
||||
stmt = '''
|
||||
WITH numbered_videos AS (
|
||||
SELECT fg.id AS fg_id, v.id AS v_id,
|
||||
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
|
||||
FROM file_group fg
|
||||
LEFT OUTER JOIN video v on fg.id = v.file_group_id
|
||||
WHERE
|
||||
v.channel_id = %(channel_id)s
|
||||
AND fg.published_datetime IS NOT NULL
|
||||
)
|
||||
SELECT v_id
|
||||
FROM numbered_videos
|
||||
WHERE row_number IN (
|
||||
SELECT row_number+i
|
||||
FROM numbered_videos
|
||||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
WHERE
|
||||
fg_id = %(fg_id)s
|
||||
)
|
||||
'''
|
||||
WITH numbered_videos AS (SELECT fg.id AS fg_id,
|
||||
v.id AS v_id,
|
||||
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
|
||||
FROM file_group fg
|
||||
LEFT OUTER JOIN video v on fg.id = v.file_group_id
|
||||
WHERE v.channel_id = %(channel_id)s
|
||||
AND fg.published_datetime IS NOT NULL)
|
||||
SELECT v_id
|
||||
FROM numbered_videos
|
||||
WHERE row_number IN (SELECT row_number + i
|
||||
FROM numbered_videos
|
||||
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
|
||||
WHERE fg_id = %(fg_id)s) \
|
||||
'''
|
||||
else:
|
||||
# No videos near this Video with upload dates, recommend the files next to this Video.
|
||||
# Only recommend videos in the same Channel (or similarly without a Channel).
|
||||
|
|
@ -244,6 +243,15 @@ class Video(ModelHelper, Base):
|
|||
self.channel = channel
|
||||
logger.debug(f'{self} has Channel {channel}')
|
||||
|
||||
# Set FileGroup.data with cached file paths for consistency with Archive
|
||||
self.file_group.data = {
|
||||
'id': self.id,
|
||||
'video_path': self.video_path,
|
||||
'info_json_path': self.info_json_path,
|
||||
'poster_path': self.poster_path,
|
||||
'caption_paths': self.caption_paths,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
|
|
@ -532,6 +540,11 @@ class Channel(ModelHelper, Base):
|
|||
source_id = Column(String) # the ID from the source website.
|
||||
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
|
||||
|
||||
# Columns updated by triggers
|
||||
video_count = Column(Integer, default=0) # update_channel_video_count
|
||||
total_size = Column(Integer, default=0) # update_channel_size
|
||||
minimum_frequency = Column(Integer) # update_channel_minimum_frequency
|
||||
|
||||
info_json = deferred(Column(JSON))
|
||||
info_date = Column(Date)
|
||||
|
||||
|
|
@ -680,18 +693,17 @@ class Channel(ModelHelper, Base):
|
|||
"""Get statistics about this channel."""
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT
|
||||
SUM(size) AS "size",
|
||||
MAX(size) AS "largest_video",
|
||||
COUNT(video.id) AS "video_count",
|
||||
SUM(fg.length) AS "length",
|
||||
-- Videos may use multiple tags.
|
||||
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
|
||||
FROM video
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
|
||||
WHERE channel_id = %(id)s
|
||||
'''
|
||||
SELECT SUM(size) AS "size",
|
||||
MAX(size) AS "largest_video",
|
||||
COUNT(video.id) AS "video_count",
|
||||
SUM(fg.length) AS "length",
|
||||
-- Videos may use multiple tags.
|
||||
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
|
||||
FROM video
|
||||
LEFT JOIN file_group fg on fg.id = video.file_group_id
|
||||
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
|
||||
WHERE channel_id = %(id)s \
|
||||
'''
|
||||
curs.execute(stmt, dict(id=self.id))
|
||||
return dict(curs.fetchone())
|
||||
|
||||
|
|
|
|||
|
|
@ -308,6 +308,43 @@ async def test_format_videos_description(async_client, test_session, test_direct
|
|||
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_size_to_duration_sort(test_session, video_factory, assert_video_search):
|
||||
"""Test the 'size_to_duration' sort option."""
|
||||
# Create videos with different size-to-duration ratios
|
||||
# All videos will have the same size (from the test file) but different durations
|
||||
|
||||
# Video with 10 second duration (highest ratio)
|
||||
vid1 = video_factory(with_video_file=True, title='vid1', with_info_json={'duration': 10})
|
||||
|
||||
# Video with 20 second duration (medium ratio)
|
||||
vid2 = video_factory(with_video_file=True, title='vid2', with_info_json={'duration': 20})
|
||||
|
||||
# Video with 30 second duration (lowest ratio)
|
||||
vid3 = video_factory(with_video_file=True, title='vid3', with_info_json={'duration': 30})
|
||||
|
||||
test_session.commit()
|
||||
|
||||
# Verify that the videos have the expected size and length values
|
||||
vid1 = test_session.query(Video).filter_by(id=vid1.id).one()
|
||||
vid2 = test_session.query(Video).filter_by(id=vid2.id).one()
|
||||
vid3 = test_session.query(Video).filter_by(id=vid3.id).one()
|
||||
|
||||
# All videos should have the same size
|
||||
assert vid1.file_group.size == vid2.file_group.size == vid3.file_group.size
|
||||
|
||||
# But different lengths
|
||||
assert vid1.file_group.length == 10
|
||||
assert vid2.file_group.length == 20
|
||||
assert vid3.file_group.length == 30
|
||||
|
||||
# Test ascending order (lowest ratio first)
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='size_to_duration')
|
||||
|
||||
# Test descending order (highest ratio first)
|
||||
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='-size_to_duration')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_file_format(async_client, test_session, fake_now):
|
||||
body = dict(video_file_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s')
|
||||
|
|
@ -324,3 +361,161 @@ async def test_video_file_format(async_client, test_session, fake_now):
|
|||
request, response = await async_client.post('/api/videos/file_format', json=body)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json['error'] == 'Filename must end with .%(ext)s'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form,
|
||||
image_bytes_factory):
|
||||
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
|
||||
from wrolpi.common import get_relative_to_media_directory
|
||||
|
||||
# Step 1: Create a Video with just a video file (no info.json, poster, or captions)
|
||||
video = video_factory(with_video_file=True, title='file_tracking_test')
|
||||
test_session.commit()
|
||||
|
||||
# Step 2: Verify Video and FileGroup are created, but optional files do not exist
|
||||
file_group = video.file_group
|
||||
assert not video.info_json_path, 'info.json should not exist yet.'
|
||||
assert not video.poster_path, 'Poster should not exist yet.'
|
||||
assert not video.caption_paths, 'Captions should not exist yet.'
|
||||
assert file_group.files is not None, 'FileGroup should have files array'
|
||||
initial_file_count = len(file_group.files)
|
||||
|
||||
# Step 3: Upload info.json via /api/files/upload
|
||||
# Extract the video stem (e.g., "file_tracking_test")
|
||||
video_stem = video.video_path.stem
|
||||
|
||||
# Create info.json content
|
||||
info_json_data = {
|
||||
'title': 'file_tracking_test',
|
||||
'duration': 60,
|
||||
'uploader': 'test_uploader'
|
||||
}
|
||||
info_json_bytes = json.dumps(info_json_data).encode()
|
||||
|
||||
# Get relative destination directory
|
||||
destination = str(get_relative_to_media_directory(video.video_path.parent))
|
||||
filename = f'{video_stem}.info.json'
|
||||
|
||||
# POST to /api/files/upload
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(info_json_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=info_json_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Step 4: Verify info.json is tracked
|
||||
# Refresh the session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert info.json properties exist
|
||||
assert video.info_json_path is not None, 'Video.info_json_path should exist'
|
||||
assert video.info_json_file is not None, 'Video.info_json_file should exist'
|
||||
|
||||
# Assert FileGroup has the correct number of files
|
||||
assert len(video.file_group.files) == initial_file_count + 1, \
|
||||
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
|
||||
assert video.file_group.data.get('info_json_path'), \
|
||||
'Video info.json should be in FileGroup.data'
|
||||
|
||||
# Step 5: Upload poster image and verify it's added to the FileGroup
|
||||
file_count_before_poster = len(video.file_group.files)
|
||||
|
||||
# Create image using factory
|
||||
image_data = image_bytes_factory()
|
||||
|
||||
# Upload image with same stem as the Video (using .jpg)
|
||||
poster_filename = f'{video_stem}.jpg'
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=poster_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(image_data))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=image_data, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert poster properties exist
|
||||
assert video.poster_path is not None, 'Video.poster_path should exist'
|
||||
assert video.poster_file is not None, 'Video.poster_file should exist'
|
||||
assert video.file_group.data.get('poster_path') == video.poster_path, 'Video poster should be in FileGroup.data'
|
||||
|
||||
# Assert poster is in FileGroup.files
|
||||
file_paths = [f['path'] for f in video.file_group.files]
|
||||
assert video.poster_path in file_paths, 'Poster should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(video.file_group.files) == file_count_before_poster + 1, \
|
||||
f'FileGroup should have {file_count_before_poster + 1} files after adding poster'
|
||||
|
||||
# Step 6: Upload caption file and verify it's added to the FileGroup
|
||||
file_count_before_caption = len(video.file_group.files)
|
||||
|
||||
# Create caption content
|
||||
caption_content = """WEBVTT
|
||||
|
||||
00:00:00.000 --> 00:00:02.000
|
||||
This is a test caption.
|
||||
|
||||
00:00:02.000 --> 00:00:04.000
|
||||
Another line of captions.
|
||||
"""
|
||||
|
||||
# Upload caption with same stem as the Video
|
||||
caption_filename = f'{video_stem}.en.vtt'
|
||||
caption_bytes = caption_content.encode()
|
||||
forms = [
|
||||
dict(name='destination', value=destination),
|
||||
dict(name='filename', value=caption_filename),
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='totalChunks', value='0'),
|
||||
dict(name='chunkSize', value=str(len(caption_bytes))),
|
||||
dict(name='overwrite', value='false'),
|
||||
dict(name='chunk', value=caption_bytes, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}'
|
||||
|
||||
await await_switches()
|
||||
|
||||
# Refresh session to get updated data
|
||||
test_session.expire_all()
|
||||
video = test_session.query(Video).filter_by(id=video.id).one()
|
||||
|
||||
# Assert caption properties exist
|
||||
assert video.caption_paths, 'Video.caption_paths should exist'
|
||||
assert len(video.caption_paths) == 1, 'Should have exactly one caption file'
|
||||
assert video.caption_files, 'Video.caption_files should exist'
|
||||
assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data'
|
||||
|
||||
# Assert caption is in FileGroup.files
|
||||
file_paths = [f['path'] for f in video.file_group.files]
|
||||
assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files'
|
||||
|
||||
# Assert FileGroup has correct number of files
|
||||
assert len(video.file_group.files) == file_count_before_caption + 1, \
|
||||
f'FileGroup should have {file_count_before_caption + 1} files after adding caption'
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import json
|
|||
import shutil
|
||||
from copy import copy
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -350,8 +349,6 @@ async def test_download_destination(test_session, test_directory, video_download
|
|||
"""A Video can be downloaded to a directory other than it's Channel's directory."""
|
||||
# yt-dlp would return the video json.
|
||||
mock_video_extract_info.return_value = example_video_json
|
||||
# Download process is successful with no logs.
|
||||
mock_video_process_runner.return_value = (0, {'stdout': '', 'stderr': ''})
|
||||
# This result will be ignored during the test, we just need some value so the download does not fail.
|
||||
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')
|
||||
|
||||
|
|
|
|||
|
|
@ -315,3 +315,14 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected):
|
|||
"""
|
||||
file_name = pathlib.Path(file_name)
|
||||
assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
|
||||
|
||||
|
||||
def test_video_location(test_session, video_factory, channel_factory, async_client):
|
||||
# Video location without a channel, it opens the preview.
|
||||
video1 = video_factory(title='vid1')
|
||||
assert video1.location == '/videos/video/1'
|
||||
|
||||
# Video location with a channel.
|
||||
channel = channel_factory(name='ChannelName')
|
||||
video2 = video_factory(title='vid2', channel_id=channel.id)
|
||||
assert video2.location == '/videos/channel/1/video/2'
|
||||
|
|
|
|||
|
|
@ -47,15 +47,17 @@ def get_video(video_id: int) -> Video:
|
|||
|
||||
|
||||
VIDEO_ORDERS = {
|
||||
'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC',
|
||||
'-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,'
|
||||
' LOWER(fg.primary_path) ASC',
|
||||
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
|
||||
'published_datetime': 'fg.effective_datetime ASC, LOWER(fg.primary_path) ASC',
|
||||
'-published_datetime': 'fg.effective_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
|
||||
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
|
||||
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
|
||||
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
|
||||
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
|
||||
'length': 'fg.length ASC, LOWER(fg.primary_path) ASC',
|
||||
'-length': 'fg.length DESC, LOWER(fg.primary_path) DESC',
|
||||
'size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC',
|
||||
'-size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC',
|
||||
'viewed': 'fg.viewed ASC',
|
||||
'-viewed': 'fg.viewed DESC',
|
||||
'view_count': 'v.view_count ASC',
|
||||
|
|
@ -70,6 +72,8 @@ NO_NULL_ORDERS = {
|
|||
'-length': 'fg.length IS NOT NULL',
|
||||
'size': 'fg.size IS NOT NULL',
|
||||
'-size': 'fg.size IS NOT NULL',
|
||||
'size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
|
||||
'-size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
|
||||
'view_count': 'v.view_count IS NOT NULL',
|
||||
'-view_count': 'v.view_count IS NOT NULL',
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,8 +43,12 @@ __all__ = [
|
|||
|
||||
def model_zim(file_group: FileGroup, session: Session) -> Zim:
|
||||
"""Create a Zim record for the provided FileGroup."""
|
||||
zim = Zim(file_group=file_group, path=file_group.primary_path)
|
||||
session.add(zim)
|
||||
# Check if a Zim already exists for this FileGroup
|
||||
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not zim:
|
||||
# Create new Zim if it doesn't exist
|
||||
zim = Zim(file_group=file_group, path=file_group.primary_path)
|
||||
session.add(zim)
|
||||
file_group.title = file_group.primary_path.name
|
||||
file_group.a_text = split_file_name_words(file_group.primary_path.name)
|
||||
file_group.indexed = True
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ gpsd
|
|||
gpsd-tools
|
||||
hostapd
|
||||
iotop
|
||||
iperf3
|
||||
kiwix
|
||||
kiwix-tools
|
||||
libagg-dev
|
||||
|
|
@ -98,4 +99,4 @@ unzip
|
|||
vim
|
||||
vlc
|
||||
wget
|
||||
zim-tools
|
||||
zim-tools
|
||||
|
|
|
|||
|
|
@ -53,11 +53,18 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
|
|||
# Enable first startup script.
|
||||
systemctl enable wrolpi-first-startup.service
|
||||
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd
|
||||
# Copy MOTD scripts, delete original.
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd/20-wrolpi.motd /etc/update-motd.d/20-wrolpi
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi-first.motd /etc/update-motd.d/30-wrolpi
|
||||
chmod +x /etc/update-motd.d/*
|
||||
truncate -s 0 /etc/motd
|
||||
|
||||
# NetworkManager for the hotspot.
|
||||
systemctl enable NetworkManager
|
||||
|
||||
# Enable iperf3 for speed testing.
|
||||
systemctl enable iperf3
|
||||
|
||||
set +x
|
||||
|
||||
echo
|
||||
|
|
|
|||
|
|
@ -144,7 +144,8 @@ chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || echo "Ignoring failure to chang
|
|||
chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi*
|
||||
|
||||
# Copy MOTD once the repair has been successful.
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd /etc/motd
|
||||
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi.motd /etc/update-motd.d/30-wrolpi
|
||||
chmod +x /etc/update-motd.d/*
|
||||
|
||||
systemctl restart renderd
|
||||
systemctl restart wrolpi-help
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ libzim==3.5.0
|
|||
mock==5.1.0
|
||||
psutil==6.0.0
|
||||
psycopg2-binary==2.9.9
|
||||
pypdf==5.0.1
|
||||
pypdf==6.1.3
|
||||
pytest-asyncio==0.24.0
|
||||
pytest==8.3.3
|
||||
python-docx==1.1.2
|
||||
|
|
@ -27,6 +27,5 @@ selenium==4.9.1
|
|||
srt==3.5.3
|
||||
tqdm==4.67.1
|
||||
vininfo[cli]==1.8.0
|
||||
websockets==13.1
|
||||
webvtt-py==0.5.1
|
||||
yt-dlp==2025.04.30
|
||||
yt-dlp==2025.09.26
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
docker volume create --name=openstreetmap-data
|
||||
docker volume create --name=openstreetmap-rendered-tiles
|
||||
git submodule update --init
|
||||
docker-compose pull
|
||||
docker-compose build --parallel
|
||||
docker-compose up -d db
|
||||
docker-compose run --rm api db upgrade
|
||||
docker-compose stop
|
||||
docker compose pull
|
||||
docker compose build --parallel
|
||||
docker compose up -d db
|
||||
docker compose run --rm api db upgrade
|
||||
docker compose stop
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import sys
|
||||
from asyncio import CancelledError
|
||||
from datetime import datetime, timezone, date
|
||||
from decimal import Decimal
|
||||
|
|
@ -13,6 +15,7 @@ from sanic import response, HTTPResponse, Request, Sanic, SanicException
|
|||
|
||||
from wrolpi.common import Base, get_media_directory, logger, LOGGING_CONFIG, TRACE_LEVEL
|
||||
from wrolpi.errors import APIError
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
|
@ -141,11 +144,15 @@ async def start_perpetual_tasks(app: Sanic):
|
|||
logger.debug('start_perpetual_tasks completed')
|
||||
|
||||
|
||||
def perpetual_signal(event: str = None, sleep: int | float = 1):
|
||||
def perpetual_signal(event: str = None, sleep: int | float = 1, run_while_testing: bool = False):
|
||||
"""Use Sanic signals to continually call the wrapped function. The wrapped function will continually be called,
|
||||
even if it has errors. If the function is long-running, it will only be called again after it has finished."""
|
||||
|
||||
def wrapper(func: callable):
|
||||
if PYTEST and not run_while_testing:
|
||||
# Do not run perpetual signal worker while testing, unless explicitly required.
|
||||
return func
|
||||
|
||||
# Create a Sanic "signal" for the provided function.
|
||||
event_ = event or f'wrolpi.perpetual.{func.__name__}'
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from functools import wraps
|
|||
from itertools import islice, filterfalse, tee
|
||||
from multiprocessing.managers import DictProxy
|
||||
from pathlib import Path
|
||||
from types import GeneratorType
|
||||
from types import GeneratorType, MappingProxyType
|
||||
from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
|
||||
from urllib.parse import urlparse, urlunsplit
|
||||
|
||||
|
|
@ -445,7 +445,7 @@ class ConfigFile:
|
|||
self.width = self.width or 90
|
||||
|
||||
# Do not load a global config on import while testing. A global instance should never be used for testing.
|
||||
self._config = self.default_config.copy()
|
||||
self._config = deepcopy({k: v for k, v in self.default_config.items()})
|
||||
|
||||
if self._config['version'] != 0:
|
||||
raise RuntimeError('Configs should start at version 0')
|
||||
|
|
@ -1921,19 +1921,17 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
|
|||
source = json.dumps([{'content': i} for i in entries])
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
WITH vectored AS (
|
||||
-- Convert the "source" json to a recordset.
|
||||
with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
|
||||
select
|
||||
to_tsvector('english'::regconfig, source.content) AS vector,
|
||||
source.content
|
||||
from source
|
||||
)
|
||||
SELECT
|
||||
ts_headline(vectored.content, websearch_to_tsquery(%s), 'MaxFragments=10, MaxWords=8, MinWords=7'),
|
||||
ts_rank(vectored.vector, websearch_to_tsquery(%s))
|
||||
FROM vectored
|
||||
'''
|
||||
WITH vectored AS (
|
||||
-- Convert the "source" json to a recordset.
|
||||
with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
|
||||
select to_tsvector('english'::regconfig, source.content) AS vector,
|
||||
source.content
|
||||
from source)
|
||||
SELECT ts_headline(vectored.content, websearch_to_tsquery(%s),
|
||||
'MaxFragments=10, MaxWords=8, MinWords=7'),
|
||||
ts_rank(vectored.vector, websearch_to_tsquery(%s))
|
||||
FROM vectored \
|
||||
'''
|
||||
curs.execute(stmt, [source, search_str, search_str])
|
||||
headlines = [tuple(i) for i in curs.fetchall()]
|
||||
|
||||
|
|
@ -1951,12 +1949,11 @@ async def search_other_estimates(tag_names: List[str]) -> dict:
|
|||
|
||||
with get_db_curs() as curs:
|
||||
stmt = '''
|
||||
SELECT
|
||||
COUNT(c.id)
|
||||
FROM channel c
|
||||
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
|
||||
WHERE t.name = %(tag_name)s
|
||||
'''
|
||||
SELECT COUNT(c.id)
|
||||
FROM channel c
|
||||
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
|
||||
WHERE t.name = %(tag_name)s \
|
||||
'''
|
||||
# TODO handle multiple tags
|
||||
params = dict(tag_name=tag_names[0])
|
||||
curs.execute(stmt, params)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import logging
|
|||
import multiprocessing
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import zipfile
|
||||
|
|
@ -18,7 +19,7 @@ from itertools import zip_longest
|
|||
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional
|
||||
from typing import Tuple, Set
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock, AsyncMock
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
from uuid import uuid1, uuid4
|
||||
|
||||
import pytest
|
||||
|
|
@ -185,6 +186,7 @@ async def async_client(test_directory) -> SanicASGITestClient:
|
|||
try:
|
||||
yield client
|
||||
finally:
|
||||
api_app.stop()
|
||||
logger.debug('Destroying async_client')
|
||||
|
||||
|
||||
|
|
@ -678,8 +680,8 @@ def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.P
|
|||
|
||||
@pytest.fixture
|
||||
def image_bytes_factory():
|
||||
def _() -> bytes:
|
||||
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
|
||||
def _(suffix: str = '.png') -> bytes:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
|
||||
Image.new('RGB', (1, 1), color='grey').save(fh)
|
||||
fh.seek(0)
|
||||
image = fh.read()
|
||||
|
|
@ -841,3 +843,10 @@ def mock_downloader_download_file():
|
|||
contents = contents_
|
||||
|
||||
yield set_contents
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def start_status_worker():
|
||||
"""Enable the status worker for testing."""
|
||||
with mock.patch('wrolpi.status.DISABLE_STATUS_WORKER', False):
|
||||
yield
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import ctypes
|
||||
import multiprocessing
|
||||
import queue
|
||||
|
||||
from sanic import Sanic
|
||||
|
||||
|
|
@ -54,6 +55,7 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.switches = manager.dict()
|
||||
app.shared_ctx.switches_lock = multiprocessing.Lock()
|
||||
app.shared_ctx.switches_changed = multiprocessing.Event()
|
||||
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
|
||||
|
||||
# Warnings
|
||||
app.shared_ctx.warn_once = manager.dict()
|
||||
|
|
@ -110,6 +112,12 @@ def reset_shared_contexts(app: Sanic):
|
|||
# Switches
|
||||
app.shared_ctx.switches.clear()
|
||||
app.shared_ctx.switches_changed.clear()
|
||||
while True:
|
||||
# Clear out any pending singlefile archive switches.
|
||||
try:
|
||||
app.shared_ctx.archive_singlefiles.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
# Events.
|
||||
app.shared_ctx.single_tasks_started.clear()
|
||||
|
|
|
|||
|
|
@ -689,6 +689,9 @@ class DownloadManager:
|
|||
download.sub_downloader = sub_downloader or None
|
||||
# Remove Channel relationship, if necessary.
|
||||
download.channel_id = (settings or dict()).get('channel_id')
|
||||
|
||||
save_downloads_config.activate_switch()
|
||||
|
||||
return download
|
||||
|
||||
@wrol_mode_check
|
||||
|
|
@ -1113,11 +1116,13 @@ class DownloadManager:
|
|||
|
||||
@staticmethod
|
||||
def add_to_skip_list(*urls: str):
|
||||
get_download_manager_config().skip_urls = list(set(get_download_manager_config().skip_urls) | set(urls))
|
||||
get_download_manager_config().skip_urls = sorted(list(set(get_download_manager_config().skip_urls) | set(urls)))
|
||||
|
||||
@staticmethod
|
||||
def remove_from_skip_list(url: str):
|
||||
get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
|
||||
skip_urls = get_download_manager_config().skip_urls
|
||||
if url in skip_urls:
|
||||
get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
|
||||
|
||||
|
||||
# The global DownloadManager. This should be used everywhere!
|
||||
|
|
|
|||
|
|
@ -96,6 +96,10 @@ class Events:
|
|||
def send_archive_uploaded(cls, message: str = None, url: str = None):
|
||||
send_event('upload_archive', message, subject='upload', url=url)
|
||||
|
||||
@classmethod
|
||||
def send_upload_archive_failed(cls, message: str = None):
|
||||
send_event('upload_archive_failed', message, subject='upload')
|
||||
|
||||
|
||||
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
|
||||
log = f'{event=}'
|
||||
|
|
|
|||
|
|
@ -228,8 +228,12 @@ class EBook(ModelHelper, Base):
|
|||
|
||||
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
|
||||
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
|
||||
ebook = EBook(file_group=file_group)
|
||||
session.add(ebook)
|
||||
# Check if an EBook already exists for this FileGroup
|
||||
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
|
||||
if not ebook:
|
||||
# Create new EBook if it doesn't exist
|
||||
ebook = EBook(file_group=file_group)
|
||||
session.add(ebook)
|
||||
|
||||
# Multiple formats may be available.
|
||||
epub_file = next(iter(ebook.file_group.my_epub_files()), None)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
|
|||
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
|
||||
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.downloader import download_manager, Download
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
|
||||
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError
|
||||
from wrolpi.events import Events
|
||||
|
|
@ -856,7 +856,6 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
FROM file_group fg
|
||||
{join}
|
||||
{f"WHERE {wheres}" if wheres else ""}
|
||||
GROUP BY fg.id
|
||||
ORDER BY {order_by}
|
||||
OFFSET %(offset)s LIMIT %(limit)s
|
||||
'''
|
||||
|
|
@ -1000,23 +999,23 @@ def split_file_name_words(name: str) -> str:
|
|||
def get_file_statistics():
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT
|
||||
-- All items in file_group.files are real individual files.
|
||||
SUM(json_array_length(files)) AS "total_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype = 'application/x-mobipocket-ebook') AS "ebook_count",
|
||||
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
|
||||
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
|
||||
(SELECT COUNT(*) FROM tag) AS "tags_count",
|
||||
SUM(size)::BIGINT AS "total_size",
|
||||
(SELECT COUNT(*) FROM archive) AS archive_count
|
||||
FROM
|
||||
file_group
|
||||
''')
|
||||
SELECT
|
||||
-- All items in file_group.files are real individual files.
|
||||
SUM(json_array_length(files)) AS "total_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
|
||||
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
|
||||
'application/x-mobipocket-ebook') AS "ebook_count",
|
||||
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
|
||||
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
|
||||
(SELECT COUNT(*) FROM tag) AS "tags_count",
|
||||
SUM(size)::BIGINT AS "total_size",
|
||||
(SELECT COUNT(*) FROM archive) AS archive_count
|
||||
FROM file_group
|
||||
''')
|
||||
statistics = dict(curs.fetchall()[0])
|
||||
statistics['total_count'] = statistics['total_count'] or 0
|
||||
statistics['total_size'] = statistics['total_size'] or 0
|
||||
|
|
@ -1137,25 +1136,25 @@ def get_refresh_progress() -> RefreshProgress:
|
|||
idempotency = api_app.shared_ctx.refresh.get('idempotency')
|
||||
if idempotency:
|
||||
stmt = '''
|
||||
SELECT
|
||||
-- Sum all the files in each FileGroup.
|
||||
SUM(json_array_length(files)) FILTER (WHERE idempotency=%(idempotency)s) AS "total_file_groups",
|
||||
COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency=%(idempotency)s) AS "indexed",
|
||||
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
|
||||
COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency=%(idempotency)s) AS "modeled"
|
||||
FROM file_group
|
||||
'''
|
||||
SELECT
|
||||
-- Sum all the files in each FileGroup.
|
||||
SUM(json_array_length(files)) FILTER (WHERE idempotency = %(idempotency)s) AS "total_file_groups",
|
||||
COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency = %(idempotency)s) AS "indexed",
|
||||
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
|
||||
COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency = %(idempotency)s) AS "modeled"
|
||||
FROM file_group \
|
||||
'''
|
||||
else:
|
||||
# Idempotency has not yet been declared.
|
||||
stmt = '''
|
||||
SELECT
|
||||
-- Sum all the files in each FileGroup.
|
||||
SUM(json_array_length(files)) AS "total_file_groups",
|
||||
COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
|
||||
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
|
||||
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
|
||||
FROM file_group
|
||||
'''
|
||||
SELECT
|
||||
-- Sum all the files in each FileGroup.
|
||||
SUM(json_array_length(files)) AS "total_file_groups",
|
||||
COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
|
||||
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
|
||||
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
|
||||
FROM file_group \
|
||||
'''
|
||||
|
||||
with get_db_curs() as curs:
|
||||
curs.execute(stmt, dict(idempotency=idempotency))
|
||||
|
|
@ -1583,52 +1582,61 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
|
|||
logger.warning('upsert_file called, but all files are in ignored directories!')
|
||||
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
|
||||
|
||||
for i in range(2):
|
||||
# Try multiple times because uploads happen concurrently and may conflict.
|
||||
# TODO convert uploads to synchronous in UI.
|
||||
with get_db_session() as session:
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
# Re-index the contents of the file.
|
||||
# Use a single session context for all database operations
|
||||
with get_db_session(commit=True) as session:
|
||||
# Create/update FileGroup with retry logic
|
||||
file_group = None
|
||||
for i in range(2):
|
||||
try:
|
||||
session.flush([file_group, ])
|
||||
file_group_id = file_group.id
|
||||
session.commit()
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
session.flush([file_group])
|
||||
break
|
||||
except IntegrityError:
|
||||
# Another process inserted this FileGroup.
|
||||
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
|
||||
session.rollback()
|
||||
if i == 1:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
|
||||
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_group = FileGroup.find_by_id(file_group_id, session)
|
||||
if not file_group:
|
||||
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
|
||||
|
||||
logger.debug(f'upsert_file: {file_group}')
|
||||
|
||||
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
|
||||
try:
|
||||
file_group.do_model(session)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
||||
# If user uploads a file, then remove it from the skip list so comments can be downloaded.
|
||||
url = file_group.url if file_group and file_group.url else None
|
||||
if url and download_manager.is_skipped(url):
|
||||
download_manager.remove_from_skip_list(url)
|
||||
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
|
||||
# Modify the download (if any) so that the user can click on it and view the uploaded file.
|
||||
if file_group.url:
|
||||
if download_manager.is_skipped(file_group.url):
|
||||
download_manager.remove_from_skip_list(file_group.url)
|
||||
if download := Download.get_by_url(file_group.url, session):
|
||||
# Mark download as completed
|
||||
download.complete()
|
||||
# Link the download to the location of the uploaded file.
|
||||
model = file_group.get_model_record()
|
||||
download.location = model.location if model else file_group.location
|
||||
|
||||
upsert_directories([], file.parents)
|
||||
|
||||
session.commit()
|
||||
|
||||
if tag_names:
|
||||
with get_db_session(commit=True) as session:
|
||||
file_group = FileGroup.find_by_id(file_group_id, session)
|
||||
# Add tags if provided
|
||||
if tag_names:
|
||||
for tag_name in tag_names:
|
||||
if tag_name not in file_group.tag_names:
|
||||
tag = Tag.get_by_name(tag_name)
|
||||
file_group.add_tag(tag.id)
|
||||
tag = Tag.get_by_name(tag_name, session)
|
||||
file_group.add_tag(tag.id, session=session)
|
||||
|
||||
# Commit all changes in one transaction
|
||||
session.commit()
|
||||
|
||||
# Upsert directories after FileGroup is committed
|
||||
upsert_directories([], file.parents)
|
||||
|
||||
file_group.flush()
|
||||
return file_group
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@ from datetime import datetime
|
|||
from functools import singledispatchmethod
|
||||
from typing import List, Type, Optional, Iterable
|
||||
|
||||
from sqlalchemy import Column, String, Computed, BigInteger, Boolean
|
||||
from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.orm import deferred, relationship, Session
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
|
||||
get_relative_to_media_directory, unique_by_predicate
|
||||
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
|
||||
from wrolpi.db import optional_session
|
||||
from wrolpi.db import optional_session, get_db_session
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.errors import FileGroupIsTagged, UnknownFile
|
||||
from wrolpi.files import indexers
|
||||
|
|
@ -87,6 +87,10 @@ class FileGroup(ModelHelper, Base):
|
|||
url = Column(String) # the location where this file can be downloaded.
|
||||
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
|
||||
|
||||
# Columns updated by triggers.
|
||||
# `file_group_effective_datetime_trigger` and `update_effective_datetime` event handler
|
||||
effective_datetime = Column(TZDateTime) # Equivalent to COALESCE(published_datetime, download_datetime)
|
||||
|
||||
tag_files: Iterable[TagFile] = relationship('TagFile', cascade='all')
|
||||
|
||||
a_text = deferred(Column(String))
|
||||
|
|
@ -119,7 +123,8 @@ class FileGroup(ModelHelper, Base):
|
|||
if PYTEST:
|
||||
raise
|
||||
# Log the error but continue processing other tags
|
||||
logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
|
||||
logger.error(
|
||||
f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
|
||||
tags = sorted(tag_names)
|
||||
d = dict(
|
||||
author=self.author,
|
||||
|
|
@ -275,6 +280,14 @@ class FileGroup(ModelHelper, Base):
|
|||
def my_mobi_files(self) -> List[dict]:
|
||||
return self.my_files('application/x-mobipocket-ebook')
|
||||
|
||||
def my_html_files(self) -> List[dict]:
|
||||
html_files = self.my_files('text/html', 'application/octet-stream')
|
||||
html_files = [i for i in html_files if i['path'].name.endswith('.html')]
|
||||
return html_files
|
||||
|
||||
def my_html_paths(self) -> List[pathlib.Path]:
|
||||
return [i['path'] for i in self.my_html_files()]
|
||||
|
||||
def delete(self, add_to_skip_list: bool = True):
|
||||
"""Delete this FileGroup record, and all of its files.
|
||||
|
||||
|
|
@ -385,7 +398,8 @@ class FileGroup(ModelHelper, Base):
|
|||
if PYTEST:
|
||||
raise
|
||||
# Log the error but continue processing other tags
|
||||
logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
|
||||
logger.error(
|
||||
f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
|
||||
return result
|
||||
|
||||
def merge(self, file_groups: List['FileGroup']):
|
||||
|
|
@ -456,6 +470,14 @@ class FileGroup(ModelHelper, Base):
|
|||
@property
|
||||
def location(self):
|
||||
"""Returns the URL that the FileGroup can be previewed."""
|
||||
klass = self.get_model_class()
|
||||
if klass:
|
||||
# Return the model location when possible.
|
||||
with get_db_session() as session:
|
||||
instance = klass.get_by_path(self.primary_path, session)
|
||||
if instance and hasattr(instance, 'location'):
|
||||
return instance.location
|
||||
|
||||
parent = str(get_relative_to_media_directory(self.primary_path.parent))
|
||||
preview = str(get_relative_to_media_directory(self.primary_path))
|
||||
if parent == '.':
|
||||
|
|
@ -481,16 +503,20 @@ class FileGroup(ModelHelper, Base):
|
|||
elif Zim.can_model(self):
|
||||
return Zim
|
||||
|
||||
def get_model_record(self) -> Optional[ModelHelper]:
|
||||
klass = self.get_model_class()
|
||||
if klass:
|
||||
with get_db_session() as session:
|
||||
return klass.get_by_path(self.primary_path, session)
|
||||
return None
|
||||
|
||||
def do_model(self, session: Session) -> Optional[ModelHelper]:
|
||||
"""Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
|
||||
if model_class := self.get_model_class():
|
||||
if model := model_class.get_by_path(self.primary_path, session):
|
||||
# Already modeled.
|
||||
logger.debug(f'already modeled: {self} {model}')
|
||||
return model
|
||||
# Create new model (Video/Archive/Ebook).
|
||||
# Always call do_model() to ensure FileGroup.data is updated when files change.
|
||||
# The model's do_model() should be idempotent (safe to call multiple times).
|
||||
model = model_class.do_model(self, session)
|
||||
logger.debug(f'new model: {self} {model}')
|
||||
logger.debug(f'modeled: {self} {model}')
|
||||
return model
|
||||
|
||||
# Index the file if no models are available.
|
||||
|
|
@ -515,6 +541,12 @@ class FileGroup(ModelHelper, Base):
|
|||
raise
|
||||
|
||||
|
||||
@event.listens_for(FileGroup, 'before_insert')
|
||||
@event.listens_for(FileGroup, 'before_update')
|
||||
def update_effective_datetime(mapper, connection, target):
|
||||
target.effective_datetime = target.published_datetime or target.download_datetime
|
||||
|
||||
|
||||
class Directory(ModelHelper, Base):
|
||||
"""A representation of a file directory in the media directory."""
|
||||
__tablename__ = 'directory'
|
||||
|
|
|
|||
|
|
@ -671,9 +671,12 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
|
|||
assert not file_group.tag_names, 'No tags this time.'
|
||||
assert file_group.indexed, 'File should be indexed after upload.'
|
||||
assert file_group.model, 'File should be modeled'
|
||||
assert isinstance(file_group.get_model_record(), Video)
|
||||
assert file_group.location == '/videos/video/2'
|
||||
# Video was modeled, but has no Channel.
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path
|
||||
assert video.file_group == file_group
|
||||
assert not video.info_json_path
|
||||
assert not video.channel and not video.channel_id
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
|
|||
from wrolpi.cmd import which, run_command
|
||||
from wrolpi.common import logger, get_warn_once, unique_by_predicate, partition, TRACE_LEVEL
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
import psutil
|
||||
|
|
@ -565,11 +566,18 @@ async def get_power_stats() -> PowerStats:
|
|||
return PowerStats()
|
||||
|
||||
|
||||
DISABLE_STATUS_WORKER = bool(PYTEST) # Do not run status worker while testing, unless explicitly needed.
|
||||
|
||||
|
||||
@perpetual_signal(sleep=5)
|
||||
async def status_worker(count: int = None, sleep_time: int = 5):
|
||||
"""A background process which will gather historical data about system statistics."""
|
||||
shared_status = api_app.shared_ctx.status
|
||||
|
||||
if PYTEST and DISABLE_STATUS_WORKER:
|
||||
# Tests are running, status worker has not been enabled by `start_status_worker` fixture.
|
||||
return
|
||||
|
||||
if count is not None and count <= 0:
|
||||
logger.debug('Status worker ran out of count')
|
||||
return dict(**shared_status)
|
||||
|
|
@ -579,13 +587,13 @@ async def status_worker(count: int = None, sleep_time: int = 5):
|
|||
# Update global `status` dict with stats that are gathered instantly.
|
||||
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \
|
||||
await asyncio.gather(
|
||||
get_cpu_stats(),
|
||||
get_load_stats(),
|
||||
get_drives_stats(),
|
||||
get_memory_stats(),
|
||||
get_processes_stats(),
|
||||
get_iostat_stats(),
|
||||
get_power_stats(),
|
||||
asyncio.create_task(get_cpu_stats(), name='get_cpu_stats'),
|
||||
asyncio.create_task(get_load_stats(), name='get_load_stats'),
|
||||
asyncio.create_task(get_drives_stats(), name='get_drives_stats'),
|
||||
asyncio.create_task(get_memory_stats(), name='get_memory_stats'),
|
||||
asyncio.create_task(get_processes_stats(), name='get_processes_stats'),
|
||||
asyncio.create_task(get_iostat_stats(), name='get_iostat_stats'),
|
||||
asyncio.create_task(get_power_stats(), name='get_power_stats'),
|
||||
)
|
||||
shared_status.update({
|
||||
'cpu_stats': cpu_stats.__json__(),
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def register_switch_handler(switch_name: str):
|
|||
DEBUG_LOGGED = False
|
||||
|
||||
|
||||
@perpetual_signal(sleep=0.1)
|
||||
@perpetual_signal(sleep=0.1, run_while_testing=True)
|
||||
async def switch_worker():
|
||||
"""Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time."""
|
||||
global DEBUG_LOGGED
|
||||
|
|
|
|||
|
|
@ -440,6 +440,12 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
|
|||
assert response.status_code == HTTPStatus.CREATED
|
||||
|
||||
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
|
||||
# Assert that the config has been saved
|
||||
config = get_download_manager_config()
|
||||
assert len(config.downloads) == 1
|
||||
assert config.downloads[0]['url'] == 'https://example.com/1'
|
||||
assert config.downloads[0]['frequency'] == 1_000
|
||||
assert config.downloads[0]['downloader'] == 'archive'
|
||||
|
||||
# Create two once-downloads.
|
||||
body = dict(
|
||||
|
|
@ -451,6 +457,10 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
|
|||
assert response.status_code == HTTPStatus.CREATED
|
||||
|
||||
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
|
||||
# Assert that the config has been saved
|
||||
config = get_download_manager_config()
|
||||
assert len(config.downloads) == 3
|
||||
assert {d['url'] for d in config.downloads} == {f'https://example.com/{i}' for i in range(1, 4)}
|
||||
|
||||
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
|
||||
assert not download1.settings
|
||||
|
|
@ -473,6 +483,11 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
|
|||
download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
|
||||
assert download2.__json__() == expected_download
|
||||
assert download2.tag_names == [tag1.name, tag2.name]
|
||||
# Assert that the config has been saved
|
||||
config = get_download_manager_config()
|
||||
assert len(config.downloads) == 3
|
||||
download_config = next(d for d in config.downloads if d['url'] == 'https://example.com/1')
|
||||
assert download_config['tag_names'] == [tag1.name, tag2.name]
|
||||
|
||||
|
||||
def test_get_downloaders(test_client):
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ async def test_get_drives_stats(async_client):
|
|||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_worker(async_client):
|
||||
async def test_status_worker(async_client, start_status_worker):
|
||||
"""Status worker calls itself perpetually, but can be limited during testing."""
|
||||
from wrolpi.api_utils import api_app
|
||||
assert async_client.sanic_app.shared_ctx.status['cpu_stats'] == dict()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue