Merge branch 'master' into bugfix/gis-map-template0

This commit is contained in:
lrnselfreliance 2025-11-24 07:11:56 -07:00 • committed by GitHub
commit f5474ea4ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
59 changed files with 1673 additions and 496 deletions

View file

@ -0,0 +1,26 @@
"""Lower FileGroup.primary_path
Revision ID: 158973fda20f
Revises: 69e9754d1a63
Create Date: 2025-08-16 10:54:58.874094
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '158973fda20f'
down_revision = '69e9754d1a63'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
op.execute('CREATE INDEX file_group_primary_path_lower_idx ON file_group (LOWER(primary_path))')
def downgrade():
op.execute('DROP INDEX IF EXISTS file_group_primary_path_lower_idx')

View file

@ -0,0 +1,47 @@
"""file_group.effective_date trigger
Revision ID: 4f03b9548f6e
Revises: 54c19aa847da
Create Date: 2025-08-17 10:49:35.754548
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '4f03b9548f6e'
down_revision = '54c19aa847da'
branch_labels = None
depends_on = None
def upgrade():
# Create trigger function
op.execute("""
CREATE OR REPLACE FUNCTION update_effective_datetime()
RETURNS TRIGGER AS
$$
BEGIN
NEW.effective_datetime = COALESCE(NEW.published_datetime, NEW.download_datetime);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
# Create trigger
op.execute("""
CREATE TRIGGER file_group_effective_datetime_trigger
BEFORE INSERT OR UPDATE OF published_datetime, download_datetime
ON file_group
FOR EACH ROW
EXECUTE FUNCTION update_effective_datetime();
""")
def downgrade():
# Drop trigger
op.execute("DROP TRIGGER IF EXISTS file_group_effective_datetime_trigger ON file_group")
# Drop trigger function
op.execute("DROP FUNCTION IF EXISTS update_effective_datetime")

View file

@ -0,0 +1,145 @@
"""Channel summary columns, triggers and indexes.
Revision ID: 54c19aa847da
Revises: 158973fda20f
Create Date: 2025-08-16 11:45:48.190427
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '54c19aa847da'
down_revision = '158973fda20f'
branch_labels = None
depends_on = None
def upgrade():
# Add columns
op.execute("""
ALTER TABLE channel
ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL,
ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL,
ADD COLUMN minimum_frequency INTEGER;
""")
# Populate columns
op.execute("""
UPDATE channel
SET video_count = (SELECT COUNT(*)
FROM video
WHERE channel_id = channel.id),
total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
FROM video v
LEFT JOIN file_group fg ON v.file_group_id = fg.id
WHERE v.channel_id = channel.id),
minimum_frequency = (SELECT MIN(frequency)
FROM download
WHERE channel_id = channel.id);
""")
# Create trigger functions
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_video_count()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id)
WHERE id = NEW.channel_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_size()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT
FROM video v
LEFT JOIN file_group fg ON v.file_group_id = fg.id
WHERE v.channel_id = NEW.id)
WHERE id = NEW.id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
op.execute("""
CREATE OR REPLACE FUNCTION update_channel_minimum_frequency()
RETURNS TRIGGER AS
$$
BEGIN
UPDATE channel
SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id)
WHERE id = NEW.channel_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
""")
# Create triggers
op.execute("""
CREATE TRIGGER video_count_trigger
AFTER INSERT OR DELETE
ON video
FOR EACH ROW
EXECUTE FUNCTION update_channel_video_count();
""")
op.execute("""
CREATE TRIGGER video_size_trigger
AFTER INSERT OR DELETE
ON video
FOR EACH ROW
EXECUTE FUNCTION update_channel_size();
""")
op.execute("""
CREATE TRIGGER file_group_size_trigger
AFTER UPDATE OF size
ON file_group
FOR EACH ROW
WHEN (OLD.size IS DISTINCT FROM NEW.size)
EXECUTE FUNCTION update_channel_size();
""")
op.execute("""
CREATE TRIGGER download_frequency_trigger
AFTER INSERT OR UPDATE OF frequency OR DELETE
ON download
FOR EACH ROW
EXECUTE FUNCTION update_channel_minimum_frequency();
""")
# Create indexes
op.execute("""
CREATE INDEX channel_video_count_idx ON channel (video_count);
CREATE INDEX channel_total_size_idx ON channel (total_size);
CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency);
""")
def downgrade():
# Drop triggers
op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;")
op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;")
op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;")
op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;")
# Drop trigger functions
op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;")
op.execute("DROP FUNCTION IF EXISTS update_channel_size;")
op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;")
# Drop indexes
op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;")
op.execute("DROP INDEX IF EXISTS channel_total_size_idx;")
op.execute("DROP INDEX IF EXISTS channel_video_count_idx;")
# Drop columns
op.execute("""
ALTER TABLE channel
DROP COLUMN IF EXISTS minimum_frequency,
DROP COLUMN IF EXISTS total_size,
DROP COLUMN IF EXISTS video_count;
""")

View file

@ -0,0 +1,37 @@
"""FileGroup.effective_datetime
Revision ID: 69e9754d1a63
Revises: f7229a67e333
Create Date: 2025-08-16 10:28:42.661108
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '69e9754d1a63'
down_revision = 'f7229a67e333'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
op.execute('ALTER TABLE file_group ADD COLUMN IF NOT EXISTS effective_datetime TIMESTAMPTZ')
# Populating effective_datetime for existing rows
op.execute("""
UPDATE file_group
SET effective_datetime = COALESCE(published_datetime, download_datetime)
WHERE published_datetime IS NOT NULL
OR download_datetime IS NOT NULL;
""")
op.execute('CREATE INDEX IF NOT EXISTS file_group_effective_datetime_idx ON file_group(effective_datetime)')
def downgrade():
op.execute('DROP INDEX IF EXISTS file_group_effective_datetime_idx')
op.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS effective_datetime')

274
app/package-lock.json generated
View file

@ -2236,12 +2236,14 @@
}
},
"node_modules/@cypress/request/node_modules/form-data": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.12"
},
"engines": {
@ -5574,6 +5576,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/callsites": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@ -7164,6 +7178,19 @@
"resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.6.tgz",
"integrity": "sha512-+3NaRjWktb5r61ZFoDejlykPEFKT5N/LkbXsaddlw6xNSXBanUYpFc2AXXpbJDilPHazcSreU/DpQIaxfX0NfQ=="
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/duplexer": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
@ -7342,12 +7369,9 @@
"integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA=="
},
"node_modules/es-define-property": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
"integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
"dependencies": {
"get-intrinsic": "^1.2.4"
},
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"engines": {
"node": ">= 0.4"
}
@ -7384,14 +7408,26 @@
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
"integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
},
"node_modules/es-set-tostringtag": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
"integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
"node_modules/es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"dependencies": {
"get-intrinsic": "^1.1.3",
"has": "^1.0.3",
"has-tostringtag": "^1.0.0"
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
@ -8866,13 +8902,15 @@
}
},
"node_modules/form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz",
"integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.35"
},
"engines": {
"node": ">= 6"
@ -8992,15 +9030,20 @@
}
},
"node_modules/get-intrinsic": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
"integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"has-proto": "^1.0.1",
"has-symbols": "^1.0.3",
"hasown": "^2.0.0"
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
@ -9022,6 +9065,18 @@
"node": ">=8.0.0"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@ -9198,11 +9253,11 @@
}
},
"node_modules/gopd": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
"dependencies": {
"get-intrinsic": "^1.1.3"
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@ -9292,9 +9347,9 @@
}
},
"node_modules/has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"engines": {
"node": ">= 0.4"
},
@ -9303,11 +9358,11 @@
}
},
"node_modules/has-tostringtag": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"dependencies": {
"has-symbols": "^1.0.2"
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
@ -12841,6 +12896,14 @@
"tmpl": "1.0.5"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/mathjs": {
"version": "14.0.1",
"resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.0.1.tgz",
@ -17367,9 +17430,9 @@
"integrity": "sha512-XHjoxak8SFQnHnmYHb3PcnW5TZ+9ErLZemZei3azuIRhQLw4IExsVbL3VZJdHcLeNaXq6NqawgpDPpjBOg4B5g=="
},
"node_modules/tmp": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
"integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.4.tgz",
"integrity": "sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ==",
"engines": {
"node": ">=14.14"
}
@ -20256,12 +20319,14 @@
},
"dependencies": {
"form-data": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
"integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.12"
}
},
@ -22700,6 +22765,15 @@
"set-function-length": "^1.2.1"
}
},
"call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"requires": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
}
},
"callsites": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@ -23831,6 +23905,16 @@
"resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.6.tgz",
"integrity": "sha512-+3NaRjWktb5r61ZFoDejlykPEFKT5N/LkbXsaddlw6xNSXBanUYpFc2AXXpbJDilPHazcSreU/DpQIaxfX0NfQ=="
},
"dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"requires": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
}
},
"duplexer": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
@ -23976,12 +24060,9 @@
"integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA=="
},
"es-define-property": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
"integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
"requires": {
"get-intrinsic": "^1.2.4"
}
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="
},
"es-errors": {
"version": "1.3.0",
@ -24009,14 +24090,23 @@
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
"integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw=="
},
"es-set-tostringtag": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz",
"integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==",
"es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"requires": {
"get-intrinsic": "^1.1.3",
"has": "^1.0.3",
"has-tostringtag": "^1.0.0"
"es-errors": "^1.3.0"
}
},
"es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"requires": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
}
},
"es-shim-unscopables": {
@ -25082,13 +25172,15 @@
}
},
"form-data": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.4.tgz",
"integrity": "sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.35"
}
},
"forwarded": {
@ -25164,15 +25256,20 @@
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-intrinsic": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
"integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"requires": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"has-proto": "^1.0.1",
"has-symbols": "^1.0.3",
"hasown": "^2.0.0"
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
}
},
"get-own-enumerable-property-symbols": {
@ -25185,6 +25282,15 @@
"resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz",
"integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q=="
},
"get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"requires": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
}
},
"get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@ -25311,12 +25417,9 @@
}
},
"gopd": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
"integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
"requires": {
"get-intrinsic": "^1.1.3"
}
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="
},
"graceful-fs": {
"version": "4.2.11",
@ -25378,16 +25481,16 @@
"integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg=="
},
"has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="
},
"has-tostringtag": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz",
"integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"requires": {
"has-symbols": "^1.0.2"
"has-symbols": "^1.0.3"
}
},
"hasown": {
@ -27942,6 +28045,11 @@
"tmpl": "1.0.5"
}
},
"math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="
},
"mathjs": {
"version": "14.0.1",
"resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.0.1.tgz",
@ -31052,9 +31160,9 @@
"integrity": "sha512-XHjoxak8SFQnHnmYHb3PcnW5TZ+9ErLZemZei3azuIRhQLw4IExsVbL3VZJdHcLeNaXq6NqawgpDPpjBOg4B5g=="
},
"tmp": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
"integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w=="
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.4.tgz",
"integrity": "sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ=="
},
"tmpl": {
"version": "1.0.5",

View file

@ -101,6 +101,10 @@ function handleEvents(events) {
() => window.open(url, '_self'));
}
if (event === 'upload_archive_failed') {
eventToast('Archive Upload Failed!', message, 'error', 5000);
}
if (subject) {
newestEvents[subject] = dt;
}

View file

@ -271,8 +271,13 @@ function VideoDownloadCountLimit({form, name = 'video_count_limit', path = 'sett
/>
}
export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions'}) {
const [inputProps, inputAttrs] = form.getSelectionProps({name, path, type: 'array'});
export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions', onChange}) {
const [inputProps, inputAttrs] = form.getSelectionProps({
name,
path,
type: 'array',
afterChange: onChange
});
return <>
<InfoHeader
@ -382,13 +387,17 @@ export function UseBrowserProfile({form}) {
/>
}
export function VideosDownloadForm({singleDownload = true, onCancel}) {
export function VideosDownloadForm({singleDownload = true, onCancel, onSuccess: propOnSuccess, download, submitter: propSubmitter, actions}) {
const [showMessage, setShowMessage] = React.useState(false);
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
const [config, setConfig] = React.useState(null);
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
// Keep video format in session to help user start downloads consistently.
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
// Use config video resolutions if available, otherwise use default
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
const defaultFormData = {
urls: '', // Textarea, one URL per line.
destination: '',
@ -397,11 +406,11 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
settings: {
use_browser_profile: false,
video_format: defaultVideoFormat,
video_resolutions: defaultVideoResolutions,
video_resolutions: configResolutions,
}
}
const submitter = async (formData) => {
const submitter = propSubmitter || (async (formData) => {
const downloadData = {
destination: formData.destination,
downloader: formData.downloader,
@ -410,24 +419,45 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
urls: formData.urls.split(/r?\n/),
}
await postDownload(downloadData);
}
});
const onSuccess = () => {
setShowMessage(true);
form.reset();
if (propOnSuccess) {
propOnSuccess();
} else {
form.reset();
}
}
const form = useForm({
submitter,
defaultFormData,
defaultFormData: download ? mergeDeep(defaultFormData, download) : defaultFormData,
onSuccess,
});
// Fetch video downloader config when the modal opens
React.useEffect(() => {
const {video_resolutions, video_format} = form.formData.settings;
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
setDefaultVideoResolutions(video_resolutions);
}
const fetchConfig = async () => {
try {
const result = await fetchVideoDownloaderConfig();
setConfig(result);
// Update form with video resolutions from config only if user hasn't changed them
// and if no download object was provided (this is a new download, not an edit)
if (result && result.video_resolutions && !userChangedResolutions && !download) {
form.setValue('settings.video_resolutions', result.video_resolutions);
}
} catch (error) {
console.error('Failed to fetch video downloader config:', error);
}
};
fetchConfig();
}, []);
React.useEffect(() => {
const {video_format} = form.formData.settings;
if (video_format && video_format !== defaultVideoFormat) {
setDefaultVideoFormat(video_format);
}
@ -473,7 +503,10 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
</Grid.Row>
<Grid.Row columns={2}>
<Grid.Column width={11}>
<VideoResolutionSelectorForm form={form}/>
<VideoResolutionSelectorForm
form={form}
onChange={() => setUserChangedResolutions(true)}
/>
</Grid.Column>
<Grid.Column width={4}>
<VideoFormatSelectorForm form={form}/>
@ -492,7 +525,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) {
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column>
<DownloadFormButtons onCancel={localOnCancel} form={form}/>
{actions ? actions({onCancel: localOnCancel, form}) : <DownloadFormButtons onCancel={localOnCancel} form={form}/>}
</Grid.Column>
</Grid.Row>
</Grid>
@ -511,6 +544,40 @@ export function VideoMinimumDurationForm({form}) {
/>
}
export function EditVideosDownloadForm({
download,
onCancel,
onSuccess,
onDelete,
actions = EditDownloadFormButtons,
}) {
const submitter = async (formData) => {
// Create a copy of settings without channel_url and channel_id
const settings = {...formData.settings};
// These settings are created when downloading the video, we can't submit them again.
delete settings.channel_url;
delete settings.channel_id;
const downloadData = {
destination: formData.destination,
downloader: formData.downloader,
settings: settings,
tag_names: formData.tag_names,
urls: formData.urls.split(/r?\n/),
}
await putDownload(download.id, downloadData);
}
return <VideosDownloadForm
singleDownload={true}
onCancel={onCancel}
onSuccess={onSuccess}
download={download}
submitter={submitter}
actions={(props) => actions({...props, onDelete})}
/>
}
export function VideoMaximumDurationForm({form}) {
return <VideoDurationLimit
form={form}
@ -535,6 +602,9 @@ export function ChannelDownloadForm({
channel_id = null,
}) {
const [showMessage, setShowMessage] = React.useState(false);
const [config, setConfig] = React.useState(null);
const [isConfigLoaded, setIsConfigLoaded] = React.useState(false);
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
// May have received submitter from EditChannelDownloadForm.
submitter = submitter || (async (formData) => {
@ -557,10 +627,12 @@ export function ChannelDownloadForm({
}
}
// Keep video settings in session to help user start downloads consistently.
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
// Keep video format in session to help user start downloads consistently.
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
// Use config video resolutions if available, otherwise use default
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
const emptyFormData = {
destination: '',
downloader: Downloaders.VideoChannel,
@ -577,7 +649,7 @@ export function ChannelDownloadForm({
use_browser_profile: false,
video_count_limit: null,
video_format: defaultVideoFormat,
video_resolutions: defaultVideoResolutions,
video_resolutions: configResolutions,
},
sub_downloader: Downloaders.Video,
tag_names: [],
@ -591,11 +663,30 @@ export function ChannelDownloadForm({
clearOnSuccess,
});
// Fetch video downloader config when the modal opens
React.useEffect(() => {
const {video_resolutions, video_format} = form.formData.settings;
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
setDefaultVideoResolutions(video_resolutions);
}
const fetchConfig = async () => {
try {
const result = await fetchVideoDownloaderConfig();
setConfig(result);
setIsConfigLoaded(true);
// Update form with video resolutions from config only if user hasn't changed them
// and if no download object was provided (this is a new download, not an edit)
if (result && result.video_resolutions && !userChangedResolutions && !download) {
form.setValue('settings.video_resolutions', result.video_resolutions);
}
} catch (error) {
console.error('Failed to fetch video downloader config:', error);
setIsConfigLoaded(true);
}
};
fetchConfig();
}, []);
React.useEffect(() => {
const {video_format} = form.formData.settings;
if (video_format && video_format !== defaultVideoFormat) {
setDefaultVideoFormat(video_format);
}
@ -656,7 +747,10 @@ export function ChannelDownloadForm({
</Grid.Row>
<Grid.Row columns={2}>
<Grid.Column width={11}>
<VideoResolutionSelectorForm form={form}/>
<VideoResolutionSelectorForm
form={form}
onChange={() => setUserChangedResolutions(true)}
/>
</Grid.Column>
<Grid.Column width={4}>
<VideoFormatSelectorForm form={form}/>
@ -730,17 +824,42 @@ function SuccessfulDownloadSubmitMessage() {
</Grid.Row>
}
export function ArchiveDownloadForm({download, onCancel}) {
const [showMessage, setShowMessage] = React.useState(false);
export function EditArchiveDownloadForm({
download,
onCancel,
onSuccess,
onDelete,
actions = EditDownloadFormButtons,
}) {
const submitter = async (formData) => {
const downloadData = {
downloader: formData.downloader,
tag_names: formData.tag_names,
urls: formData.urls.split(/\r?\n/),
}
await putDownload(download.id, downloadData);
}
return <ArchiveDownloadForm
download={download}
onCancel={onCancel}
onSuccess={onSuccess}
submitter={submitter}
actions={(props) => actions({...props, onDelete})}
/>
}
export function ArchiveDownloadForm({download, onCancel, onSuccess: propOnSuccess, submitter: propSubmitter, actions}) {
const [showMessage, setShowMessage] = React.useState(false);
const submitter = propSubmitter || (async (formData) => {
const downloadData = {
downloader: formData.downloader,
tag_names: formData.tag_names,
urls: formData.urls.split(/\r?\n/),
}
await postDownload(downloadData);
};
});
const emptyFormData = {
downloader: Downloaders.Archive,
@ -748,14 +867,28 @@ export function ArchiveDownloadForm({download, onCancel}) {
tag_names: [],
};
const onSuccess = () => {
setShowMessage(true);
if (propOnSuccess) {
propOnSuccess();
}
}
const form = useForm({
submitter,
defaultFormData: mergeDeep(emptyFormData, download),
emptyFormData,
clearOnSuccess: true,
onSuccess: async () => setShowMessage(true),
clearOnSuccess: !propOnSuccess,
onSuccess,
});
const localOnCancel = (e) => {
if (e) e.preventDefault();
if (onCancel) {
onCancel();
}
}
return <Form>
<Header as='h3'><Icon name='file text' color='green'/> Archives</Header>
<p>Create a Singlefile Archive for each of the URLs provided below.</p>
@ -774,7 +907,7 @@ export function ArchiveDownloadForm({download, onCancel}) {
{showMessage && <SuccessfulDownloadSubmitMessage/>}
<Grid.Row>
<Grid.Column textAlign='right'>
<DownloadFormButtons onCancel={onCancel} form={form}/>
{actions ? actions({onCancel: localOnCancel, form}) : <DownloadFormButtons onCancel={localOnCancel} form={form}/>}
</Grid.Column>
</Grid.Row>
</Grid>
@ -783,8 +916,10 @@ export function ArchiveDownloadForm({download, onCancel}) {
export function RSSDownloadForm({download, submitter, onDelete, onCancel, actions, clearOnSuccess = true}) {
const [showMessage, setShowMessage] = React.useState(false);
const [config, setConfig] = React.useState(null);
const [isConfigLoaded, setIsConfigLoaded] = React.useState(false);
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
submitter = submitter || (async (formData) => {
@ -800,6 +935,9 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
await postDownload(downloadData);
});
// Use config video resolutions if available, otherwise use default
const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions;
const emptyFormData = {
destination: null,
downloader: Downloaders.RSS,
@ -809,7 +947,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
excluded_urls: null,
title_exclude: null,
title_include: null,
video_resolutions: defaultVideoResolutions,
video_resolutions: configResolutions,
video_format: defaultVideoFormat,
},
tag_names: [],
@ -824,11 +962,30 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
onSuccess: async () => setShowMessage(true),
});
// Fetch video downloader config when the modal opens
React.useEffect(() => {
const {video_resolutions, video_format} = form.formData.settings;
if (video_resolutions && video_resolutions !== defaultVideoResolutions) {
setDefaultVideoResolutions(video_resolutions);
}
const fetchConfig = async () => {
try {
const result = await fetchVideoDownloaderConfig();
setConfig(result);
setIsConfigLoaded(true);
// Update form with video resolutions from config only if user hasn't changed them
// and if no download object was provided (this is a new download, not an edit)
if (result && result.video_resolutions && !userChangedResolutions && !download) {
form.setValue('settings.video_resolutions', result.video_resolutions);
}
} catch (error) {
console.error('Failed to fetch video downloader config:', error);
setIsConfigLoaded(true);
}
};
fetchConfig();
}, []);
React.useEffect(() => {
const {video_format} = form.formData.settings;
if (video_format && video_format !== defaultVideoFormat) {
setDefaultVideoFormat(video_format);
}
@ -859,7 +1016,10 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
</Grid.Row>
<Grid.Row columns={2}>
<Grid.Column mobile={16} computer={11}>
<VideoResolutionSelectorForm form={form}/>
<VideoResolutionSelectorForm
form={form}
onChange={() => setUserChangedResolutions(true)}
/>
</Grid.Column>
<Grid.Column mobile={8} computer={3}>
<VideoFormatSelectorForm form={form}/>

View file

@ -193,6 +193,11 @@ export function FilePreviewProvider({children}) {
// Navigated to a page which has a file preview active, fetch the information about the file, then display it.
if (previewQuery && !previewFile) {
initPreviewFile();
} else if (!previewQuery && (previewFile || previewModal)) {
// User navigated back/forward and preview query is gone, close the preview.
// Only close if there's actually a preview open (previewFile or previewModal exists).
setPreviewFile(null);
setPreviewModal(null);
}
}, [previewQuery]);

View file

@ -208,6 +208,10 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
// The results summarized.
const [suggestionsSums, setSuggestionsSums] = useState({});
// Disable selecting a result briefly after new suggestions arrive.
const [selectDisabled, setSelectDisabled] = React.useState(false);
const selectDisableTimerRef = React.useRef(null);
const noResults = [{title: 'No results'}];
const normalizeSuggestionsResults = (newSuggestions) => {
@ -303,17 +307,40 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
});
}
const suggestionSelectDelay = 250;
React.useEffect(() => {
setSuggestionsSums({});
setSuggestionsResults({});
if (!_.isEmpty(suggestions)) {
// Start a brief window where selecting results is disabled to prevent accidental clicks
// when the dropdown re-renders with new suggestions.
if (selectDisableTimerRef.current) {
clearTimeout(selectDisableTimerRef.current);
}
setSelectDisabled(true);
selectDisableTimerRef.current = setTimeout(() => setSelectDisabled(false), suggestionSelectDelay);
normalizeSuggestionsResults(suggestions);
}
}, [JSON.stringify(suggestions)]);
// Clear timer on unmount.
React.useEffect(() => {
return () => {
if (selectDisableTimerRef.current) {
clearTimeout(selectDisableTimerRef.current);
}
};
}, []);
// User clicked on a result in the dropdown.
const handleResultSelect = ({result}) => {
if (selectDisabled) {
console.debug('Selection temporarily disabled after suggestions update');
return;
}
if (result.location) {
console.info(`useSearchSuggestions Navigating: ${result.location}`)
navigate(result.location);

View file

@ -93,6 +93,7 @@ function VideosPage() {
{value: 'published_datetime', text: 'Published Date', short: 'P.Date'},
{value: 'length', text: 'Length'},
{value: 'size', text: 'Size'},
{value: 'size_to_duration', text: 'Size to Duration', short: 'S/Duration'},
{value: 'view_count', text: 'Views'},
{value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'},
{value: 'download_datetime', text: 'Download Date', short: 'D.Date'},

View file

@ -18,7 +18,7 @@ import {
WROLModeMessage
} from "../Common";
import {
Button as SButton,
Button as SButton, ButtonGroup,
Label,
Loader,
PlaceholderLine,
@ -35,7 +35,9 @@ import {
EditChannelDownloadForm,
EditRSSDownloadForm,
EditScrapeFilesDownloadForm,
EditZimDownloadForm
EditZimDownloadForm,
EditVideosDownloadForm,
EditArchiveDownloadForm
} from "../Download";
import {Downloaders} from "../Vars";
@ -246,6 +248,9 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
class OnceDownloadRow extends React.Component {
constructor(props) {
super(props);
this.state = {
editModalOpen: false
};
}
handleDelete = async () => {
@ -272,8 +277,21 @@ class OnceDownloadRow extends React.Component {
}
};
handleEditOpen = () => {
this.setState({ editModalOpen: true });
};
handleEditClose = () => {
this.setState({ editModalOpen: false });
};
handleEditSuccess = async () => {
await this.props.fetchDownloads();
this.handleEditClose();
};
render() {
let {url, last_successful_download, status, location, error} = this.props;
let {url, last_successful_download, status, location, error, downloader, settings, id} = this.props;
// Open downloads (/download), or external links in an anchor.
const link = location && !location.startsWith('/download') ?
@ -297,6 +315,8 @@ class OnceDownloadRow extends React.Component {
} else if (status === 'failed' || status === 'deferred') {
buttonCell = (
<TableCell>
<ButtonGroup>
<APIButton
color='red'
icon='trash'
@ -305,6 +325,13 @@ class OnceDownloadRow extends React.Component {
confirmButton='Delete'
obeyWROLMode={true}
/>
{(downloader === Downloaders.Video || downloader === Downloaders.Archive) && status !== 'pending' && (
<Button
icon='edit'
color='blue'
onClick={this.handleEditOpen}
/>
)}
<APIButton
color='green'
icon='redo'
@ -313,6 +340,7 @@ class OnceDownloadRow extends React.Component {
onClick={this.handleRestart}
obeyWROLMode={true}
/>
</ButtonGroup>
</TableCell>
);
} else if (status === 'complete' && location) {
@ -336,6 +364,55 @@ class OnceDownloadRow extends React.Component {
)
}
// Create edit modal for video or archive downloads
let editModal = null;
if (downloader === Downloaders.Video) {
editModal = (
<Modal
closeIcon
open={this.state.editModalOpen}
onClose={this.handleEditClose}
>
<ModalHeader>Edit Video Download</ModalHeader>
<ModalContent>
<EditVideosDownloadForm
download={{
urls: url,
settings: settings || {},
id: id
}}
onCancel={this.handleEditClose}
onSuccess={this.handleEditSuccess}
onDelete={this.handleDelete}
/>
</ModalContent>
</Modal>
);
} else if (downloader === Downloaders.Archive) {
editModal = (
<Modal
closeIcon
open={this.state.editModalOpen}
onClose={this.handleEditClose}
>
<ModalHeader>Edit Archive Download</ModalHeader>
<ModalContent>
<EditArchiveDownloadForm
download={{
urls: url,
tag_names: settings?.tag_names || [],
id: id
}}
onCancel={this.handleEditClose}
onSuccess={this.handleEditSuccess}
onDelete={this.handleDelete}
/>
</ModalContent>
</Modal>
);
}
return <TableRow>
<TableCell className='column-ellipsis'>
{link(url)}
@ -345,6 +422,7 @@ class OnceDownloadRow extends React.Component {
{status === 'pending' ? <Loader active inline size='tiny'/> : null}
</TableCell>
{buttonCell}
{editModal}
</TableRow>
}
}

View file

@ -179,7 +179,7 @@ export function useForm({
patchFormData(_.set(formData, path, newValue));
}
const getCustomProps = ({name, validator, type = 'text', path, required = false}) => {
const getCustomProps = ({name, validator, type = 'text', path, required = false, afterChange}) => {
// Props for any other elements.
path = path || name;
@ -202,7 +202,12 @@ export function useForm({
type,
disabled,
value,
onChange: (newValue) => {setValue(path, newValue)},
onChange: (newValue) => {
setValue(path, newValue);
if (afterChange) {
afterChange(newValue);
}
},
'data-path': path,
}
// Attributes about the input, but should not be passed as properties.
@ -214,7 +219,15 @@ export function useForm({
return [inputProps, inputAttrs]
}
const getInputProps = ({name, validator, path, required = false, type = 'text', onChange = null}) => {
const getInputProps = ({
name,
validator,
path,
required = false,
type = 'text',
onChange = null,
afterChange = null
}) => {
// Props for <input/>
path = path || name;
@ -223,7 +236,7 @@ export function useForm({
validator = validURL;
}
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required});
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required, afterChange});
const localHandleInputEvent = async (e) => {
if (e) e.preventDefault();
@ -244,12 +257,15 @@ export function useForm({
return [inputProps, inputAttrs]
}
const getSelectionProps = ({name, validator, path, type, required = false}) => {
const getSelectionProps = ({name, validator, path, type, required = false, afterChange = null}) => {
// Props for Dropdowns.
path = path || name;
const [customProps, inputAttrs] = getCustomProps({name, validator, type, path, required});
const [customProps, inputAttrs] = getCustomProps({name, validator, type, path, required, afterChange});
customProps.onChange = (e, {value}) => {
patchFormData(_.set(formData, path, value))
patchFormData(_.set(formData, path, value));
if (afterChange) {
afterChange(value);
}
};
return [customProps, inputAttrs]
}
@ -388,50 +404,49 @@ export function UrlInput({form, required = true, name = 'url', path = 'url', dis
export function UrlsTextarea({name = 'urls', required, form}) {
required = required !== undefined;
const validator = (value) => {
if (!validURLs(value)) {
return 'Invalid URLs';
}
}
// Memoize input props to prevent recomputation during render
const [inputProps, inputAttrs] = React.useMemo(() => {
const [inputProps, inputAttrs] = form.getInputProps({name, validator, required});
const validator = (value) => {
if (!validURLs(value)) {
return 'Invalid URLs';
}
};
return form.getInputProps({name, validator, required});
}, [form, name, required]);
const handleDrop = (e) => {
if (e) e.preventDefault();
// Handle user dropping multiple URLs into the textarea. Assume each drop is a URL. Ensure each URL
// is on its own line.
const droppedUrl = e.dataTransfer.getData('text');
let urls = (inputProps.value || '').split('\n');
urls = [...urls, droppedUrl];
urls = urls.filter(i => !!i).join('\n');
// Leave trailing newline for user to type.
inputAttrs.setValue(`${urls}\n`)
inputAttrs.setValue(`${urls}\n`);
};
const handleKeyDown = (event) => {
// Check if Ctrl (or Cmd for Mac) + Enter are pressed
if ((event.ctrlKey || event.metaKey) && event.key === 'Enter') {
event.preventDefault(); // Prevents the addition of a new line in textarea if necessary
form.onSubmit(); // Call your submit handler
event.preventDefault();
form.onSubmit();
}
};
return <FormInput required
error={inputProps.error}
label='URLs'
>
<TextArea
id='urls_textarea'
placeholder={'Enter one URL per line'}
name='urls'
onDrop={handleDrop}
onKeyDown={handleKeyDown}
{...inputProps}
/>
</FormInput>
return (
<FormInput required error={inputProps.error} label="URLs">
<TextArea
id="urls_textarea"
placeholder="Enter one URL per line"
name="urls"
onDrop={handleDrop}
onKeyDown={handleKeyDown}
{...inputProps}
/>
</FormInput>
);
}
export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled=false}) {
export function ToggleForm({form, name, label, path, icon = null, iconSize = 'big', disabled = false}) {
const [inputProps, inputAttrs] = form.getCustomProps({name, path})
const iconElm = icon ? <Icon size={iconSize} name={icon}/> : null;

View file

@ -60,11 +60,18 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi-*.service /etc/systemd/system/
cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf
cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd
# Copy MOTD scripts, delete original.
cp /opt/wrolpi/etc/raspberrypios/motd/20-wrolpi.motd /etc/update-motd.d/20-wrolpi
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi-first.motd /etc/update-motd.d/30-wrolpi
chmod +x /etc/update-motd.d/*
truncate -s 0 /etc/motd
# NetworkManager for the hotspot.
systemctl enable NetworkManager
# Enable iperf3 for speed testing.
systemctl enable iperf3
set +x
echo

View file

@ -32,6 +32,7 @@ gpsd
hostapd
htop
iotop
iperf3
kiwix
kiwix-tools
libagg-dev
@ -101,4 +102,4 @@ vim
vlc
wget
wpasupplicant
zim-tools
zim-tools

View file

@ -86,6 +86,8 @@ services:
map_https: # HTTPS for map tile server
depends_on:
- map
environment:
- UPSTREAM_HOST=map
build:
context: .
dockerfile: docker/https_proxy/Dockerfile
@ -130,6 +132,8 @@ services:
zim_https: # HTTPS for Zim service.
depends_on:
- zim
environment:
- UPSTREAM_HOST=zim
build:
context: .
dockerfile: docker/https_proxy/Dockerfile
@ -154,6 +158,8 @@ services:
help_https: # HTTPS for Help service.
depends_on:
- help
environment:
- UPSTREAM_HOST=help
build:
context: .
dockerfile: docker/https_proxy/Dockerfile

View file

@ -1,11 +1,9 @@
FROM python:3.11-buster
FROM python:3.11-bookworm
ENV DOCKER=true
WORKDIR /opt/wrolpi
# Update dependencies for the services install.
RUN apt update
# ffmpeg for Videos
RUN apt-get install -y ffmpeg catdoc sysstat aria2
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping
RUN ffmpeg -version
# yt-dlp needs cache directory
RUN mkdir /.cache && chown -R 1000:1000 /.cache

View file

@ -1,5 +1,5 @@
# This Dockerfile is used as a reverse proxy to serve a HTTPS nieve service with HTTPS.
FROM nginx:latest as https_proxy
FROM nginx:latest AS https_proxy
# Create a self-signed cert and key pair for HTTPS.
RUN openssl genrsa -out /cert.key 2048
@ -7,6 +7,10 @@ RUN openssl req -new -x509 -nodes -key /cert.key -out /cert.crt -days 3650 \
-subj "/C=US/ST=State/L=City/O=Org/OU=WROLPi/CN=wrolpi.local"
RUN chmod 777 /cert.key /cert.crt
# Copy Nginx configuration
COPY docker/https_proxy/nginx.proxy.conf /etc/nginx/nginx.conf
# Apply docker-compose environment variables to config
ENV UPSTREAM_HOST=you-must-change-this-in-docker-compose
ENV UPSTREAM_PORT=80
COPY docker/https_proxy/nginx.proxy.conf.template /etc/nginx/nginx.conf
RUN envsubst < /etc/nginx/nginx.conf > /etc/nginx/nginx.conf
CMD ["nginx", "-g", "daemon off;"]

View file

@ -8,7 +8,7 @@ http {
ssl_certificate_key /cert.key;
location / {
proxy_pass http://localhost:80;
proxy_pass http://${UPSTREAM_HOST}:${UPSTREAM_PORT};
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

View file

@ -1,21 +0,0 @@
The programs included with the Debian GNU/Linux system are free software;
the exact distribution terms for each program are described in the
individual files in /usr/share/doc/*/copyright.
Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
permitted by applicable law.
██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗
██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║
██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██
██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║
╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║
╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝
Welcome to your WROLPi!
Run the help script to find potential problems:
/opt/wrolpi/help.sh
Help documentation is available: https://wrolpi.local:8086 or /opt/wrolpi-help/docs

View file

@ -0,0 +1,16 @@
#!/bin/sh
echo ""
echo "The programs included with the Debian GNU/Linux system are free software;"
echo "the exact distribution terms for each program are described in the"
echo "individual files in /usr/share/doc/*/copyright."
echo ""
echo "Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent"
echo "permitted by applicable law."
echo ""
echo "██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗"
echo "██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║"
echo "██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██"
echo "██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║"
echo "╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║"
echo " ╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝"
echo ""

View file

@ -0,0 +1,17 @@
#!/bin/sh
ip_address=$(ip addr show eth0 | grep -oP 'inet \K[\d.]+')
hostname="$(hostname).local"
echo "Welcome to your new WROLPi! Follow the steps below to finish the setup:"
echo ""
echo " 1. If you have an external drive, mount it to /media/wrolpi"
echo " 2. Run the repair script: /opt/wrolpi/repair.sh"
echo " 3. Refresh your files in the UI at https://${hostname} or https://127.0.0.1"
echo ""
echo "Run the help script to find potential problems:"
echo " /opt/wrolpi/help.sh"
echo ""
echo "Help documentation is available: https://${ip_address}:8086"
echo " or in this directory /opt/wrolpi-help/docs"
echo ""

View file

@ -0,0 +1,12 @@
#!/bin/sh
ip_address=$(ip addr show eth0 | grep -oP 'inet \K[\d.]+')
echo " Welcome to your WROLPi!"
echo ""
echo " Run the help script to find potential problems:"
echo " /opt/wrolpi/help.sh"
echo ""
echo " Help documentation is available: https://${ip_address}:8086"
echo " or in this directory /opt/wrolpi-help/docs"
echo ""

View file

@ -1,25 +0,0 @@
The programs included with the Debian GNU/Linux system are free software;
the exact distribution terms for each program are described in the
individual files in /usr/share/doc/*/copyright.
Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
permitted by applicable law.
██╗ ██╗██████╗ ██████╗ ██╗ ██████╗ ██╗
██║ ██║██╔══██╗██╔═══██╗██║ ██╔══██╗ ║
██║ █╗ ██║██████╔╝██║ ██║██║ ██████╔╝██
██║███╗██║██╔══██╗██║ ██║██║ ██╔═══╝ ██║
╚███╔███╔╝██║ ██║╚██████╔╝███████╗██║ ██║
╚══╝╚══╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝
Welcome to your new WROLPi! Follow the steps below to finish the setup:
1. If you have an external drive, mount it to /media/wrolpi
2. Run the repair script: /opt/wrolpi/repair.sh
3. Refresh your files in the UI at https://wrolpi.local or https://127.0.0.1
Run the help script to find potential problems:
/opt/wrolpi/help.sh
Help documentation is available: https://wrolpi.local:8086 or /opt/wrolpi-help/docs

43
main.py
View file

@ -224,24 +224,31 @@ async def start_single_tasks(app: Sanic):
with suppress(Exception):
get_wrolpi_config().import_config()
logger.debug('wrolpi config imported')
with suppress(Exception):
tags.import_tags_config()
logger.debug('tags config imported')
with suppress(Exception):
get_videos_downloader_config().import_config()
logger.debug('videos downloader config imported')
with suppress(Exception):
await import_downloads_config()
logger.debug('downloads config imported')
# Channels uses both downloads and tags.
with suppress(Exception):
import_channels_config()
logger.debug('channels config imported')
with suppress(Exception):
import_inventories_config()
logger.debug('inventories config imported')
with suppress(Exception):
init_inventory()
wrolpi_config = get_wrolpi_config()
if wrolpi_config.successful_import:
# Only import other configs if WROLPi config was imported, and WROL mode is not enabled.
if wrolpi_config.wrol_mode:
logger.warning('Refusing to import other configs when WROL mode is enabled!')
else:
with suppress(Exception):
tags.import_tags_config()
logger.debug('tags config imported')
with suppress(Exception):
get_videos_downloader_config().import_config()
logger.debug('videos downloader config imported')
with suppress(Exception):
await import_downloads_config()
logger.debug('downloads config imported')
# Channels uses both downloads and tags.
with suppress(Exception):
import_channels_config()
logger.debug('channels config imported')
with suppress(Exception):
import_inventories_config()
logger.debug('inventories config imported')
with suppress(Exception):
init_inventory()
from modules.zim.lib import flag_outdated_zim_files
try:

View file

@ -107,13 +107,32 @@ archive_downloader = ArchiveDownloader()
def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
"""
Models an Archive from a given FileGroup.
This function takes in a FileGroup and attempts to create an Archive object.
It does this by checking for the presence of HTML files within the FileGroup,
determining if any of these are SingleFiles, and then extracting relevant data
(title, contents) from either the JSON or text readability files.
Args:
file_group: The FileGroup to model as an Archive.
session: An optional database session for committing changes.
Returns:
An Archive object representing the modeled archive.
Raises:
InvalidArchive: If no HTML SingleFile is found in the FileGroup, or if
any other error occurs during modeling.
"""
file_group_id = file_group.id
if not file_group_id:
session.flush([file_group])
file_group_id = file_group.id
# All Archives have an HTML Singlefile.
html_paths = file_group.my_paths('text/html')
# All Archives have an HTML Singlefile. Singlefile may be text HTML, or bytes HTML.
html_paths = file_group.my_html_paths()
if not html_paths:
logger.error('Query returned a group without an HTML file!')
raise InvalidArchive('FileGroup does not contain any html files')
@ -162,8 +181,12 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
if readability_txt_path:
contents = get_article(readability_txt_path)
archive = Archive(file_group_id=file_group_id, file_group=file_group)
session.add(archive)
# Check if an Archive already exists for this FileGroup
archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none()
if not archive:
# Create new Archive if it doesn't exist
archive = Archive(file_group_id=file_group_id, file_group=file_group)
session.add(archive)
archive.validate()
archive.flush()
@ -177,6 +200,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
'readability_txt_path': archive.readability_txt_path,
'screenshot_path': archive.screenshot_path,
'singlefile_path': archive.singlefile_path,
'info_json_path': archive.info_json_path,
}
return archive

View file

@ -1,11 +1,14 @@
import multiprocessing
import queue
from http import HTTPStatus
from sanic import response, Request, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from sqlalchemy.orm import Session
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, api_param_limiter
from wrolpi.api_utils import json_response, api_app
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
@ -84,20 +87,50 @@ async def get_upload_singlefile(request: Request):
@register_switch_handler('singlefile_upload_switch_handler')
async def singlefile_upload_switch_handler(singlefile):
async def singlefile_upload_switch_handler(url=None):
"""Used by `post_upload_singlefile` to upload a single file"""
from wrolpi.downloader import download_manager, Download
from wrolpi.downloader import Download
from . import ArchiveDownloader
archive = await lib.singlefile_to_archive(singlefile)
logger.info(f'Created Archive from upload: {archive}')
q: multiprocessing.Queue = api_app.shared_ctx.archive_singlefiles
trace_enabled = logger.isEnabledFor(TRACE_LEVEL)
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called for {url}')
try:
singlefile = q.get_nowait()
except queue.Empty:
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
return
try:
q_size = q.qsize()
except NotImplementedError:
# qsize() is not implemented on macOS
q_size = '?'
logger.info(f'singlefile_upload_switch_handler queue size: {q_size}')
try:
archive = await lib.singlefile_to_archive(singlefile)
except Exception as e:
logger.error(f'singlefile_upload_switch_handler failed', exc_info=e)
Events.send_upload_archive_failed(f'Failed to convert singlefile to archive: {e}')
raise
name = archive.file_group.title or archive.file_group.url
logger.info(f'Created Archive from upload ({q_size}): {archive}')
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
url = archive.file_group.url
if url and (download := Download.get_by_url(url)):
if (download.is_failed or download.is_deferred) and download.downloader == ArchiveDownloader.name:
# Download was attempted and failed, user manually archived the URL.
download_manager.delete_download(download.id)
download.complete()
download.location = archive.location
Session.object_session(download).commit()
# Call this function again so any new singlefiles can be archived.
singlefile_upload_switch_handler.activate_switch()
singlefile_upload_switch_handler: ActivateSwitchMethod
@ -112,10 +145,11 @@ async def post_upload_singlefile(request: Request):
singlefile = request.files['singlefile_contents'][0].body
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
# Get the URL to ensure that the singlefile is valid.
# Extract the URL to ensure that the singlefile is valid.
lib.get_url_from_singlefile(singlefile)
# Send processing to background task so the extension can continue.
singlefile_upload_switch_handler.activate_switch(dict(singlefile=singlefile))
api_app.shared_ctx.archive_singlefiles.put(singlefile)
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)

View file

@ -19,7 +19,7 @@ def archive_directory(test_directory) -> pathlib.Path:
@pytest.fixture
def archive_factory(test_session, archive_directory, make_files_structure):
def archive_factory(test_session, archive_directory, make_files_structure, image_bytes_factory):
def time_generator():
timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0).astimezone(pytz.UTC)
while True:
@ -29,7 +29,7 @@ def archive_factory(test_session, archive_directory, make_files_structure):
now = time_generator()
def _(domain: str = None, url: str = None, title: str = 'NA', contents: str = None,
singlefile_contents: str = None, tag_names: List[str] = None) -> Archive:
singlefile_contents: str = None, tag_names: List[str] = None, screenshot: bool = True) -> Archive:
if domain:
assert '/' not in domain
@ -49,10 +49,9 @@ def archive_factory(test_session, archive_directory, make_files_structure):
if url:
json_contents['url'] = url
json_contents = json.dumps(json_contents)
singlefile_path, screenshot_path, readability_path, readability_txt_path, readability_json_path = \
singlefile_path, readability_path, readability_txt_path, readability_json_path = \
make_files_structure({
str(domain_dir / f'{timestamp}_{title}.html'): singlefile_contents or '<html></html>',
str(domain_dir / f'{timestamp}_{title}.png'): None,
str(domain_dir / f'{timestamp}_{title}.readability.html'): '<html></html>',
str(domain_dir / f'{timestamp}_{title}.readability.txt'): contents,
str(domain_dir / f'{timestamp}_{title}.readability.json'): json_contents,
@ -61,6 +60,11 @@ def archive_factory(test_session, archive_directory, make_files_structure):
if domain:
domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
screenshot_path = None
if screenshot:
screenshot_path = domain_dir / f'{timestamp}_{title}.png'
screenshot_path.write_bytes(image_bytes_factory())
if not domain and domain_dir.name != 'NA':
domain = Domain(
domain=domain_dir.name,

View file

@ -319,7 +319,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True
except json.decoder.JSONDecodeError:
# Was not valid JSON.
schema = None
if isinstance(schema, dict) and '://schema.org' in schema.get('@context'):
if isinstance(schema, dict) and (context := schema.get('@context')) and '://schema.org' in context:
# Found https://schema.org/
if headline := schema.get('headline'):
metadata.title = metadata.title or headline
@ -429,21 +429,22 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
def get_domains():
with get_db_curs() as curs:
stmt = '''
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
FROM domains
LEFT JOIN archive a on domains.id = a.domain_id
LEFT JOIN file_group fg on fg.id = a.file_group_id
GROUP BY domains.domain
ORDER BY domains.domain
'''
SELECT domains.domain AS domain, COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size
FROM domains
LEFT JOIN archive a on domains.id = a.domain_id
LEFT JOIN file_group fg on fg.id = a.file_group_id
GROUP BY domains.domain
ORDER BY domains.domain \
'''
curs.execute(stmt)
domains = [dict(i) for i in curs.fetchall()]
return domains
ARCHIVE_ORDERS = {
'published_datetime': (date := 'COALESCE(fg.published_datetime, fg.download_datetime) ASC NULLS LAST'),
'-published_datetime': (_date := 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST'),
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
'published_datetime': (date := 'fg.effective_datetime ASC NULLS LAST'),
'-published_datetime': (_date := 'fg.effective_datetime DESC NULLS LAST'),
'published_modified_datetime': f'fg.published_modified_datetime ASC NULLS LAST, {date}',
'-published_modified_datetime': f'fg.published_modified_datetime DESC NULLS LAST, {_date}',
'download_datetime': 'fg.download_datetime ASC NULLS LAST',
@ -456,8 +457,8 @@ ARCHIVE_ORDERS = {
'-viewed': 'fg.viewed DESC',
}
ORDER_GROUP_BYS = {
'published_datetime': 'fg.published_datetime, fg.download_datetime',
'-published_datetime': 'fg.published_datetime, fg.download_datetime',
'published_datetime': 'fg.effective_datetime',
'-published_datetime': 'fg.effective_datetime',
'published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
'-published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime',
'download_datetime': 'fg.download_datetime',
@ -482,7 +483,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
-> Tuple[List[dict], int]:
# Always filter FileGroups to Archives.
wheres = []
group_by = 'fg.published_datetime, fg.download_datetime, a.file_group_id'
group_by = 'fg.effective_datetime, a.file_group_id'
params = dict(search_str=search_str, offset=int(offset), limit=int(limit))
order_by = ARCHIVE_ORDERS['-published_datetime']

View file

@ -141,8 +141,7 @@ class Archive(Base, ModelHelper):
@property
def singlefile_file(self) -> Optional[dict]:
from modules.archive.lib import is_singlefile_file
files = self.file_group.my_files('text/html')
for file in files:
for file in self.file_group.my_html_files():
if is_singlefile_file(file['path']):
return file
@ -198,6 +197,18 @@ class Archive(Base, ModelHelper):
if screenshot_file := self.screenshot_file:
return screenshot_file['path']
@property
def info_json_file(self) -> Optional[dict]:
files = self.file_group.my_files()
for file in files:
if file['path'].name.endswith('.info.json'):
return file
@property
def info_json_path(self) -> Optional[pathlib.Path]:
if info_json_file := self.info_json_file:
return info_json_file['path']
def apply_readability_data(self):
"""Read the Readability JSON file, apply its contents to this record."""
readability_json_path = self.readability_json_path

View file

@ -4,8 +4,11 @@ from http import HTTPStatus
import pytest
from modules.archive import lib, Archive
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci, skip_macos
from wrolpi.common import get_relative_to_media_directory
def check_results(test_client, data, ids):
request, response = test_client.post('/api/archive/search', content=json.dumps(data))
@ -202,7 +205,7 @@ def test_archive_and_domain_crud(test_session, test_client, archive_factory):
request, response = test_client.get(f'/api/archive/domains')
assert response.status_code == HTTPStatus.OK
assert response.json['domains'][0]['domain'] == 'example.com'
assert response.json['domains'][0]['size'] == 116
assert response.json['domains'][0]['size'] == 254
# Deleting works.
request, response = test_client.delete(f'/api/archive/{archive1.id}')
@ -257,3 +260,107 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events = list(events_history)
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
@pytest.mark.asyncio
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
await_switches, make_multipart_form, image_bytes_factory):
"""Test that uploading an info.json and image file via /api/files/upload properly tracks them in Archive and FileGroup."""
archive = archive_factory('example.com', 'https://example.com/test-url', 'tracking test', screenshot=False)
test_session.commit()
# Verify Archive and FileGroup are created, but screenshot does not exist.
file_group = archive.file_group
assert not archive.screenshot_path, 'Screenshot should not exist yet.'
assert file_group.files is not None, 'FileGroup should have files array'
initial_file_count = len(file_group.files)
# Upload info.json via /api/files/upload
# Extract the prefix from singlefile path (e.g., "2000-01-01-00-00-01_tracking test")
singlefile_stem = archive.singlefile_path.stem # e.g., "2000-01-01-00-00-01_tracking test"
# Create info.json content
info_json_data = {
'title': 'test',
'url': 'https://example.com/test-url',
'custom_field': 'value'
}
info_json_bytes = json.dumps(info_json_data).encode()
# Get relative destination directory
destination = str(get_relative_to_media_directory(archive.singlefile_path.parent))
filename = f'{singlefile_stem}.info.json'
# POST to /api/files/upload
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(info_json_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=info_json_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Step 4: Verify info.json is tracked
# Refresh the session to get updated data
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert info.json properties exist
assert archive.info_json_path is not None, 'Archive.info_json_path should exist'
assert archive.info_json_file is not None, 'Archive.info_json_file should exist'
# Assert FileGroup has the correct number of files.
assert len(archive.file_group.files) == initial_file_count + 1, \
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
assert archive.file_group.data.get('info_json_path'), \
'Archive readability info json should be in FileGroup.data'
# Upload an image file and verify it's added to the FileGroup.
# Store file count before adding image
file_count_before_image = len(archive.file_group.files)
# Create image using factory
image_data = image_bytes_factory()
# Upload image with same stem as the Archive (using .jpg to avoid conflict with factory-created .png)
image_filename = f'{singlefile_stem}.jpg'
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=image_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(image_data))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=image_data, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Image upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Assert screenshot properties exist
assert archive.screenshot_path is not None, 'Archive.screenshot_path should exist'
assert archive.screenshot_file is not None, 'Archive.screenshot_file should exist'
assert archive.file_group.data.get('screenshot_path') == archive.screenshot_path, 'Archive screenshot should be in FileGroup.data'
# Assert image is in FileGroup.files
file_paths = [f['path'] for f in archive.file_group.files]
assert archive.screenshot_path in file_paths, 'Screenshot should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(archive.file_group.files) == file_count_before_image + 1, \
f'FileGroup should have {file_count_before_image + 1} files after adding image'

View file

@ -276,6 +276,7 @@ def test_get_domains(test_session, archive_factory):
assert [i['domain'] for i in get_domains()] == []
@skip_circleci
@pytest.mark.asyncio
async def test_new_archive(test_session, test_directory, fake_now):
singlefile, readability, screenshot = make_fake_archive_result()

View file

@ -29,52 +29,24 @@ async def get_minimal_channels() -> List[dict]:
# Get all channels, even if they don't have videos. Also get the minimum frequency download because this is the
# one that will consume the most resources.
stmt = '''
SELECT
c.id, c.name AS "name", directory, c.url, t.name AS "tag_name",
COUNT(v.id) as video_count,
SUM(fg.size)::BIGINT AS size,
(
select min(d.frequency)
from download d
where d.channel_id = c.id
) AS minimum_frequency
FROM
channel AS c
LEFT JOIN video v on c.id = v.channel_id
LEFT JOIN file_group fg on fg.id = v.file_group_id
LEFT JOIN tag t ON t.id = c.tag_id
GROUP BY 1, 2, 3, 4, 5
'''
SELECT c.id,
c.name AS "name",
c.directory,
c.url,
t.name AS "tag_name",
c.video_count,
c.total_size,
c.minimum_frequency
FROM channel AS c
LEFT JOIN tag t ON t.id = c.tag_id
'''
curs.execute(stmt)
logger.debug(stmt)
channels = sorted([dict(i) for i in curs.fetchall()], key=lambda i: i['name'].lower())
return channels
async def get_channels_video_count() -> Dict[int, int]:
"""
Add video counts to all channels
"""
with get_db_curs() as curs:
stmt = 'SELECT id FROM channel'
curs.execute(stmt)
# Get all channel IDs, start them with a count of 0.
video_counts = {int(i['id']): 0 for i in curs.fetchall()}
stmt = '''
SELECT
c.id, COUNT(v.id) AS video_count
FROM
channel AS c
LEFT JOIN video AS v ON v.channel_id = c.id
GROUP BY 1
'''
curs.execute(stmt)
# Replace all the counts of those channels with videos.
video_counts.update({int(i['id']): int(i['video_count']) for i in curs.fetchall()})
return video_counts
COD = Union[dict, Channel]

View file

@ -202,7 +202,7 @@ async def test_channel_download_requires_refresh(
assert channel.refreshed == expected
assert bool(channel.info_json) == expected
await async_client.sanic_app.dispatch('wrolpi.download.')
await async_client.sanic_app.dispatch('wrolpi.download.download')
assert_refreshed(False)
test_session.commit()
@ -373,16 +373,14 @@ def test_search_videos_channel(test_client, test_session, video_factory):
@pytest.mark.asyncio
async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search):
async def test_get_channel_videos_pagination(test_session, video_factory, assert_video_search, channel_factory):
# Create two channels, one with 50 videos...
channel1, channel2 = channel_factory(), channel_factory()
for i in range(50):
video_factory(channel_id=simple_channel.id)
channel2 = Channel(name='Bar')
test_session.add(channel2)
test_session.flush()
test_session.refresh(channel2)
video_factory(channel_id=channel1.id)
# The other channel only has one video.
video_factory(channel_id=channel2.id)
video_factory(title='vid2', channel_id=channel2.id, with_video_file=True)
video_factory(title='vid2', channel_id=channel2.id)
test_session.commit()
# Get first, second, third, and empty pages of videos.
@ -395,10 +393,11 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
]
last_ids = []
for offset, video_count in tests:
_, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime',
_, response = await assert_video_search(channel_id=channel1.id, order_by='published_datetime',
offset=offset,
limit=20)
assert len(response.json['file_groups']) == video_count, 'Returned videos does not match'
assert len(response.json['file_groups']) == video_count, \
f'Returned videos does not match for ({offset}, {video_count})'
current_ids = [i['id'] for i in response.json['file_groups']]
assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}'
last_ids = current_ids

View file

@ -5,6 +5,7 @@ import logging
import os.path
import pathlib
import re
import sys
import traceback
from abc import ABC
from datetime import timedelta
@ -41,30 +42,47 @@ from .video.lib import download_video_info_json
logger = logger.getChild(__name__)
VIDEO_RESOLUTION_MAP = {
'360p': ['134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
'hls-175+bestaudio', # 360p avc1 + best audio
],
'480p': ['135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480',
'hls-312+bestaudio', # 480p avc1 + best audio
],
'360p': [
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
'hls-175+bestaudio', # 360p avc1 + best audio
'396+bestaudio', '243+bestaudio', # Vertical
'hls-655+bestaudio', 'hls-655',
'bestvideo[height=360]+bestaudio/best[height=360]',
],
'480p': [
'135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480',
'hls-312+bestaudio', # 480p avc1 + best audio
'788+bestaudio', '780+bestaudio', '397+bestaudio', # Vertical
'bestvideo[height=480]+bestaudio/best[height=480]',
],
'720p': [
'298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio
'136+bestaudio', '22', '311+bestaudio', 'res:720',
'298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio
'hls-517+bestaudio', 'hls-637+bestaudio', # 720p avc1 + best audio
'398+bestaudio', '247+bestaudio', # Vertical
'hls-2890+bestaudio', 'hls-2890',
'bestvideo[height=720]+bestaudio/best[height=720]',
],
'1080p': [
'299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio
'137+bestaudio', '614+bestaudio', 'res:1080',
'299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio
'hls-899+bestaudio', 'hls-1271+bestaudio', # 1080p avc1 + best audio
'399+bestaudio', '616+bestaudio', # Vertical
'hls-4026+bestaudio', 'hls-4026',
'bestvideo[height=1080]+bestaudio/best[height=1080]',
],
'1440p': [
'639+bestaudio', # vp9, best audio, hdr
'623+bestaudio', # vp9, best audio, no hdr
'620+bestaudio', 'res:1440'],
'620+bestaudio', 'res:1440',
'bestvideo[height=1440]+bestaudio/best[height=1440]',
],
'2160p': [
'642+bestaudio', # vp9, best audio, hdr
'628+bestaudio', # vp9, best audio, no hdr
'625+bestaudio', 'res:2160'],
'625+bestaudio', 'res:2160',
'bestvideo[height=2160]+bestaudio/best[height=2160]',
],
'maximum': ['bestvideo*+bestaudio/best'],
}
@ -272,9 +290,9 @@ class ChannelDownloader(Downloader, ABC):
# Filter videos by their length.
try:
if minimum_duration := settings.get('minimum_duration'):
downloads = [i for i in downloads if int(i['duration']) >= minimum_duration]
downloads = [i for i in downloads if (d := i.get('duration')) and int(d) >= minimum_duration]
if maximum_duration := settings.get('maximum_duration'):
downloads = [i for i in downloads if int(i['duration']) <= maximum_duration]
downloads = [i for i in downloads if (d := i.get('duration')) and int(d) <= maximum_duration]
except KeyError as e:
raise RuntimeError('Unable to filter videos because there is no duration') from e

View file

@ -100,8 +100,12 @@ class Video(ModelHelper, Base):
@staticmethod
def do_model(file_group: FileGroup, session: Session) -> 'Video':
video = Video(file_group_id=file_group.id, file_group=file_group)
session.add(video)
# Check if a Video already exists for this FileGroup
video = session.query(Video).filter_by(file_group_id=file_group.id).one_or_none()
if not video:
# Create new Video if it doesn't exist
video = Video(file_group_id=file_group.id, file_group=file_group)
session.add(video)
video.validate()
file_group.indexed = True
video.flush()
@ -152,25 +156,20 @@ class Video(ModelHelper, Base):
if self.file_group.published_datetime:
# Get videos next to this Video's upload date.
stmt = '''
WITH numbered_videos AS (
SELECT fg.id AS fg_id, v.id AS v_id,
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
FROM file_group fg
LEFT OUTER JOIN video v on fg.id = v.file_group_id
WHERE
v.channel_id = %(channel_id)s
AND fg.published_datetime IS NOT NULL
)
SELECT v_id
FROM numbered_videos
WHERE row_number IN (
SELECT row_number+i
FROM numbered_videos
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
WHERE
fg_id = %(fg_id)s
)
'''
WITH numbered_videos AS (SELECT fg.id AS fg_id,
v.id AS v_id,
ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number
FROM file_group fg
LEFT OUTER JOIN video v on fg.id = v.file_group_id
WHERE v.channel_id = %(channel_id)s
AND fg.published_datetime IS NOT NULL)
SELECT v_id
FROM numbered_videos
WHERE row_number IN (SELECT row_number + i
FROM numbered_videos
CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n
WHERE fg_id = %(fg_id)s) \
'''
else:
# No videos near this Video with upload dates, recommend the files next to this Video.
# Only recommend videos in the same Channel (or similarly without a Channel).
@ -244,6 +243,15 @@ class Video(ModelHelper, Base):
self.channel = channel
logger.debug(f'{self} has Channel {channel}')
# Set FileGroup.data with cached file paths for consistency with Archive
self.file_group.data = {
'id': self.id,
'video_path': self.video_path,
'info_json_path': self.info_json_path,
'poster_path': self.poster_path,
'caption_paths': self.caption_paths,
}
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
file_group = FileGroup.from_paths(session, *paths)
@ -532,6 +540,11 @@ class Channel(ModelHelper, Base):
source_id = Column(String) # the ID from the source website.
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
# Columns updated by triggers
video_count = Column(Integer, default=0) # update_channel_video_count
total_size = Column(Integer, default=0) # update_channel_size
minimum_frequency = Column(Integer) # update_channel_minimum_frequency
info_json = deferred(Column(JSON))
info_date = Column(Date)
@ -680,18 +693,17 @@ class Channel(ModelHelper, Base):
"""Get statistics about this channel."""
with get_db_curs() as curs:
stmt = '''
SELECT
SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s
'''
SELECT SUM(size) AS "size",
MAX(size) AS "largest_video",
COUNT(video.id) AS "video_count",
SUM(fg.length) AS "length",
-- Videos may use multiple tags.
COUNT(video.id) FILTER ( WHERE tf.file_group_id IS NOT NULL ) AS "video_tags"
FROM video
LEFT JOIN file_group fg on fg.id = video.file_group_id
LEFT JOIN public.tag_file tf on fg.id = tf.file_group_id
WHERE channel_id = %(id)s \
'''
curs.execute(stmt, dict(id=self.id))
return dict(curs.fetchone())

View file

@ -308,6 +308,43 @@ async def test_format_videos_description(async_client, test_session, test_direct
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'
@pytest.mark.asyncio
async def test_size_to_duration_sort(test_session, video_factory, assert_video_search):
"""Test the 'size_to_duration' sort option."""
# Create videos with different size-to-duration ratios
# All videos will have the same size (from the test file) but different durations
# Video with 10 second duration (highest ratio)
vid1 = video_factory(with_video_file=True, title='vid1', with_info_json={'duration': 10})
# Video with 20 second duration (medium ratio)
vid2 = video_factory(with_video_file=True, title='vid2', with_info_json={'duration': 20})
# Video with 30 second duration (lowest ratio)
vid3 = video_factory(with_video_file=True, title='vid3', with_info_json={'duration': 30})
test_session.commit()
# Verify that the videos have the expected size and length values
vid1 = test_session.query(Video).filter_by(id=vid1.id).one()
vid2 = test_session.query(Video).filter_by(id=vid2.id).one()
vid3 = test_session.query(Video).filter_by(id=vid3.id).one()
# All videos should have the same size
assert vid1.file_group.size == vid2.file_group.size == vid3.file_group.size
# But different lengths
assert vid1.file_group.length == 10
assert vid2.file_group.length == 20
assert vid3.file_group.length == 30
# Test ascending order (lowest ratio first)
await assert_video_search(assert_total=3, assert_ids=[vid3.id, vid2.id, vid1.id], order_by='size_to_duration')
# Test descending order (highest ratio first)
await assert_video_search(assert_total=3, assert_ids=[vid1.id, vid2.id, vid3.id], order_by='-size_to_duration')
@pytest.mark.asyncio
async def test_video_file_format(async_client, test_session, fake_now):
body = dict(video_file_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s')
@ -324,3 +361,161 @@ async def test_video_file_format(async_client, test_session, fake_now):
request, response = await async_client.post('/api/videos/file_format', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['error'] == 'Filename must end with .%(ext)s'
@pytest.mark.asyncio
async def test_video_upload_file_tracking(test_session, async_client, video_factory, await_switches, make_multipart_form,
image_bytes_factory):
"""Test that uploading info.json, poster, and caption files via /api/files/upload properly tracks them in Video and FileGroup."""
from wrolpi.common import get_relative_to_media_directory
# Step 1: Create a Video with just a video file (no info.json, poster, or captions)
video = video_factory(with_video_file=True, title='file_tracking_test')
test_session.commit()
# Step 2: Verify Video and FileGroup are created, but optional files do not exist
file_group = video.file_group
assert not video.info_json_path, 'info.json should not exist yet.'
assert not video.poster_path, 'Poster should not exist yet.'
assert not video.caption_paths, 'Captions should not exist yet.'
assert file_group.files is not None, 'FileGroup should have files array'
initial_file_count = len(file_group.files)
# Step 3: Upload info.json via /api/files/upload
# Extract the video stem (e.g., "file_tracking_test")
video_stem = video.video_path.stem
# Create info.json content
info_json_data = {
'title': 'file_tracking_test',
'duration': 60,
'uploader': 'test_uploader'
}
info_json_bytes = json.dumps(info_json_data).encode()
# Get relative destination directory
destination = str(get_relative_to_media_directory(video.video_path.parent))
filename = f'{video_stem}.info.json'
# POST to /api/files/upload
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(info_json_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=info_json_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Step 4: Verify info.json is tracked
# Refresh the session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert info.json properties exist
assert video.info_json_path is not None, 'Video.info_json_path should exist'
assert video.info_json_file is not None, 'Video.info_json_file should exist'
# Assert FileGroup has the correct number of files
assert len(video.file_group.files) == initial_file_count + 1, \
f'FileGroup should have {initial_file_count + 1} files after adding info.json'
assert video.file_group.data.get('info_json_path'), \
'Video info.json should be in FileGroup.data'
# Step 5: Upload poster image and verify it's added to the FileGroup
file_count_before_poster = len(video.file_group.files)
# Create image using factory
image_data = image_bytes_factory()
# Upload image with same stem as the Video (using .jpg)
poster_filename = f'{video_stem}.jpg'
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=poster_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(image_data))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=image_data, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Poster upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert poster properties exist
assert video.poster_path is not None, 'Video.poster_path should exist'
assert video.poster_file is not None, 'Video.poster_file should exist'
assert video.file_group.data.get('poster_path') == video.poster_path, 'Video poster should be in FileGroup.data'
# Assert poster is in FileGroup.files
file_paths = [f['path'] for f in video.file_group.files]
assert video.poster_path in file_paths, 'Poster should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(video.file_group.files) == file_count_before_poster + 1, \
f'FileGroup should have {file_count_before_poster + 1} files after adding poster'
# Step 6: Upload caption file and verify it's added to the FileGroup
file_count_before_caption = len(video.file_group.files)
# Create caption content
caption_content = """WEBVTT
00:00:00.000 --> 00:00:02.000
This is a test caption.
00:00:02.000 --> 00:00:04.000
Another line of captions.
"""
# Upload caption with same stem as the Video
caption_filename = f'{video_stem}.en.vtt'
caption_bytes = caption_content.encode()
forms = [
dict(name='destination', value=destination),
dict(name='filename', value=caption_filename),
dict(name='chunkNumber', value='0'),
dict(name='totalChunks', value='0'),
dict(name='chunkSize', value=str(len(caption_bytes))),
dict(name='overwrite', value='false'),
dict(name='chunk', value=caption_bytes, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, f'Caption upload failed with status {response.status_code}: {response.json}'
await await_switches()
# Refresh session to get updated data
test_session.expire_all()
video = test_session.query(Video).filter_by(id=video.id).one()
# Assert caption properties exist
assert video.caption_paths, 'Video.caption_paths should exist'
assert len(video.caption_paths) == 1, 'Should have exactly one caption file'
assert video.caption_files, 'Video.caption_files should exist'
assert video.file_group.data.get('caption_paths'), 'Video captions should be in FileGroup.data'
# Assert caption is in FileGroup.files
file_paths = [f['path'] for f in video.file_group.files]
assert video.caption_paths[0] in file_paths, 'Caption should be in FileGroup.files'
# Assert FileGroup has correct number of files
assert len(video.file_group.files) == file_count_before_caption + 1, \
f'FileGroup should have {file_count_before_caption + 1} files after adding caption'

View file

@ -3,7 +3,6 @@ import json
import shutil
from copy import copy
from http import HTTPStatus
from pathlib import Path
from unittest import mock
import pytest
@ -350,8 +349,6 @@ async def test_download_destination(test_session, test_directory, video_download
"""A Video can be downloaded to a directory other than it's Channel's directory."""
# yt-dlp would return the video json.
mock_video_extract_info.return_value = example_video_json
# Download process is successful with no logs.
mock_video_process_runner.return_value = (0, {'stdout': '', 'stderr': ''})
# This result will be ignored during the test, we just need some value so the download does not fail.
mock_video_prepare_filename.return_value = str(test_directory / 'test video.mp4')

View file

@ -315,3 +315,14 @@ def test_browser_profile_to_yt_dlp_arg(file_name, expected):
"""
file_name = pathlib.Path(file_name)
assert lib.browser_profile_to_yt_dlp_arg(file_name) == expected
def test_video_location(test_session, video_factory, channel_factory, async_client):
# Video location without a channel, it opens the preview.
video1 = video_factory(title='vid1')
assert video1.location == '/videos/video/1'
# Video location with a channel.
channel = channel_factory(name='ChannelName')
video2 = video_factory(title='vid2', channel_id=channel.id)
assert video2.location == '/videos/channel/1/video/2'

View file

@ -47,15 +47,17 @@ def get_video(video_id: int) -> Video:
VIDEO_ORDERS = {
'published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) ASC, LOWER(fg.primary_path) ASC',
'-published_datetime': 'COALESCE(fg.published_datetime, fg.download_datetime) DESC NULLS LAST,'
' LOWER(fg.primary_path) ASC',
# Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
'published_datetime': 'fg.effective_datetime ASC, LOWER(fg.primary_path) ASC',
'-published_datetime': 'fg.effective_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
'rank': '2 DESC, LOWER(fg.primary_path) DESC',
'-rank': '2 ASC, LOWER(fg.primary_path) ASC',
'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
'-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
'length': 'fg.length ASC, LOWER(fg.primary_path) ASC',
'-length': 'fg.length DESC, LOWER(fg.primary_path) DESC',
'size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC',
'-size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC',
'viewed': 'fg.viewed ASC',
'-viewed': 'fg.viewed DESC',
'view_count': 'v.view_count ASC',
@ -70,6 +72,8 @@ NO_NULL_ORDERS = {
'-length': 'fg.length IS NOT NULL',
'size': 'fg.size IS NOT NULL',
'-size': 'fg.size IS NOT NULL',
'size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
'-size_to_duration': 'fg.size IS NOT NULL AND fg.length IS NOT NULL',
'view_count': 'v.view_count IS NOT NULL',
'-view_count': 'v.view_count IS NOT NULL',
}

View file

@ -43,8 +43,12 @@ __all__ = [
def model_zim(file_group: FileGroup, session: Session) -> Zim:
"""Create a Zim record for the provided FileGroup."""
zim = Zim(file_group=file_group, path=file_group.primary_path)
session.add(zim)
# Check if a Zim already exists for this FileGroup
zim = session.query(Zim).filter_by(file_group_id=file_group.id).one_or_none()
if not zim:
# Create new Zim if it doesn't exist
zim = Zim(file_group=file_group, path=file_group.primary_path)
session.add(zim)
file_group.title = file_group.primary_path.name
file_group.a_text = split_file_name_words(file_group.primary_path.name)
file_group.indexed = True

View file

@ -32,6 +32,7 @@ gpsd
gpsd-tools
hostapd
iotop
iperf3
kiwix
kiwix-tools
libagg-dev
@ -98,4 +99,4 @@ unzip
vim
vlc
wget
zim-tools
zim-tools

View file

@ -53,11 +53,18 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
# Enable first startup script.
systemctl enable wrolpi-first-startup.service
cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd
# Copy MOTD scripts, delete original.
cp /opt/wrolpi/etc/raspberrypios/motd/20-wrolpi.motd /etc/update-motd.d/20-wrolpi
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi-first.motd /etc/update-motd.d/30-wrolpi
chmod +x /etc/update-motd.d/*
truncate -s 0 /etc/motd
# NetworkManager for the hotspot.
systemctl enable NetworkManager
# Enable iperf3 for speed testing.
systemctl enable iperf3
set +x
echo

View file

@ -144,7 +144,8 @@ chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || echo "Ignoring failure to chang
chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi*
# Copy MOTD once the repair has been successful.
cp /opt/wrolpi/etc/raspberrypios/motd /etc/motd
cp /opt/wrolpi/etc/raspberrypios/motd/30-wrolpi.motd /etc/update-motd.d/30-wrolpi
chmod +x /etc/update-motd.d/*
systemctl restart renderd
systemctl restart wrolpi-help

View file

@ -13,7 +13,7 @@ libzim==3.5.0
mock==5.1.0
psutil==6.0.0
psycopg2-binary==2.9.9
pypdf==5.0.1
pypdf==6.1.3
pytest-asyncio==0.24.0
pytest==8.3.3
python-docx==1.1.2
@ -27,6 +27,5 @@ selenium==4.9.1
srt==3.5.3
tqdm==4.67.1
vininfo[cli]==1.8.0
websockets==13.1
webvtt-py==0.5.1
yt-dlp==2025.04.30
yt-dlp==2025.09.26

View file

@ -4,8 +4,8 @@
docker volume create --name=openstreetmap-data
docker volume create --name=openstreetmap-rendered-tiles
git submodule update --init
docker-compose pull
docker-compose build --parallel
docker-compose up -d db
docker-compose run --rm api db upgrade
docker-compose stop
docker compose pull
docker compose build --parallel
docker compose up -d db
docker compose run --rm api db upgrade
docker compose stop

View file

@ -1,6 +1,8 @@
import asyncio
import json
import logging
import multiprocessing
import sys
from asyncio import CancelledError
from datetime import datetime, timezone, date
from decimal import Decimal
@ -13,6 +15,7 @@ from sanic import response, HTTPResponse, Request, Sanic, SanicException
from wrolpi.common import Base, get_media_directory, logger, LOGGING_CONFIG, TRACE_LEVEL
from wrolpi.errors import APIError
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -141,11 +144,15 @@ async def start_perpetual_tasks(app: Sanic):
logger.debug('start_perpetual_tasks completed')
def perpetual_signal(event: str = None, sleep: int | float = 1):
def perpetual_signal(event: str = None, sleep: int | float = 1, run_while_testing: bool = False):
"""Use Sanic signals to continually call the wrapped function. The wrapped function will continually be called,
even if it has errors. If the function is long-running, it will only be called again after it has finished."""
def wrapper(func: callable):
if PYTEST and not run_while_testing:
# Do not run perpetual signal worker while testing, unless explicitly required.
return func
# Create a Sanic "signal" for the provided function.
event_ = event or f'wrolpi.perpetual.{func.__name__}'

View file

@ -24,7 +24,7 @@ from functools import wraps
from itertools import islice, filterfalse, tee
from multiprocessing.managers import DictProxy
from pathlib import Path
from types import GeneratorType
from types import GeneratorType, MappingProxyType
from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
from urllib.parse import urlparse, urlunsplit
@ -445,7 +445,7 @@ class ConfigFile:
self.width = self.width or 90
# Do not load a global config on import while testing. A global instance should never be used for testing.
self._config = self.default_config.copy()
self._config = deepcopy({k: v for k, v in self.default_config.items()})
if self._config['version'] != 0:
raise RuntimeError('Configs should start at version 0')
@ -1921,19 +1921,17 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
source = json.dumps([{'content': i} for i in entries])
with get_db_curs() as curs:
stmt = '''
WITH vectored AS (
-- Convert the "source" json to a recordset.
with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
select
to_tsvector('english'::regconfig, source.content) AS vector,
source.content
from source
)
SELECT
ts_headline(vectored.content, websearch_to_tsquery(%s), 'MaxFragments=10, MaxWords=8, MinWords=7'),
ts_rank(vectored.vector, websearch_to_tsquery(%s))
FROM vectored
'''
WITH vectored AS (
-- Convert the "source" json to a recordset.
with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
select to_tsvector('english'::regconfig, source.content) AS vector,
source.content
from source)
SELECT ts_headline(vectored.content, websearch_to_tsquery(%s),
'MaxFragments=10, MaxWords=8, MinWords=7'),
ts_rank(vectored.vector, websearch_to_tsquery(%s))
FROM vectored \
'''
curs.execute(stmt, [source, search_str, search_str])
headlines = [tuple(i) for i in curs.fetchall()]
@ -1951,12 +1949,11 @@ async def search_other_estimates(tag_names: List[str]) -> dict:
with get_db_curs() as curs:
stmt = '''
SELECT
COUNT(c.id)
FROM channel c
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
WHERE t.name = %(tag_name)s
'''
SELECT COUNT(c.id)
FROM channel c
LEFT OUTER JOIN public.tag t on t.id = c.tag_id
WHERE t.name = %(tag_name)s \
'''
# TODO handle multiple tags
params = dict(tag_name=tag_names[0])
curs.execute(stmt, params)

View file

@ -9,6 +9,7 @@ import logging
import multiprocessing
import pathlib
import shutil
import sys
import tempfile
import threading
import zipfile
@ -18,7 +19,7 @@ from itertools import zip_longest
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock, AsyncMock
from unittest.mock import MagicMock, AsyncMock, patch
from uuid import uuid1, uuid4
import pytest
@ -185,6 +186,7 @@ async def async_client(test_directory) -> SanicASGITestClient:
try:
yield client
finally:
api_app.stop()
logger.debug('Destroying async_client')
@ -678,8 +680,8 @@ def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.P
@pytest.fixture
def image_bytes_factory():
def _() -> bytes:
with tempfile.NamedTemporaryFile(suffix='.png') as fh:
def _(suffix: str = '.png') -> bytes:
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
@ -841,3 +843,10 @@ def mock_downloader_download_file():
contents = contents_
yield set_contents
@pytest.fixture
def start_status_worker():
"""Enable the status worker for testing."""
with mock.patch('wrolpi.status.DISABLE_STATUS_WORKER', False):
yield

View file

@ -1,5 +1,6 @@
import ctypes
import multiprocessing
import queue
from sanic import Sanic
@ -54,6 +55,7 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.switches = manager.dict()
app.shared_ctx.switches_lock = multiprocessing.Lock()
app.shared_ctx.switches_changed = multiprocessing.Event()
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
# Warnings
app.shared_ctx.warn_once = manager.dict()
@ -110,6 +112,12 @@ def reset_shared_contexts(app: Sanic):
# Switches
app.shared_ctx.switches.clear()
app.shared_ctx.switches_changed.clear()
while True:
# Clear out any pending singlefile archive switches.
try:
app.shared_ctx.archive_singlefiles.get_nowait()
except queue.Empty:
break
# Events.
app.shared_ctx.single_tasks_started.clear()

View file

@ -689,6 +689,9 @@ class DownloadManager:
download.sub_downloader = sub_downloader or None
# Remove Channel relationship, if necessary.
download.channel_id = (settings or dict()).get('channel_id')
save_downloads_config.activate_switch()
return download
@wrol_mode_check
@ -1113,11 +1116,13 @@ class DownloadManager:
@staticmethod
def add_to_skip_list(*urls: str):
get_download_manager_config().skip_urls = list(set(get_download_manager_config().skip_urls) | set(urls))
get_download_manager_config().skip_urls = sorted(list(set(get_download_manager_config().skip_urls) | set(urls)))
@staticmethod
def remove_from_skip_list(url: str):
get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
skip_urls = get_download_manager_config().skip_urls
if url in skip_urls:
get_download_manager_config().skip_urls = [i for i in get_download_manager_config().skip_urls if i != url]
# The global DownloadManager. This should be used everywhere!

View file

@ -96,6 +96,10 @@ class Events:
def send_archive_uploaded(cls, message: str = None, url: str = None):
send_event('upload_archive', message, subject='upload', url=url)
@classmethod
def send_upload_archive_failed(cls, message: str = None):
send_event('upload_archive_failed', message, subject='upload')
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'

View file

@ -228,8 +228,12 @@ class EBook(ModelHelper, Base):
def model_ebook(file_group: FileGroup, session: Session) -> EBook:
"""Creates an EBook model based off a File. Searches for its cover in the provided `files`."""
ebook = EBook(file_group=file_group)
session.add(ebook)
# Check if an EBook already exists for this FileGroup
ebook = session.query(EBook).filter_by(file_group_id=file_group.id).one_or_none()
if not ebook:
# Create new EBook if it doesn't exist
ebook = EBook(file_group=file_group)
session.add(ebook)
# Multiple formats may be available.
epub_file = next(iter(ebook.file_group.my_epub_files()), None)

View file

@ -30,7 +30,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.downloader import download_manager
from wrolpi.downloader import download_manager, Download
from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \
NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError
from wrolpi.events import Events
@ -856,7 +856,6 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
FROM file_group fg
{join}
{f"WHERE {wheres}" if wheres else ""}
GROUP BY fg.id
ORDER BY {order_by}
OFFSET %(offset)s LIMIT %(limit)s
'''
@ -1000,23 +999,23 @@ def split_file_name_words(name: str) -> str:
def get_file_statistics():
with get_db_curs() as curs:
curs.execute('''
SELECT
-- All items in file_group.files are real individual files.
SUM(json_array_length(files)) AS "total_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype = 'application/x-mobipocket-ebook') AS "ebook_count",
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
(SELECT COUNT(*) FROM tag) AS "tags_count",
SUM(size)::BIGINT AS "total_size",
(SELECT COUNT(*) FROM archive) AS archive_count
FROM
file_group
''')
SELECT
-- All items in file_group.files are real individual files.
SUM(json_array_length(files)) AS "total_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'image/%') AS "image_count",
COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'audio/%') AS "audio_count",
COUNT(id) FILTER (WHERE file_group.mimetype = 'application/epub+zip' OR file_group.mimetype =
'application/x-mobipocket-ebook') AS "ebook_count",
(SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files",
(SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims",
(SELECT COUNT(*) FROM tag) AS "tags_count",
SUM(size)::BIGINT AS "total_size",
(SELECT COUNT(*) FROM archive) AS archive_count
FROM file_group
''')
statistics = dict(curs.fetchall()[0])
statistics['total_count'] = statistics['total_count'] or 0
statistics['total_size'] = statistics['total_size'] or 0
@ -1137,25 +1136,25 @@ def get_refresh_progress() -> RefreshProgress:
idempotency = api_app.shared_ctx.refresh.get('idempotency')
if idempotency:
stmt = '''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) FILTER (WHERE idempotency=%(idempotency)s) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency=%(idempotency)s) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency=%(idempotency)s) AS "modeled"
FROM file_group
'''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) FILTER (WHERE idempotency = %(idempotency)s) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency = %(idempotency)s) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency = %(idempotency)s) AS "modeled"
FROM file_group \
'''
else:
# Idempotency has not yet been declared.
stmt = '''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
FROM file_group
'''
SELECT
-- Sum all the files in each FileGroup.
SUM(json_array_length(files)) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
FROM file_group \
'''
with get_db_curs() as curs:
curs.execute(stmt, dict(idempotency=idempotency))
@ -1583,52 +1582,61 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
logger.warning('upsert_file called, but all files are in ignored directories!')
raise IgnoredDirectoryError(f'all files are in ignored directories: {file}')
for i in range(2):
# Try multiple times because uploads happen concurrently and may conflict.
# TODO convert uploads to synchronous in UI.
with get_db_session() as session:
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
# Use a single session context for all database operations
with get_db_session(commit=True) as session:
# Create/update FileGroup with retry logic
file_group = None
for i in range(2):
try:
session.flush([file_group, ])
file_group_id = file_group.id
session.commit()
file_group = FileGroup.from_paths(session, *paths)
session.flush([file_group])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
if i == 1:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
continue
else:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
logger.debug(f'upsert_file: {file_group}')
try:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
if not file_group:
raise RuntimeError(f'upsert_file failed to create FileGroup! {file}')
logger.debug(f'upsert_file: {file_group}')
# Model the FileGroup (creates Archive/Video/etc. and sets indexed=True)
try:
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
# If user uploads a file, then remove it from the skip list so comments can be downloaded.
url = file_group.url if file_group and file_group.url else None
if url and download_manager.is_skipped(url):
download_manager.remove_from_skip_list(url)
# If user uploads a file, then remove it from the download skip list so comments can be downloaded.
# Modify the download (if any) so that the user can click on it and view the uploaded file.
if file_group.url:
if download_manager.is_skipped(file_group.url):
download_manager.remove_from_skip_list(file_group.url)
if download := Download.get_by_url(file_group.url, session):
# Mark download as completed
download.complete()
# Link the download to the location of the uploaded file.
model = file_group.get_model_record()
download.location = model.location if model else file_group.location
upsert_directories([], file.parents)
session.commit()
if tag_names:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
# Add tags if provided
if tag_names:
for tag_name in tag_names:
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
tag = Tag.get_by_name(tag_name, session)
file_group.add_tag(tag.id, session=session)
# Commit all changes in one transaction
session.commit()
# Upsert directories after FileGroup is committed
upsert_directories([], file.parents)
file_group.flush()
return file_group

View file

@ -6,14 +6,14 @@ from datetime import datetime
from functools import singledispatchmethod
from typing import List, Type, Optional, Iterable
from sqlalchemy import Column, String, Computed, BigInteger, Boolean
from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event
from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \
get_relative_to_media_directory, unique_by_predicate
from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime
from wrolpi.db import optional_session
from wrolpi.db import optional_session, get_db_session
from wrolpi.downloader import Download
from wrolpi.errors import FileGroupIsTagged, UnknownFile
from wrolpi.files import indexers
@ -87,6 +87,10 @@ class FileGroup(ModelHelper, Base):
url = Column(String) # the location where this file can be downloaded.
viewed = Column(TZDateTime) # the most recent time a User viewed this file.
# Columns updated by triggers.
# `file_group_effective_datetime_trigger` and `update_effective_datetime` event handler
effective_datetime = Column(TZDateTime) # Equivalent to COALESCE(published_datetime, download_datetime)
tag_files: Iterable[TagFile] = relationship('TagFile', cascade='all')
a_text = deferred(Column(String))
@ -119,7 +123,8 @@ class FileGroup(ModelHelper, Base):
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
logger.error(
f"Found TagFile with problematic tag reference in __json__: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
tags = sorted(tag_names)
d = dict(
author=self.author,
@ -275,6 +280,14 @@ class FileGroup(ModelHelper, Base):
def my_mobi_files(self) -> List[dict]:
return self.my_files('application/x-mobipocket-ebook')
def my_html_files(self) -> List[dict]:
html_files = self.my_files('text/html', 'application/octet-stream')
html_files = [i for i in html_files if i['path'].name.endswith('.html')]
return html_files
def my_html_paths(self) -> List[pathlib.Path]:
return [i['path'] for i in self.my_html_files()]
def delete(self, add_to_skip_list: bool = True):
"""Delete this FileGroup record, and all of its files.
@ -385,7 +398,8 @@ class FileGroup(ModelHelper, Base):
if PYTEST:
raise
# Log the error but continue processing other tags
logger.error(f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
logger.error(
f"Found TagFile with problematic tag reference: {tag_file.id if hasattr(tag_file, 'id') else 'unknown'}")
return result
def merge(self, file_groups: List['FileGroup']):
@ -456,6 +470,14 @@ class FileGroup(ModelHelper, Base):
@property
def location(self):
"""Returns the URL that the FileGroup can be previewed."""
klass = self.get_model_class()
if klass:
# Return the model location when possible.
with get_db_session() as session:
instance = klass.get_by_path(self.primary_path, session)
if instance and hasattr(instance, 'location'):
return instance.location
parent = str(get_relative_to_media_directory(self.primary_path.parent))
preview = str(get_relative_to_media_directory(self.primary_path))
if parent == '.':
@ -481,16 +503,20 @@ class FileGroup(ModelHelper, Base):
elif Zim.can_model(self):
return Zim
def get_model_record(self) -> Optional[ModelHelper]:
klass = self.get_model_class()
if klass:
with get_db_session() as session:
return klass.get_by_path(self.primary_path, session)
return None
def do_model(self, session: Session) -> Optional[ModelHelper]:
"""Get/Create the Model record (Video/Archive/etc.) for this FileGroup, if any."""
if model_class := self.get_model_class():
if model := model_class.get_by_path(self.primary_path, session):
# Already modeled.
logger.debug(f'already modeled: {self} {model}')
return model
# Create new model (Video/Archive/Ebook).
# Always call do_model() to ensure FileGroup.data is updated when files change.
# The model's do_model() should be idempotent (safe to call multiple times).
model = model_class.do_model(self, session)
logger.debug(f'new model: {self} {model}')
logger.debug(f'modeled: {self} {model}')
return model
# Index the file if no models are available.
@ -515,6 +541,12 @@ class FileGroup(ModelHelper, Base):
raise
@event.listens_for(FileGroup, 'before_insert')
@event.listens_for(FileGroup, 'before_update')
def update_effective_datetime(mapper, connection, target):
target.effective_datetime = target.published_datetime or target.download_datetime
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""
__tablename__ = 'directory'

View file

@ -671,9 +671,12 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
assert not file_group.tag_names, 'No tags this time.'
assert file_group.indexed, 'File should be indexed after upload.'
assert file_group.model, 'File should be modeled'
assert isinstance(file_group.get_model_record(), Video)
assert file_group.location == '/videos/video/2'
# Video was modeled, but has no Channel.
video: Video = test_session.query(Video).one()
assert video.video_path
assert video.file_group == file_group
assert not video.info_json_path
assert not video.channel and not video.channel_id

View file

@ -18,6 +18,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command
from wrolpi.common import logger, get_warn_once, unique_by_predicate, partition, TRACE_LEVEL
from wrolpi.dates import now
from wrolpi.vars import PYTEST
try:
import psutil
@ -565,11 +566,18 @@ async def get_power_stats() -> PowerStats:
return PowerStats()
DISABLE_STATUS_WORKER = bool(PYTEST) # Do not run status worker while testing, unless explicitly needed.
@perpetual_signal(sleep=5)
async def status_worker(count: int = None, sleep_time: int = 5):
"""A background process which will gather historical data about system statistics."""
shared_status = api_app.shared_ctx.status
if PYTEST and DISABLE_STATUS_WORKER:
# Tests are running, status worker has not been enabled by `start_status_worker` fixture.
return
if count is not None and count <= 0:
logger.debug('Status worker ran out of count')
return dict(**shared_status)
@ -579,13 +587,13 @@ async def status_worker(count: int = None, sleep_time: int = 5):
# Update global `status` dict with stats that are gathered instantly.
cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \
await asyncio.gather(
get_cpu_stats(),
get_load_stats(),
get_drives_stats(),
get_memory_stats(),
get_processes_stats(),
get_iostat_stats(),
get_power_stats(),
asyncio.create_task(get_cpu_stats(), name='get_cpu_stats'),
asyncio.create_task(get_load_stats(), name='get_load_stats'),
asyncio.create_task(get_drives_stats(), name='get_drives_stats'),
asyncio.create_task(get_memory_stats(), name='get_memory_stats'),
asyncio.create_task(get_processes_stats(), name='get_processes_stats'),
asyncio.create_task(get_iostat_stats(), name='get_iostat_stats'),
asyncio.create_task(get_power_stats(), name='get_power_stats'),
)
shared_status.update({
'cpu_stats': cpu_stats.__json__(),

View file

@ -85,7 +85,7 @@ def register_switch_handler(switch_name: str):
DEBUG_LOGGED = False
@perpetual_signal(sleep=0.1)
@perpetual_signal(sleep=0.1, run_while_testing=True)
async def switch_worker():
"""Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time."""
global DEBUG_LOGGED

View file

@ -440,6 +440,12 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
# Assert that the config has been saved
config = get_download_manager_config()
assert len(config.downloads) == 1
assert config.downloads[0]['url'] == 'https://example.com/1'
assert config.downloads[0]['frequency'] == 1_000
assert config.downloads[0]['downloader'] == 'archive'
# Create two once-downloads.
body = dict(
@ -451,6 +457,10 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
assert response.status_code == HTTPStatus.CREATED
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
# Assert that the config has been saved
config = get_download_manager_config()
assert len(config.downloads) == 3
assert {d['url'] for d in config.downloads} == {f'https://example.com/{i}' for i in range(1, 4)}
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
assert not download1.settings
@ -473,6 +483,11 @@ async def test_download_crud(test_session, async_client, test_download_manager_c
download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
assert download2.__json__() == expected_download
assert download2.tag_names == [tag1.name, tag2.name]
# Assert that the config has been saved
config = get_download_manager_config()
assert len(config.downloads) == 3
download_config = next(d for d in config.downloads if d['url'] == 'https://example.com/1')
assert download_config['tag_names'] == [tag1.name, tag2.name]
def test_get_downloaders(test_client):

View file

@ -67,7 +67,7 @@ async def test_get_drives_stats(async_client):
@skip_circleci
@pytest.mark.asyncio
async def test_status_worker(async_client):
async def test_status_worker(async_client, start_status_worker):
"""Status worker calls itself perpetually, but can be limited during testing."""
from wrolpi.api_utils import api_app
assert async_client.sanic_app.shared_ctx.status['cpu_stats'] == dict()