diff --git a/alembic/versions/3b6918aeca4b_file_group.py b/alembic/versions/3b6918aeca4b_file_group.py new file mode 100644 index 00000000..d932da3c --- /dev/null +++ b/alembic/versions/3b6918aeca4b_file_group.py @@ -0,0 +1,233 @@ +"""File group. + +Revision ID: 3b6918aeca4b +Revises: 930b9b7adb79 +Create Date: 2023-02-27 14:52:46.943351 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '3b6918aeca4b' +down_revision = '930b9b7adb79' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DELETE FROM video') # noqa + session.execute('ALTER TABLE video DROP COLUMN caption_path') + session.execute('ALTER TABLE video DROP COLUMN censored') + session.execute('ALTER TABLE video DROP COLUMN favorite') + session.execute('ALTER TABLE video DROP COLUMN info_json_path') + session.execute('ALTER TABLE video DROP COLUMN modification_datetime') + session.execute('ALTER TABLE video DROP COLUMN poster_path') + session.execute('ALTER TABLE video DROP COLUMN size') + session.execute('ALTER TABLE video DROP COLUMN title') + session.execute('ALTER TABLE video DROP COLUMN validated') + session.execute('ALTER TABLE video DROP COLUMN video_path') + + session.execute('DELETE FROM archive') # noqa + session.execute('ALTER TABLE archive DROP COLUMN readability_json_path') + session.execute('ALTER TABLE archive DROP COLUMN readability_path') + session.execute('ALTER TABLE archive DROP COLUMN readability_txt_path') + session.execute('ALTER TABLE archive DROP COLUMN screenshot_path') + session.execute('ALTER TABLE archive DROP COLUMN singlefile_path') + session.execute('ALTER TABLE archive DROP COLUMN title') + session.execute('ALTER TABLE archive DROP COLUMN validated') + + session.execute('DELETE FROM ebook') # noqa + session.execute('ALTER TABLE ebook DROP COLUMN cover_path') + session.execute('ALTER TABLE ebook DROP COLUMN ebook_path') + session.execute('ALTER TABLE ebook DROP COLUMN title') + + session.execute('DROP TABLE IF EXISTS file') + + session.execute(''' + CREATE TABLE file_group ( + id BIGSERIAL PRIMARY KEY, + data JSON, + files JSON, + full_stem TEXT NOT NULL UNIQUE, + idempotency TIMESTAMP WITH TIME ZONE, + indexed BOOLEAN DEFAULT FALSE, + mimetype TEXT NOT NULL, + model TEXT, + modification_datetime TIMESTAMP WITH TIME ZONE, + primary_path TEXT NOT NULL UNIQUE, + size BIGINT, + title TEXT, + a_text TEXT, + b_text TEXT, + c_text TEXT, + d_text TEXT, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char") + ) STORED + )''') + + session.execute('ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') + session.execute('ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') + session.execute('ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') + + session.execute(''' + CREATE TABLE tag ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + color VARCHAR(7) + )''') + + session.execute(''' + CREATE TABLE tag_file ( + tag_id INTEGER REFERENCES tag(id), + file_group_id BIGINT REFERENCES file_group(id), + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tag_id, file_group_id) + )''') + + session.execute(''' + INSERT INTO tag (name, color) VALUES + ('Automotive','#000000'), + ('Clothing','#9858C5'), + ('Comms','#ECD71C'), + ('Cooking','#00A04A'), + ('Electronics','#000000'), + ('Favorite','#000000'), + ('Firearms','#000000'), + ('Fire Safety','#000000'), + ('First Aid','#C40405'), + ('Food','#2F8055'), + ('Fuel','#000000'), + ('Garden','#3E8B22'), + ('Husbandry','#000000'), + ('Hygiene','#E6E6E6'), + ('Knots','#000000'), + ('Light','#EEEEEE'), + ('Maps','#333333'), + ('Mechanical','#7A7A7A'), + ('Medical','#000000'), + ('Nuclear','#000000'), + ('Power','#D837FF'), + ('Radio','#000000'), + ('Security','#333333'), + ('Shelter','#A18A62'), + ('Survival','#A18A62'), + ('Tools','#333333'), + ('Water','#291BC2') + ''') + + session.execute('UPDATE wrolpi_flag SET refresh_complete = false') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.file_group OWNER TO wrolpi') + session.execute('ALTER TABLE public.tag OWNER TO wrolpi') + session.execute('ALTER TABLE public.tag_file OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP TABLE IF EXISTS tag_file') + session.execute('DROP TABLE IF EXISTS tag') + + session.execute('ALTER TABLE video DROP COLUMN file_group_id') + session.execute('ALTER TABLE ebook DROP COLUMN file_group_id') + session.execute('ALTER TABLE archive DROP COLUMN file_group_id') + + session.execute('DROP TABLE file_group') + + session.execute(''' + create table file + ( + path text not null primary key, + idempotency timestamp with time zone, + indexed boolean default false, + mimetype text, + suffix text, + model text, + associated boolean default false, + modification_datetime timestamp with time zone, + size bigint, + title text, + a_text text, + b_text text, + c_text text, + d_text text, + textsearch tsvector generated always as (( + ((setweight(to_tsvector('english'::regconfig, COALESCE(a_text, ''::text)), 'A'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(b_text, ''::text)), 'B'::"char")) || + setweight(to_tsvector('english'::regconfig, COALESCE(c_text, ''::text)), 'C'::"char")) || + setweight(to_tsvector('english'::regconfig, COALESCE(d_text, ''::text)), 'D'::"char"))) stored, + full_stem text + ); + + alter table file + owner to postgres; + + create index file_mimetype_idx + on file (mimetype); + + create index file_textsearch_idx + on file using gin (textsearch); + + create index file_modification_datetime_idx + on file (modification_datetime); + + create index file_size_idx + on file (size); + + create index file_model_idx + on file (model); + + create index file_idempotency_idx + on file (idempotency); + + create index file_suffix_idx + on file (suffix); + + create index file_full_stem_idx + on file (full_stem); + ''') + + session.execute('DELETE FROM video') # noqa + session.execute('ALTER TABLE video ADD COLUMN caption_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE video ADD COLUMN censored BOOLEAN DEFAULT FALSE') + session.execute('ALTER TABLE video ADD COLUMN favorite TIMESTAMPTZ') + session.execute('ALTER TABLE video ADD COLUMN info_json_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE video ADD COLUMN poster_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE video ADD COLUMN video_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ') + session.execute('ALTER TABLE video ADD COLUMN size BIGINT') + session.execute('ALTER TABLE video ADD COLUMN title TEXT') + session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE') + + session.execute('DELETE FROM archive') # noqa + session.execute('ALTER TABLE archive ADD COLUMN readability_json_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE archive ADD COLUMN readability_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE archive ADD COLUMN readability_txt_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE archive ADD COLUMN screenshot_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE archive ADD COLUMN singlefile_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE archive ADD COLUMN title TEXT') + session.execute('ALTER TABLE archive ADD COLUMN validated BOOLEAN DEFAULT FALSE') + + session.execute('DELETE FROM ebook') # noqa + session.execute('ALTER TABLE ebook ADD COLUMN cover_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE ebook ADD COLUMN ebook_path TEXT REFERENCES file(path)') + session.execute('ALTER TABLE ebook ADD COLUMN title TEXT') + + session.execute('UPDATE wrolpi_flag SET refresh_complete = false') + + if not DOCKERIZED: + session.execute('ALTER TABLE public.file OWNER TO wrolpi') diff --git a/app/package-lock.json b/app/package-lock.json index b8d9b300..8c51d76a 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -15,6 +15,7 @@ "lodash": "^4.17.21", "query-string": "^8.1.0", "react": "^18.2.0", + "react-colorful": "^5.6.1", "react-dom": "^18.2.0", "react-dropzone": "^14.2.3", "react-qr-code": "^2.0.11", @@ -14438,6 +14439,15 @@ "node": ">=14" } }, + "node_modules/react-colorful": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz", + "integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==", + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, "node_modules/react-dev-utils": { "version": "12.0.1", "resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz", @@ -27847,6 +27857,12 @@ "whatwg-fetch": "^3.6.2" } }, + "react-colorful": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/react-colorful/-/react-colorful-5.6.1.tgz", + "integrity": "sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==", + "requires": {} + }, "react-dev-utils": { "version": "12.0.1", "resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz", diff --git a/app/package.json b/app/package.json index 916c2f00..deb59118 100644 --- a/app/package.json +++ b/app/package.json @@ -10,6 +10,7 @@ "lodash": "^4.17.21", "query-string": "^8.1.0", "react": "^18.2.0", + "react-colorful": "^5.6.1", "react-dom": "^18.2.0", "react-dropzone": "^14.2.3", "react-qr-code": "^2.0.11", diff --git a/app/src/App.js b/app/src/App.js index 6913ada0..f64e2287 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -19,6 +19,7 @@ import {Donate} from "./components/Donate"; import {useEventsInterval} from "./Events"; import {SemanticToastContainer} from "react-semantic-toasts-2"; import {FilePreviewWrapper} from "./components/FilePreview"; +import {TagsContext, useTags} from "./Tags"; function PageNotFound() { const {t} = useContext(ThemeContext); @@ -75,7 +76,8 @@ const router = createBrowserRouter(createRoutesFromElements()); export default function App() { - const status = useStatusInterval(); + const statusValue = useStatusInterval(); + const tagsValue = useTags(); useEventsInterval(); return @@ -85,9 +87,11 @@ export default function App() { {/* Toasts can be on any page. */} - - - + + + + + diff --git a/app/src/Dashboard.js b/app/src/Dashboard.js index a15e5746..16665359 100644 --- a/app/src/Dashboard.js +++ b/app/src/Dashboard.js @@ -12,6 +12,7 @@ import {Divider, Icon, Message} from "semantic-ui-react"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {refreshFiles} from "./api"; import _ from "lodash"; +import {TagsSegment} from "./Tags"; function FlagsMessages({flags}) { if (!flags) { @@ -62,7 +63,7 @@ function FlagsMessages({flags}) { export function Dashboard() { useTitle('Dashboard'); - const {searchStr, setSearchStr} = useSearchFiles(); + const {searchStr, setSearchStr, activeTags} = useSearchFiles(); const {status} = useContext(StatusContext); const wrol_mode = status ? status['wrol_mode'] : null; @@ -74,13 +75,14 @@ export function Dashboard() { // Only show dashboard parts if not searching. let body; - if (searchStr) { + if (searchStr || activeTags) { body = ; } else { body = <> {!wrol_mode && downloads} {/* Hide Status when user is starting a download */} {!downloadOpen && } + ; } diff --git a/app/src/Tags.js b/app/src/Tags.js new file mode 100644 index 00000000..593291ba --- /dev/null +++ b/app/src/Tags.js @@ -0,0 +1,315 @@ +import React, {useEffect, useState} from "react"; +import {addTag, deleteTag, getTags, postTag, removeTag} from "./api"; +import { + Button as SButton, + Confirm, + Dimmer, + Divider, + Form, + FormField, + FormInput, + Grid, + Header, + Label, + Loader, + Loader as SLoader, + Modal, +} from "semantic-ui-react"; +import {contrastingColor, HelpPopup} from "./components/Common"; +import {Button, Segment} from "./components/Theme"; +import _ from "lodash"; +import {Link} from "react-router-dom"; +import {HexColorPicker} from "react-colorful"; + +export const TagsContext = React.createContext({tags: [], fetchTags: null}); + +export function useTags() { + const [tags, setTags] = React.useState(null); + const [tagNames, setTagNames] = React.useState(null); + + const fetchTags = async () => { + try { + const t = await getTags(); + setTags(t); + setTagNames(t.map(i => i['name'])); + } catch (e) { + setTags([]); + setTagNames([]); + console.error(e); + } + } + + const NameToTagLabel = ({name, color, to, ...props}) => { + if (tags && tags.length) { + for (let i = 0; i < tags.length; i++) { + const tag = tags[i]; + if (name === tag['name']) { + const textColor = contrastingColor(tag['color']); + const label = ; + return label; + } + } + } + + // Could not find tag by name, or no tags have been fetched. + return ; + } + + const TagsGroup = ({tagNames, onClick}) => { + if (!tagNames || tagNames.length === 0) { + return ; + } + return + {tagNames.map(i => onClick(i)}/>)} + + } + + const TagsLinkGroup = ({tagNames}) => { + if (!tagNames || tagNames.length === 0) { + return ; + } + return + {tagNames.map(i => + + + + )} + + } + + useEffect(() => { + setTags([]); + fetchTags(); + }, []); + + return {tags, tagNames, NameToTagLabel, TagsGroup, TagsLinkGroup, fetchTags} +} + +function DeleteTagLabel({name, onConfirm}) { + const {NameToTagLabel} = React.useContext(TagsContext); + const [confirmOpen, setConfirmOpen] = useState(false); + + return
+ setConfirmOpen(true)} icon='close'/> + setConfirmOpen(false)} + onConfirm={async () => onConfirm(name)} + /> + +
+
+} + +function DeleteTagModal() { + const {fetchTags, tagNames} = React.useContext(TagsContext); + + const [open, setOpen] = useState(false); + + const localDeleteTag = async (name) => { + await deleteTag(name); + if (fetchTags) { + await fetchTags(); + } + } + + const content = tagNames && tagNames.length ? + tagNames.map(i => ) + : ; + + return <> + setOpen(true)} + onClose={() => setOpen(false)} + > + Delete Tags + + {content} + + + setOpen(true)}> + Delete + + +} + +function CreateTagModal() { + const {fetchTags} = React.useContext(TagsContext); + + const [open, setOpen] = useState(false); + const [tagName, setTagName] = useState(''); + const [tagColor, setTagColor] = useState('#000000'); + const textColor = contrastingColor(tagColor); + + const saveTag = async () => { + await postTag(tagName, tagColor); + if (fetchTags) { + await fetchTags(); + } + setTagName(''); + } + + return <> + setOpen(true)} + onClose={() => setOpen(false)} + > + Create New Tag + + + + + + + +
+ + setTagName(value)} + /> + + + + + + Save + + + +
+
+ + setOpen(true)}>New + +} + +export function TagsModal({fileGroup, onClick}) { + const {tags: usedTags} = fileGroup; + const {TagsGroup} = useTags(); + const {tags: availableTags} = React.useContext(TagsContext); + const [open, setOpen] = useState(false); + const [loading, setLoading] = useState(false); + + const localAddTag = async (name) => { + setLoading(true); + try { + await addTag(fileGroup, name); + await onClick(); + console.debug('Added tag'); + } finally { + setLoading(false); + } + }; + + const localRemoveTag = async (name) => { + setLoading(true); + try { + await removeTag(fileGroup, name); + await onClick(); + console.debug('Removed tag'); + } finally { + setLoading(false); + } + }; + + const availableTagNames = availableTags && availableTags.length ? availableTags.map(i => i['name']) : []; + + const unusedTags = _.difference(availableTagNames, usedTags); + + let availableTagsGroup = ; + if (unusedTags && unusedTags.length) { + availableTagsGroup = ; + } + + let usedTagsGroup = 'Add tags by clicking them below'; + if (usedTags && usedTags.length) { + // TODO sort these labels. + usedTagsGroup = ; + } + + const modal = setOpen(true)} + onClose={() => setOpen(false)}> + + {loading && } +
Used Tags
+ + {usedTagsGroup} + + + + {availableTagsGroup} +
+ + + + +
+ + return <> + ); - let descriptionPane = { menuItem: 'Description', render: () =>
@@ -119,13 +103,13 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {
     let aboutPane = {
         menuItem: 'About', render: () => 
             

Size

-

{video.size ? humanFileSize(video.size) : 'Unknown'}

+

{videoFile.size ? humanFileSize(videoFile.size) : 'Unknown'}

Source URL

{video.url ? {video.url} : 'N/A'}

View Count

-

{viewCount ? humanNumber(viewCount) : 'N/A'}

+

{video.view_count ? humanNumber(video.view_count) : 'N/A'}

Censored

{video.censored ? 'Yes' : 'No'}

@@ -138,25 +122,30 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) { }; + const {poster_file, caption_files, info_json_file} = video; let filesPane = { menuItem: 'Files', render: () =>

Video File

{videoFileLink(video['video_path'])}

Info JSON File

- {video['info_json_path']} + {info_json_file && + {info_json_file['path']} + } -

Caption File

- {video['caption_path']} +

Caption Files

+ {caption_files && + caption_files.map(i => {i['path']}) + }

Poster File

- {video['poster_path']} + {poster_file && + {poster_file['path']} + }

Directory

{videoFileLink(videoFile['directory'], true)} @@ -188,7 +177,7 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) { -
{video.title || video.video_path}
+
{videoFile.title || videoFile.stem || video.video_path}
{video.upload_date &&

{isoDatetimeToString(video.upload_date)}

}

{channel && @@ -197,7 +186,6 @@ function VideoPage({videoFile, prevFile, nextFile, setFavorite, ...props}) {

- {favorite_button}