From 95c8610df28cddbf8a129ae0f53dc63f5317ea54 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 27 Dec 2022 15:41:55 -0700 Subject: [PATCH] File.d_text is reformatted to break up long lines. --- modules/archive/__init__.py | 3 ++- modules/archive/test/test_lib.py | 2 +- modules/videos/__init__.py | 3 ++- test/example.pdf | Bin 11901 -> 16360 bytes wrolpi/common.py | 40 ++++++++++++++++++++++++++++++- wrolpi/files/indexers.py | 14 +++++------ wrolpi/files/models.py | 6 +---- wrolpi/files/test/test_lib.py | 21 +++++++++++++--- wrolpi/test/test_common.py | 33 +++++++++++++++++++++++-- 9 files changed, 101 insertions(+), 21 deletions(-) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index dfb8a27f..4307fbb6 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -5,7 +5,7 @@ from typing import Tuple, Dict, List from sqlalchemy.orm import Session -from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length from wrolpi.db import optional_session, get_db_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError @@ -182,4 +182,5 @@ class ArchiveIndexer(Indexer, ABC): return None, None, None, None a = cls.get_title(file) d = cls.get_article(file) + d = split_lines_by_length(d) return a, None, None, d diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 70d3c73c..55d6b200 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -675,7 +675,7 @@ async def test_archive_download_index(test_session, test_directory, image_file): assert archive.readability_path and archive.readability_path.exists() assert archive.readability_json_path assert archive.readability_txt_path and archive.readability_txt_path.exists() - assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed' + assert archive.readability_txt_file.d_text == 'the readability', 'Readability text was not indexed' assert archive.title == 'the singlefile', 'Did not get the title from the singlefile' assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \ 'Did not store the screenshot' diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 46d084f0..9830e573 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from modules.videos.models import Video, Channel from wrolpi.captions import read_captions, extract_captions -from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent, split_lines_by_length from wrolpi.db import get_db_curs from wrolpi.files.indexers import Indexer, register_indexer from wrolpi.files.lib import split_path_stem_and_suffix @@ -159,4 +159,5 @@ class VideoIndexer(Indexer, ABC): elif EXTRACT_SUBTITLES: d = extract_captions(file.path) or '' + d = split_lines_by_length(d) return a, None, c, d diff --git a/test/example.pdf b/test/example.pdf index bbcda4bbd0f32742840804b49ad56ab7a72fe711..4db0cbbfa0658a0a29693fecce8ab4962aa2809b 100644 GIT binary patch delta 11349 zcmbt)Wl&t*((a1~cY-rG1e;-CaQ8uiI|O&vK!AbZE`z%U2reNIED+o^xVt+92+1XH zopbN`&Z$%1k2_VnX7#gsb@%G0Ygf(e=gDv%Ry%``D1rrx)f|jl0%Em4vUzagr6m{j zikkr9M8a25ffG`b25AOCVo8mwyvLMI-w@8(L~+_7HDQe> zHvRzb1WdqASSNVtq_21Jwj1WHF0KfTXC4fVhk0?f2TNZl_z&ST(!bL-5A+eVjFurI z#t=isrMMF40_LcLb_1~-H;z-AN5j>F^xcu6%&3NZuH_rA%Cd^%h1eAgC0L}3nHjuZ za_*OL<>I>vNR-QdQ-Ua+TzfvwR2keoX!3^y@V|h0*MzE)A#+3n@dS*W{iSedW07NP zke$Ac5S4|*yi&;c*s4%)5sGlD#m4DgS*^_llbnsX(k5fof6s3z>3Y}=d z7nwWv2`B7|La5dpx_S57015Tc#H{usn&zl$x&AMMQvNvFR1lPBCFGUiL~TT5bmgW~*3L?F^IWp4swv7pkfX4rKdlxk$s7v2=&sVC$gi%x^BnGY{hRlVwJF!StG}U|KK9e%t(_=@ z``lxSt9EEI#08fbf`X_5+U){Tu7L_oRbruE?>=(H)ClspFrOGl!WS z28I9t2ylL0K)AX82LE6n&_6(7VJ;<0Cus+hxii`Xg;eY!tlFYm|Zao z?Zn#YJ6OA`)$X$Uc*|Y3`?$c}a%7SeJ*berD1_1MPMpEjW^S}7B;mKr3(_y%%%Le+ z&@_5snHN``?+Ap@sPDrmxCjgRWg;GC#(uSnMAGD96BZsZ_PBQNkwi9W!+I45x{3Mw zHjSlwrryoYtC+o0ITOG9E=Ked2#Y%jul1LMwFzym`8VbzxY0ND1`Sk2`pTdMpLjW7 z1}6s1hsXZ}MmcWJ;V^H08)RwmMh*YPS1tx3QgnE0(H|Yc&I$+-dC&h<>|lzvENKy| zLiOVeN~UC0b(!w?MHqp3`JhAH;&B6|%H{9hCBjNjg=ja%NjS2Zn|6aJ58O)!WcJ^d63mDy-md_NeGca{$b#S*=5ZcTnC@cNTNPgSh-Kse4cQd z=hk|^ds|cieq=0h$?KN%b6TqbdGN0mD$9F_4;4uwO;JV9;G?SeDFfGL*%`gl)=ijM z)WxJ!SFZ3e#rDw%ZGBuz1S~eZg!)6&#(pA;mSbjNa<$3Q-9_W55Bq4Y z%Fz~whqwBohJ-@9%COE)?`euL`iMkJN6VLOcRb4g(hpubsVDVOd1F}2F+)_Q1w|SD zs~k@q&k0&*iAWRiHWMbCNa7qiVR7JitdMfRcW%Jj_I@uF9Ys zoe7aCEMMOp`yku8M0KfMsZ9Ki-1eoORJc<4_u?53`GVI3Ig|uUubBEfJTmEpx_7>m zF_X8{fqP{Wyki{6DO#C((@II+l51c(wc1FGnwMj|HrM!QNbdQRx-5KDqlC_CNw$aI z_e)xpYFx87=fK-d@j+riq)k+zo$>U4OFtxZrd@PEvqzSt)iGT?B%Qt$L+&c6e zC-%vZU8XJ-%FXK$%<$v#uW6F3oXeQQlJ$o{97h_ASB-OqVpCabghI%i}{7NneiEBnDl%*mNB;({sUsl#6a$0>Hg3FVM{8~PQn(^Lz@>D0u;IK4=(``6quxZUoZ$WLy zG5M^d6Z$S4PW^I!Sbbo!LwQu7DQ7H@jht^pZFg{**0pYA-oaQZBJ~!LEM3VVkZSIx zeQnLo&_bne)0LbfC>QYneYf`j8GjgE5=o58~%beFtEQ9F^|<#4`G#(Jll-1|E2~{uuG+WhDFvcXoAQ-`Jnh#EBF=eO^hQ<}Q75YAXl3;v3FKdb(t;|(Z%zW7Q9eL^hoGIV2-9}AJ1z;r^7d*u+dj8n>7l0cn!;?%aBe+Cxz;~O z)Ww9PB&bUzkWa}Q)T)UeoQy|@#7r5Z$V_H0(w2oCjn+5$Z*Q3n*Lh=TkPTbfj_ZAo)r z6A`yqy)5bxWawun&D@^-O=EDu<(!l2NW%LP%$44$=;+ubC>$?^awkspOqZVlg#xrO zwJs(fWKGNow8mtfz=ufh47f1D_g;ij*iXO^&1m79EUa22B!`MOkP2C199te4R3r)wjBTN)Lsz0%MJ0%m1k{xWrPO&VzG4BMDJCswQ6;nMmV$E1uR&1?)AD7a=kBN z;&y!0Rc+MS-qU)rvSNq5;1k4!ec27-$Hi;5eMy|9yMF?n(eZTbjEv$ce}9=}dC!Pq zr&N-(f2Enb;CJ%MCh*H*OYVa2`3wiDCn%X*t%dRYkp2ysLaAaH)2e~cuZfLa&&j3* z#k{}V=2ndEc)jLo*M3)Y=8D?fIME;u-!ADiM4{2~X~(u5D+mz}odmO{!B8+|rZRA9 zYuJu0sF}R+sfn#$ef&5go8p*a&YJN?Q+n4o*(-dsOz*_{=sm5wk59pr=v??3#<&U#a~wsHB0#9N~xCS2gGs; zzsp@j7v&R-s3R3_*!w=qX{?U!smSnE$DErBRH4i+7;SJ#K==iHh8^}{tf-r5=_+`y z37LF8wb#UVKPVCA?|0eHvGMR-j^=3t(^6p$WxRkF6LIvQ2=nNa^l8bt^fk_MZTn|u zx%8H!Nah>wi!r3Z4s)tjv_mgQ%Chd=@s~LoALtuWF0uMw@L#^JDs8?;4?^jQ{*Sn) z1Um!9{W&U4j{pxw1QLKg5pth01!iOXL)-+V^}NrCe)`Xi94yF$+TJ za{Sl`T1}|qK+yqm2hoJ=>0(R1ZVPO0vuJj9ZT_K8QA_~I{{U8V)3@=9V#D*>{5OeU zukwmZN7aDyfG^gVlV-GkKW%c&dm(jwg8|;iJQsou6C*7ZrI=moslx{O1O8g>YlK zDQrcbR%t}gzGg&AJa23l2oabHkDD0>3uh`N$o(U zU{VBX+PvZgFb+GS*Dbp;8+vr>F|QZ0WUI!<7OBV!(jLM>yqI|e3%;YfXPcC+>{BUq zIOoe{yGxEay8Zase6;APY_=qL8PLnF8h5cGL5U>Zp=2l@O>wpGf`Z-)^`%pm+_gbW>e)l z!OGRx)U^)wCB$-+BvhpWGmOWsYE}M4r+~ClyqdF3t$2#e6y4{Ad%sH+HRrOL@c6R+&CR`6ZNn65WXB6F zUqOy#N;gLwL&hL9gmkeI#$>IRq2r_j6y8;^cDY8yWd5ZWfe!zK&N&rkgNUktEQop= zlL>Hm4>%cEAJ16Fq9Sf3VA?Mns7NYr%rw&Rdpz8RP8cS2`7NN_>V5RPudcqf=|XBw zF0q3N{WvC*H+6pcyjHCUW5%(LYroIBh54W0?;r4TKTXRmEV(eAOrvGYU(clr;d7Qi zk`F%^A1qO^6T=3}KI!Ko8P!#@X^z-68qpSpnQ2#jx2g4=B2~F?$qp|Prd+2Tl)}X| z@Og0#OhY5YeMQZAbq;&WKmINL?c2C;W}kVO^59~Fh1k{xJMR6~_6u^KvJC`e>KI`> zYRA!UGJ>Q8vOZ;_Ucj`s>6;srxNu>Hw|G8fyNsfoxOtnV*|(^yU8P=i)jwShzx)#Q zzM*Dr0V?{$FvuSwff8f~>tP5z-P(3mb#Bi>~{sa1hZ^ZT|3-72*H>#b`I{>Z@- zEaNa{(I0_nB^)?TI%qXHTrrzJosy|3ydjXws(s0BzTI4C5;0xj8A@s#TCy+3d1%yn z`BAPM@9JKaMU+B2W2Fo1-1LfTel#6DJ$xu_)~{~&q7gPu4Ybj2YkThx60zY^er&)O zRd+XMLQ5qqtS@wkUe|LCn=7{(b%Wo-uG)=1$KA}bf5ZwSIoNBpz7TY7;UjfB7*?!$ zl_G1tOfcngRa~sg#MrwcI)514E1JHA1jp4B{qYs;rN7H6n%HHRD~<4eFqLRvJ}$Yx zs`5dMsOpdWJN4A_WZWcCJbzWi*1EHL+|~20VqA5P7$nh%hizJJ9>dh*Y}~O2*t&rz zzvQ6PMVA}Riv@vHD<;xAiFve>1%a64E4m%w!(g>+WA^c*&l;=Intw6Lpc8obW^0`d zYdsmw-4ve8dDjc`wRY#qc`#1y5d=VJ2L!}AM~1BYZF}u20i^-s$7Zrm8A1{!Pa0;^ z5$0QLv1zv-ggSVf?CtDWc5q9dVBvy-f@gx+t5G-_-C@INv)|Jmr#r__ahQRVuSjaf z97$;V8+kwL$>-TJikV>zVM|97&#Xm=NSA6{v@P`5ElP%<#91o#s!IViKh9nFQe%Ep zFCrtd!tw)Cf{@@qsd+33aD-JlkJJ#>RwHYA5Eu#cjWJ4M%5SKGgS~VfcG6gwWZq}B z5s<`Deu=5k;$%{zyX!S+KpRDDTz7ua==l?n%(gnx*c+d=qiGo-)E=a4+Lj)fMM=ab zbb}Gx<-p4Ga_lx8TXtr`hKt~R0Ui8Xb(1p=>kMV|4SmH+5#S-hX3&Eju9A;QQqwnT zysLFo-0kbc>x5|PwG^xbn6m7`*2_Z;I<$68OR%n67A2WZ;KqHG z%9ykjxTmi5oKp)M(8w%^~rnk!2ovd z!s2B<$`^n9(|5M}OTj1=g#pccSQFciDg5w_s#fy6wyN(NPRBUQu6aj0K_5-b{VS?I zxQ%LOo*nBx8h;h+mWFk_^1_!&lq>+*jF4DHmLtys4~Vh)9ngtgnR1v&r&8jkKYDT- z^n8E4Gy5qdRkY=SFQv*JOxlHbzhfT&^0rj)pQ|jjz+{hR2rrhiE8qMKZ55e`Uz_}G zd48hsyCDBQG)(dHdqNLp!x01RGv;{Rc8cBE;KYN1u7L#4vMrdw(6{0c{(-XQ!LkX0 ziWChO6&>`KSSco0W`7afuxq9_&>@+x#xRPxO1Mv;6k>if0yO`%1m`3zNyozb@K%pm zucR$mhJ7x1U$cNaMQ+_5yE)Tc=Y8eNiOu-DBmDHd(+7Ru-(2H&{aW#o=G;%Fi@d%j zb4?Sf5Ad&$A2)e0gpt$WG>+By+|RxnJ7@mZ)K?^VOBw<*YErp)giyx<*-%ebS}4h| zc++(o$(Y?nGD)2MMi~i?4Q+X78Og3qR~bqu$rVlse_`)TQD`m6dwsl5!1B;!l0IAb zG~#48uX_Qpv_J-feC+DtjOJvcK3cp4x2Pmoo%|;+`v;l%*8xy|by|H* z-C|QqD-&W^u;+pXUs=>R8NlJI0Q&ck`^d_XGW9uyy&WGnNbpnqN}gLPFXVe<7K#7G zeJDj5Q;cM!oS`9%{!@m>seP|5B(b&`@wGow3Hob)-|H*HF$;^>EU8NL9AB+pgC_(c z4VIJ?7vs9Hd_}k8?A;~zFUDH6sRer&!q{fU7-qB@XAM=+Dh`TV_bW$tFpV&IOP9CSGTb=xW74OXE;^tv!Ee6I zID7Q7`yS|EvNes_@r2&oTZ!g@PP4oaSGnWo(Ogr<6h@j#-m6*Vtd=({uQ*mc>@E_& zueFQh{EBa>R?6NqzC?W#>??;azpCaXS%S{keMomjZ=)wg%xxehIrOkPP(&)y{uoSK zVnk%33Hysq++Q`&b@2se;QSyv`wM}Udf&n7bRkJ&;=J|YR18J8MCg_~2wU|##lG^{ zmxlAy{^`dzFkbxEX=h^1e)F^_ERPRzT=GjOEc8K;w?`r|vmOf>3g-A%DonCx8TZ)7 zj5QnN`qCU~WBbEu>;U4I0XAW*1>3cE z=rvoESwuA0PF7=t=%~dkHd77Qm7raq^HN+50YM`vm<8>GlE+PTxQh}ehLVN9d8A7Q zXUyFtHCTg^a&Mgl4UPW?YwLsbB1$F;23+?9);Mn? zcWfhfqpO|E2C`!C?QK?dLm2elq-wB@^z6%Z2T=G<3mAA<;j#rO^2=QPMp?*g3z(YA z|LjtAyxm5sVT&yzSW?wvqcwl>UA;+qup+uKw1E-_MBu2#AA@5RSM+^yKSB_P9~LJx zCO9Ub~B{|^WoXIkfMg#V! zotW@c&f0v{{kojrkix>l)V^A+Zo{h!2W6_e?F1)|n^cc~4ZLxgQ{LH8_nqerNlb@* z$Oo~OAUS@Bn6VHO9AGkAgF3I=!F(UauGetIjULn~b64+9+`YR~e_e;G?%nu~)`CRT z4sYDf@%&`C4Icd085&CRRCJ&M&i5nCL-;Y)*xC~mJ+Dd!@yP|)fc3%5=3!wE*svDm>(0!8 zqnWC;AM4)JG~K}K;@^;`(d%QvxB4)QO5o$x5~c}+I=(YUG)1K(PjfqGDYkj6vsYw5 z!Wi$4plDKG)3CD6E}Ju)gVYU=9x|f6EHYZch*hL{X&?!85^lY7$vClza*{y*a;u%eqn+nspzijysLQ){r~7xSqA`p@WJE}UMfjDkL|Ci@JQz1r z(>SZm+XPJ_XaApdyhYirsMJhv;S0tCOxclVKm$231BH-A#NG{6*zj1JHW{z^A#@7&(f z+4)^3eC2y_y>sDl9eIGc9DX2YTUnr}M`PT68{gV`&Gt*$Fk|ih$$hIO&+wjDM=-y> za85xC(ZYi0vV7%$=`xfO-|%XhEA1n76^~MCL%2fp`ves~G)AuwgNtgj!$`IJ*CA>@ z!`Q8JnJ^PNC|zMY^LtiQ#Z+rvZF96v2$!GeI(y_;;u78^d^U8Vz%!0>rjMHj#L6*} z(&qs&GhD5*bH0fV2Nd&vIMx&`f5!*`Jm&jbQNIQc<#4>wwUEvc={&F zE}AJCm*}_9Y@duaA>_)OUzk=Czj7yord~`kd*&#lD|-xP;QoN5?v!F0R;Cb-)v(Wo z#9b+#`OcE`i1ji7t0E%TPbzG78{!5H!;HXxWvLtN8TBR%(-iMLcO_#(`q@r^fWhET z^{{ApDg(f`-k!ed<}_JusSiY}i|@ zePw69Hr8RW=2%yT7t25|=`CF6UFK}KgQ^azYJ!B~@Mr->up}@=L#8jXOhpmX7lvHp!qNNb zq^hpKv|LT2rEJQ(*-DG!a;$Mj7}hj?=*q;s;a$sf!hurtF!^ohmyd0xnaSNag?JuRvMF$4zt~ z8(pyQ@Fl!lW&%%=j(gvcoGgmjjOE~ECZ8d#Gy?cLW!&9aUk2l0moEFO(Fn7%xEp#x zm1UavHM)aoLK;blqCsQl+-;GIE{6KPPiX{N@tu>R&hq@z85gK0YSC0odbFOL88N6>Rkl z0?HXe8~3i{DkN;8*<~N+MF1@#2P$W2`-R{P-B|xGMzVWcRo-)w0$RUF&27zw9 z2~1E{FG^`on0N% z0*qp+SAn-ZHf&J2l)lsF$10S#I6H|k$)iaEIvbEf#zcW+@C5;2X>>} zFkbk;SJWO}hOGAdtKX5-`qwpv;Tqp!nMdnEHU+D(9qzQKSW~!VJ2%9>RJ{kEZNUr| zN;kTsxbfnXb+VhHD1^oRRJ;a8DC$iuu^T&Z#4W!!&!NbAp*e7SS}<-Zcm^1wRlVn~ z@Vi+?{G<`0FVZlDG@n)6xDs*J-`Vb#jGeE=?S7K5WrG_#WURM^AXqJ~f6*T9E0bNl zl>ZK5w8f@9_(M7VxD;_m89-U8Z*j6;Voz^NP?Q8!4UzqYv%O#HZfyZODp|!Zu&02} z2`caoEV8|_f2EL^L%e zMa`#6C9>E@ThT9!pG8Gx*0@PxrzYF-Os}ZzvWkW6v!O)2og-1!er$1V^d2JI7s(g9 z+okL;3vOeF{m?U0lIq!ENIasxVoqaP1@(wE`XZ?6?#Q3>=zFIs^ z=N$bywH7-fW%a0b<^7q}J(aj0VMo^Xt-ixnR)_=#)#@M6u{YJ5jwmOk3RCY8=1>LsAs}$)mU-t#}dh4r7mSI@fRB^i6#oHCvci7eb(#@cn=kNJK^mj>W_xh zJRv%zR+2Xf_VrH-=**pkqjtw+ildiq3`oB-@VS4OKFoN2<&4rrq3+`Ru~OMLD(iEJ z=G_PR{w4}R=D#{91>gPvUI=|JwObU_FqO0Me-#zTf>89rB@UhNB0MY{zH|Nl(k(>p zTwzE~Xb8W>pFfzfB|u#i>sxwZ7%O42B60>~y}|HMDx?Q(pxr-R9=E(f_#1;O?(jFY z-H~9#)RW)n&Zw#`@Fg-zTz&y=-v6hv0mD~c*1-$O35j|BLvI7-<$-r05dKk#86^!m z1q1QK&-_B;4`>kyQ>7*4St{z91ywUh`wO`{UHp}q8s-lI zex5~NzOsK=wLzPH>y6Pv?9`5K!R}GB>nY>R+X#8#e`=XwMM^#$D}x{6{L3M8g$^Po zV!T27;!QT{Tvpb+pXSTXj^(N+*n5N#_QHJj(6F&NAG;HGA#r)VCE3TV0OKan9jyVfFA|F~7WKO``3o!`IdqW+dZ@&1z%cn&2R z{5P30A{Yo)CI`VFARqt?1_Su`_yF9`*Rv7|0Ol9K`1Aee`h#)v0s%Zc;J@v^v1gl) zoBRLT|6Kox^CzAW2A8a}lZUjWySbaKi-)rt00e%{ua3c`>8#~sYwm1e2>|^W5_Jr* zXVujo@z&o(LjGB#FqiBfQP$rc|LVZ^FAmQM03Zl_pIn^qIpuRKE_F+HXHPeCOLqW> z4=zaI|6g!5I5{N)96<5nzuhO`{}b+C^8(>)l(YzZaAiu6@Shsg{~S@u+4H#s@W1-~ z+4$!u&q6az9~Vn5DN_$q2WM+8Ra5IfRRQ4*l+FkOaC)jG25txl1m@%A=K%6C1A)w} z{}Se{o23=TbLW9@aSmdPXJr}?2m*m1yl@F>K9uKOKmoY_-zy$oFuaAD0rq_NS@-tm z18@TTa|5rybEW?SdyWGB4?iG?_gNnIKm0&I5Z^Oj|AvA2p1b^S7@xp1i$LXP^j{JK zpC{(u9G?sPkIf)H;C~bY;^*atJJOKDxPe^i&d$#S|4ju3mx7a(GvLo5{|u3WG{68L zECJ-<0ZD*md1bg|!F*r{4-g_Fz$3*EDQ1Xx-?mY<(bLV{bCA1uHN5&-c_19>Dco_TWjFm?0znpw;tBmln!tiSjt=kQfp21IUP12UaDCWIdV{s(2;eB93? z|36#I%`fmDRDeMcFg%8X0>%x(`2TMcK-d)YzjX4yv`LTezqg4?)y>lDS!Vg%pFe$I qlyS0>Fn&F>`!Z-`&{)q|Le=z8^D!O#TP7e#_hX+h%f{qApvr8cSV^x zfV|RH5f1KTlL@Co3W+>`f&eClH>R6VSRLs7mh`-t$3huX*>c&}o8Hs}wWS9{Y?rV%V(&iEFkj#=qCtG>Fd$yVu&F+3ddwi(C0n3d0m7g=x2& z_H67eq~of6PJji8&7s*npM|E=CTh+EDk3Sq2%LLSt#8Op65r+Ez5;H8aRVyjD~hVE zaoTlneFA7A?nS0uHyAVAD&)Im%|}SI$3O@3%X$cg6UH?0(ZvO}m^5KXGdJj_^=OfS zR+cSy4kquL(9OXZ?$MkG-W%kLhlWy8@l6%wy}zL|k~G-;tfcg)q{K zfz3U<`wO@+=!l-UA~0P#E=QB>`GG9Qyo>jDyf-Fz-=8o=eh-Z@x?RmNe8A>PzYI)0#IBvvUD&$G4m`4ZWn1de9!lY#Ta+s1tkp?LJ6KlgKRpNT=RPtAB1Lg2l& zXHbk1;XJd!CkFplItwz>n3?3I!vlfd);7W)sr%ZyRQ9X$QY+-E&Yr`!%3G)jP?qQBCJPLskaP7Kp z)BOtBECl{id_Af0QYQn%@pO%HW3h?ugUY19>x%or8V|C;ziaf#^LwNC_}hC@)Ea^B_}0%L07;-6Z*ZN5-vM}aK6Das3JW>>u!B2B@)(q4pha5li?z>+IG#Wu#sOWBk-$$dOCEF zK{DtYTnp9S8uLps&uNlpBXd>{40~f8GVKz^*K+ufAa(TBt1!n>1-Nv%t|yH7drW+I z9h=>DA>(=F^H@tAl_|dNRq0U?6Jwf0?y+SQS9priH zI!y!>5f-K?ndDP$_^SEaWfc}|x{ju2V$m@9)|gX%S*0=h<)^OnXFg zg41!`W!`$7fbPtW*~>fuR7C0A1w$t_navEHrMC|pEyldw)78*rHiI(LYfeRrxelz| z)WTP+Cv#uZ?MyBD&A-|UmiK8IaS5_JeB_57h+X%4)*5;z$2DFljQl=_G_MD9HtH~O z{o$?Uh+?zbA$a-tkQC1-BpA|_XA@GBVr{WHiM)lH5qw41Tf2)6to1q zhwrDy&ns+3+IUmu0d^H~VPTmUh>3Y-vT3YGO?jVKgX0(s*$5}C^L)JuRjB-RgWDGO z@U=OcU)#13jz_t0Xl%J(F!?FEcqT5qOHbETZ@u)x4e>X@Dj#3py!3mC%dXU$MM}!e zctV=Z&*=0n<}K?APmX(H3=)}vI?|=C%m^!09x%4(w;8fZ2BsPF&CSVo-p|=@cf~4L zfC+oBA}NrCf^Z!zDuT>5`yN^wYskdt$Pb#T{Ki14E!)Wtk#FBBZS6tI3i&A_g7ukN zmXBWCX$O9OOx`JJ44_0bIX-*K)8*E3XDXmTSID8fraB{4x@8z5Ol;Juy}M z>)AD`;xTmu0-%YUt`bOyNi{54I{1i6cGQ12Vq^j1J2*Q+rp6AeMj}7|P@zfaDc}rb zioQp<%&qVnw4^6R@~!DzFp;J37Kmp1%Ez;Me^Uc;-KF%)9qVI06U{|9k4v)4kmZ=d zkIDGA14ptYCd#n$+(MEEFYwOKH_Ic22YHlKuCsP#$$%b1%pIS^(o4|*Jo;_PQM$ra zlk4^ePIR1Q(`fLT_^IQfcCEv>FOuw)c4oaWa;QQe9u$(NS{Z8}~2$h|<#Bv;g zn(MG4y$2BI!mcQy_*v0AJ$FiR0@3Hn&go8?fHrc8GePv#pfmS1RnGRM7i(A$eRyhz z#AN=cEa!c5VOjyXj1;o|qtc6{)<$JK^!>CTK}hXndK3!Qugs&A%)7~As4=VfOHHFA z=!I28lh5VpJY85srPb+faG;N*l)4RQ`tG;B6TqET%K~=W4m2_Ot$jepx+J%ZoHiGd z@n)cZYqF#%C*{k6R6xDK_|hXvI~6fuWzHQ^GR)l~bMS*VV?-M zhz;wq4rwiEOM+L)?Gg*Yc{&{pbJoBFDxhRV6fKig7c{T(5~()SFJZpkF^lz7>y2V3 zuU4n4b?2A6-AsWZT*jJeTB*cDrG7Q}YqT$#&(RsiEV0Qw9niylzC&bNC+8EXVNt_l z?dmN>D(3NT9v3a$*v^-76vDLGmSo85Uz|q|}aA9jxX$4fu zlEQjaYzI4^vrSTa_o?I$$qJ+0EM3P`x`$XC+581vk`xV49s0;ahz3y(4HdKIZ@O`n z;zf_s(saza`CQWI%l@gbFlaoVrl_LeBrJ*jx-NObr1AQ~u;tfrFu&MY>qhhCy=XMt zDnbxAi;lkjb-lIVv)J#{atXS63Mg#s8C{z|?vd z6Z2KWE13E8`+R2x#|2M~f6p79tj-_dbiRp>5e0quidvKE>$0?uJraqCCIM22hK73X z+RUHSZrz)GmsGd@Tz4S~k9cUd=>>n^>Qw@fiT_5od6%|!S6XcWuo)>wz7o8|HK@9{ ztzD+%A?c~`);yR`(xm#$`h!&MI*~eh6AYWY3I6l#!+=emtq`POW$Gr^8iS{YM?X3{ zD>Dm|7*g16M%{IR)X>$^WdUexO0AXAJJ;7f_d;lZ<6yI>YP8zd4S=inQ4&{DcZ{7Z54P<16 znM7gg4F(yR+B7I_cyfp1LbMWhexEdy3IYtw4@MfVX66?fJyQrn2a;u2}) zKYfnusk^+-XC~6wWCHm1151ycTKtz8+7hq%cNkLmueR4DSp^-BmCr2Yep@e-lm4(# zvNCd>Y$T~$-J>wp!D)_VHN^qw476`OYxF4;ZJ%+OuDipb^;@A5fyLmGu{`Xuezxib z@+%b0(fiUy>Dyf`>DT(o$L#_3!aDE<*8eeXqjBNUx;dERWdVTgmKh+0yLKRMEgPWC z;ieVvH3shqhco5IA}y#S0LNJ>;G??22MPr3VVGus2aA#zC=EyX)4R3bjAd*7rL8a3 zN9-sN3`%FJFmH-0t1aqlK3q0%p%?BN*o2h}a?eh?m_jdDKn7G2Li$w(l!mMGb&^~U zhoO*0)aPS^IJc_%~aPj3VXe6Zpy9s&{XwgB$Zhv zR%_)k2gmKac$V8{2UlUe>Il{n!V*$jJDo{<-pX&EofWLZS(hq<;=?<$zt-m)cG(6y z!bZ9BxPmd(nf1&9BuWDy@hn-3ibpV*C=V-!1t-fI5)Y7k2r^`K?Ajw_jUz8AeP(R@ z0{ybso`q9Enu`%vhFN2Yo>BM{gqZO@O*o5Y)@@Ser&;Co1qS;HMd~nCh0)BeBvat6e6xZCe<{;0dhi+GXhHzyqFY0a6*S~Mm!^R=XZ`Y=dz1N-1sTFxw z1M-HT$G!_Yhq*>AY3+-Vvj+IS%rjp-rL2Tl3lBs>czNy2>CTKUFK6D}=co^(RafI) zMOOzvj7IeBj{*X-0jtqtW06%CV{%6|9{A}lgVD6F$;bngOv?myb<1^(p3^WNXjO?) zQxn@Mg$lfaX}lPZ={9j^HeizN-47dN7itieB&^6+8=nIrSX0cf#Kt2x5Ohz z+sqSfiQ72385vOoEKj+E1f_fL&i*_H`@GRX(SzigIpiyLP^C{`a57YY#M`x@ruXu-RA7xbh1~s zfZ~-ji&sY+oFq_SNP#&xnEfOeIq!ZYNRW%F1+4O zvGC8?ka>QPSKRoyw)K#uvZR1pJaCpA4G@9-s3b2FjD-XB6)$PjPevs$+#UVl@=3;6 zm}NhPt#GPh!y0$$;)9`*+)GRKP0fJK?)y=d+Jmc=;a@|_SUQ>VVmdvqZ`l$GE^K~l zFL}hGGZUk+MQ+$;)EwBaWb?%ZS%m#yi=ci%MoDe2-7R|=Lpe@8xUFh{Hl=hD06IM5 zc4ZgCk_{OZV<#|O`z6m^2)D@dEgnoI5 zI#$>#aEBL2B#MNs!t*z}ri$V<%&1K-nFmE7#9IelV^O1{PlUEQ293$oPgFjVR$l?7 z^2_1XgqFL}oxVKGROX>ompMNh07sbl?_uq1ypc>|&u5(p*Dl=2WR8`OUksI6t=W27 z@)^n1(=xv)&=XTk9I(ZNFs{7F4kcEf?=n5X4{=Y0ChtrMJeGc}kVM$VRB>O@p8e{x zJf@XxskH&;rCKALQEl(51zAMSEL*M<(wBJ*{(Z`(!;X1aEsfpzgL6KsEuh{W^`aqA zm+@H`P5Dwngy~Xf1 z&ade%_?OQ41+M4&PCrQ+nV@`5F1ogi)*G(&Brf?(e)5gBDAza-IS;uF0lfY@vmbfV zuethy0ru{NXZlt?KI5y+05El=!(h~%U|d$WT_@(R-(1qHa@Jhyg&u9)?UTbM-d8Pb<|oy~_vibd9 z$Gz9a=+(4SI-IAKej1)FC|bU(`b^~gqJ}8G)cx}EA;zD4x-O?1lY>fdcxo(_LGz$Q zT1wn$nf6>wqC4X|{~)&8=}ZLUK^Y^u)Dub^~%zVFG1L(hQ*Ui~ZH6(0Y&k$V36m!Uohb34w4%5%yFyngxh*%Hj7 zr_?`={OH=GC#5osC5zXOF8r?;#mhsH-}gi7QdDBm=q7OriXA{RWop+`e`|^po#U#1 zuSvAtvsYgpwJ}pU{z$R+ou7OZ1K#?WXC0=8`|{nz^ePye7!<+$Tu;;nyMRAgMY}?Z zi;IN5i(4x~2ki1hKsETKgcge;X>=irCKH7gIFhVUN1dLukX~v>Tru-Oe~AM3)16$i zD9!?YLdf2^lK@WS>teLZZQ6T6LU1?;cx97MbU}CN`6{b39i>CiYR($WVsevL7iPAkkA5%S{gInjS z>z5n89ei^#R_@Qs=lkigAAWpFPubvo*Zq|utwy|-4RsmCyenl?#l}u8YwK+C=!B4| zQ)TV#meIMUDx6u8lO+l-2e$glPe+xzy6er)?~nl;?5Z*p5eMn)i|@M#35_q+x-i5u z5(d||_fl~T%#2r_=4w|U7~2R0v5G{TtmiO9SUK0n!&A_RL(Q{7p6s}v%uY=2dJC8i z9M-rF7=J(6$DO{yH?7RuZr%=vaO6Gj$>DIlA zbdQI9!Rxfy{jZ4r!20{v{QqDRK=W`JVTqu_xuAeCh#%?X=q`(LGk3Ldc6V|GLm_|h z}n9#p0#@*G_#vbMR2fn6__f0-AJFeF3<0hLBd2@AudMWo~qQbICN3DCb;bp9I; z_y)7AlevsF%G}P)!vV|>fy*HUAP6K>NC+W|6qZ6lWo6(9DLDaQIcZs01mfmU-u$vL zb#?c0MOlL2aEK5(8Tt@Q2we*W-^4Qobt8lR75*RN-{lDvMwZi?xDj1KsJumg*s1phyH|M%!_P72Bs v{D;{eIIMDx7IK_kHkMjWPWIZ)7N+hfFhT%^_!qoTs1RHj#L6o71PS^tLeYt} diff --git a/wrolpi/common.py b/wrolpi/common.py index adc073e7..e4c9e6c2 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -975,7 +975,7 @@ def limit_concurrent(limit: int, throw: bool = False): return wrapper -def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> List[str]: +def truncate_object_bytes(obj: Union[List[str], str, None], maximum_bytes: int) -> Union[List[str], str]: """ Shorten an object. This is useful when inserting something into a tsvector. @@ -1082,3 +1082,41 @@ def cancelable_wrapper(func: callable): return wrapped + +WHITESPACE_SPLITTER = re.compile(r'\s+') +TAB = re.compile(r'\t+') + + +def split_lines_by_length(text: str, max_line_length: int = 38) -> str: + """ + Break up any long lines along word boundaries. + + Newlines are preserved, but whitspace (spaces/tabs) is replaced with one space. + + @param max_line_length: The most amount of characters a line will contain. 38 is default for mobile vertical. + + >>> split_lines_by_length('Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do') + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do' + """ + if not text: + return text + + text = TAB.sub(' ', text) + new_text = '' + for line in text.splitlines(): + if len(line) > max_line_length: + # This line is too long, break it up. + new_line = '' + words = WHITESPACE_SPLITTER.split(line) + while words: + word = words.pop(0) + if len(possible_line := f'{new_line} {word}') <= max_line_length: + new_line = possible_line + else: + # New word makes the line too long, start a new line. + new_text += f'\n{new_line.strip()}' + new_line = word + new_text += f'\n{new_line}' + else: + new_text += f'\n{line}' + return new_text.lstrip('\n') diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 9fd0819e..7b23a581 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -1,6 +1,5 @@ import dataclasses import pathlib -import re from abc import ABC from collections import defaultdict from typing import List, Type, Tuple, Generator @@ -13,7 +12,7 @@ try: except ImportError: PdfReader = None -from wrolpi.common import logger, truncate_generator_bytes +from wrolpi.common import logger, truncate_generator_bytes, split_lines_by_length, truncate_object_bytes logger = logger.getChild(__name__) @@ -91,6 +90,7 @@ class ZipIndexer(Indexer, ABC): try: with ZipFile(path, 'r') as zip_: file_names = [' '.join(split_file_name_words(pathlib.Path(name).name)) for name in zip_.namelist()] + file_names = truncate_object_bytes(file_names, FILE_MAX_TEXT_SIZE) return file_names except Exception as e: logger.error(f'Unable to get information from zip: {path}', exc_info=e) @@ -98,9 +98,6 @@ class ZipIndexer(Indexer, ABC): raise -WORD_SPLITTER = re.compile(r"([\w’'-]+)") - - @register_indexer('text/plain') class TextIndexer(Indexer, ABC): """Handles plain text files. @@ -115,11 +112,12 @@ class TextIndexer(Indexer, ABC): return a, None, None, words @staticmethod - def get_words(path: pathlib.Path) -> List[str]: + def get_words(path: pathlib.Path) -> str: """Read the words from a file.""" # TODO this only supports English. contents = path.read_text() - words = WORD_SPLITTER.findall(contents) + contents = truncate_object_bytes(contents, FILE_MAX_TEXT_SIZE) + words = split_lines_by_length(contents) return words @@ -153,6 +151,8 @@ class PDFIndexer(Indexer, ABC): try: # PDFs are complex, don't fail to create title index if text extraction fails. words = '\n'.join(truncate_generator_bytes(cls.get_words(reader, path), FILE_MAX_TEXT_SIZE)) + words = truncate_object_bytes(words, FILE_MAX_TEXT_SIZE) + words = split_lines_by_length(words) except Exception as e: logger.error(f'Failed to index {path}', exc_info=e) if PYTEST: diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 4ef5a06e..000418da 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -103,11 +103,7 @@ class File(ModelHelper, Base): indexer = self.indexer # Only read the contents of the file once. start = now() - a_text, b_text, c_text, d_text = indexer.create_index(self) - self.a_text = truncate_object_bytes(a_text, FILE_MAX_TEXT_SIZE) - self.b_text = truncate_object_bytes(b_text, FILE_MAX_TEXT_SIZE) - self.c_text = truncate_object_bytes(c_text, FILE_MAX_TEXT_SIZE) - self.d_text = truncate_object_bytes(d_text, FILE_MAX_TEXT_SIZE) + self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self) if (total_seconds := (now() - start).total_seconds()) > 1: logger.info(f'Indexing {self.path} took {total_seconds} seconds') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index d4859157..5b681ae1 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -428,7 +428,7 @@ async def test_large_text_indexer(test_session, make_files_structure): large_file: File = test_session.query(File).one() assert len(large_file.d_text) < large.stat().st_size - assert len(large_file.d_text) == 46_117 + assert len(large_file.d_text) == 90_072 def test_glob_shared_stem(make_files_structure): @@ -499,8 +499,23 @@ def test_pdf_indexer(example_pdf): assert b_text == 'roland' # The parsed file name. assert c_text == ['example', 'pdf'] - # Both pages of the example PDF are extracted. - assert d_text == 'Page one\nPage two' + # All pages are extracted. Text is formatted to fit on vertical screen. + assert d_text == 'Page one\n' \ + 'Page two\n' \ + 'Lorem ipsum dolor sit amet,\n' \ + 'consectetur adipiscing elit, sed do\n' \ + 'eiusmod tempor incididunt ut labore et\n' \ + '\n' \ + 'dolore magna aliqua. Ut enim ad minim\n' \ + 'veniam, quis nostrud exercitation\n' \ + 'ullamco laboris nisi ut \n' \ + 'aliquip ex ea commodo consequat. Duis\n' \ + 'aute irure dolor in reprehenderit in\n' \ + 'voluptate velit esse cillum \n' \ + 'dolore eu fugiat nulla pariatur.\n' \ + 'Excepteur sint occaecat cupidatat non\n' \ + 'proident, sunt in culpa qui officia \n' \ + 'deserunt mollit anim id est laborum.' def test_pdf_indexer_max_size(example_pdf): diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 57e5780d..a03fe3ef 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -17,8 +17,6 @@ import pytz import wrolpi.vars from wrolpi import common from wrolpi.common import cum_timer, TIMERS, print_timer, limit_concurrent, run_after -from wrolpi.dates import now -from wrolpi.errors import InvalidTimezone from wrolpi.test.common import build_test_directories @@ -567,3 +565,34 @@ async def test_run_after(): # Sleep so "after" will run. await asyncio.sleep(0) assert count.value == 2, 'Counter did not run after' + + +@pytest.mark.parametrize( + 'max_line_length,text,expected', [ + (38, '', ''), + (38, 'Lorem', 'Lorem'), + ( + 38, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do', + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do' + ), + ( + 10, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do', + 'Lorem\nipsum\ndolor sit\namet,\nconsectetur\nadipiscing\nelit, sed\ndo' + ), + ( + 20, + 'Lorem\nipsum\ndolor sit amet,\t\tconsectetur\n\nadipiscing elit, sed do', + 'Lorem\nipsum\ndolor sit amet,\nconsectetur\n\nadipiscing elit,\nsed do' + ), + ( + 38, + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + 'Lorem ipsum dolor sit amet,\nconsectetur adipiscing elit, sed do\neiusmod tempor incididunt ut labore et\ndolore magna aliqua.' + ), + (38, 'long-word-which-should-not-be-split-up', 'long-word-which-should-not-be-split-up'), + ] +) +def test_split_lines_by_length(max_line_length, text, expected): + assert common.split_lines_by_length(text, max_line_length=max_line_length) == expected