mirror of
https://github.com/jojo2357/kiwix-zim-updater.git
synced 2026-10-25 18:40:03 -04:00
Support new* kiwix file date format (#60)
* update sorting and regexes * improve regex validation and quote escaping * comment * more safety Co-authored-by: Jailford <egk10@cam.ac.uk> --------- Co-authored-by: Jailford <egk10@cam.ac.uk>
This commit is contained in:
jojo2357
GitHub
parent
1345a1598b
commit
b751227537
1 changed files with 81 additions and 25 deletions
|
|
@ -2,10 +2,10 @@
|
|||
|
||||
# Kiwix-Zim-Updater. A script to update your kiwix library
|
||||
# Copyright (C) 2022 DocDrydenn
|
||||
# Copyright (C) 2023 jojo2357
|
||||
# Copyright (C) 2023-2026 jojo2357
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
VER="3.4"
|
||||
VER="3.5"
|
||||
|
||||
# This array will contain all of the local zims, with the file extension
|
||||
LocalZIMArray=()
|
||||
|
|
@ -60,6 +60,12 @@ BLUE_REGULAR="\033[0;34m"
|
|||
BLUE_BOLD="\033[1;34m"
|
||||
CLEAR="\033[0m"
|
||||
|
||||
MONTH_REGEX="\d{2}[a-z]?"
|
||||
YEAR_REGEX="\d{4}"
|
||||
END_ANCHOR_REGEX="\.zim"
|
||||
|
||||
COMPLETE_ENDING_REGEX="$YEAR_REGEX-$MONTH_REGEX$END_ANCHOR_REGEX"
|
||||
|
||||
# This will ask the api what files it has to offer and store them in arrays
|
||||
master_scrape() {
|
||||
unset RemoteFiles
|
||||
|
|
@ -88,10 +94,26 @@ master_scrape() {
|
|||
RawLibrary=$(grep -i '<link rel' < kiwix-index)
|
||||
fi
|
||||
|
||||
IFS=$'\n' read -r -d '' -a FileSizes < <(echo "$RawLibrary" | grep -ioP '(?<=length=")\d+(?=")')
|
||||
REMOTE_FILE_REGEX="[^/]/\K[\w:\/\-.]+"
|
||||
BASE_REGEX="[^/]/\K[\w:\/\-.]+(?=$COMPLETE_ENDING_REGEX)"
|
||||
REMOTE_PATH_REGEX="^[\w:\/\-.]+"
|
||||
REMOTE_CATEGORY_REGEX="^[^/]+"
|
||||
|
||||
ValidLibrary="$RawLibrary"
|
||||
# ensure that every line matches every regex
|
||||
ValidLibrary="$(echo "$ValidLibrary" | grep -iP '(?<=length=")\d+(?=")')"
|
||||
ValidLibrary="$(echo "$ValidLibrary" | grep -iP "(?<=href=\")[\w:\/\-.]+(?=\.meta4\")")"
|
||||
# we have to add extra here so that we dont have to change the regexes
|
||||
ValidLibrary="$(echo "$ValidLibrary" | grep -iP "href=\"https?://[^/]+$REMOTE_FILE_REGEX")"
|
||||
ValidLibrary="$(echo "$ValidLibrary" | grep -iP "href=\"https?://[^/]+$BASE_REGEX")"
|
||||
# these probably arent gonna be a problem so we will ignore them
|
||||
# ValidLibrary="$(echo "$ValidLibrary" | grep -iP "$REMOTE_PATH_REGEX")"
|
||||
# ValidLibrary="$(echo "$ValidLibrary" | grep -iP "$REMOTE_CATEGORY_REGEX")"
|
||||
|
||||
IFS=$'\n' read -r -d '' -a FileSizes < <(echo "$ValidLibrary" | grep -ioP '(?<=length=")\d+(?=")')
|
||||
unset IFS
|
||||
|
||||
RawLinks=$(echo "$RawLibrary" | grep -ioP "(?<=href=\")[\w:\/\-.]+(?=\.meta4\")")
|
||||
RawLinks=$(echo "$ValidLibrary" | grep -ioP "(?<=href=\")[\w:\/\-.]+(?=\.meta4\")")
|
||||
|
||||
BaseURL=$(echo "$RawLinks" | grep -ioP 'https?://[^/]+' | uniq)
|
||||
|
||||
|
|
@ -108,15 +130,15 @@ master_scrape() {
|
|||
echo -e "${GREEN_REGULAR} ✓ Using $BaseURL as base download directory for metadata"
|
||||
fi
|
||||
|
||||
hrefs=$(echo "$RawLibrary" | grep -ioP "(?<=href=\")[\w:\/\-.]+(?=\.meta4\")" | grep -ioP "$BaseURL\K.*")
|
||||
hrefs=$(echo "$ValidLibrary" | grep -ioP "(?<=href=\")[\w:\/\-.]+(?=\.meta4\")" | grep -ioP "$BaseURL\K.*")
|
||||
|
||||
IFS=$'\n' read -r -d '' -a RemoteFiles < <(echo "$hrefs" | grep -ioP "[^/]/\K[\w:\/\-.]+")
|
||||
IFS=$'\n' read -r -d '' -a RemoteFiles < <(echo "$hrefs" | grep -ioP "$REMOTE_FILE_REGEX")
|
||||
unset IFS
|
||||
IFS=$'\n' read -r -d '' -a Basenames < <(echo "$hrefs" | grep -ioP "[^/]/\K[\w:\/\-.]+(?=\d{4}-\d{2}\.zim)")
|
||||
IFS=$'\n' read -r -d '' -a Basenames < <(echo "$hrefs" | grep -ioP "$BASE_REGEX")
|
||||
unset IFS
|
||||
IFS=$'\n' read -r -d '' -a RemotePaths < <(echo "$hrefs" | grep -ioP "^[\w:\/\-.]+")
|
||||
IFS=$'\n' read -r -d '' -a RemotePaths < <(echo "$hrefs" | grep -ioP "$REMOTE_PATH_REGEX")
|
||||
unset IFS # distinct from above for processing speed reasons
|
||||
IFS=$'\n' read -r -d '' -a RemoteCategory < <(echo "$hrefs" | grep -ioP "^[^/]+")
|
||||
IFS=$'\n' read -r -d '' -a RemoteCategory < <(echo "$hrefs" | grep -ioP "$REMOTE_CATEGORY_REGEX")
|
||||
unset IFS
|
||||
|
||||
if [[ ${#RemoteFiles[@]} -eq 0 ]]; then
|
||||
|
|
@ -128,7 +150,18 @@ master_scrape() {
|
|||
echo "✓ Found ${#RemoteFiles[@]} files online" >> download.log
|
||||
fi
|
||||
|
||||
if [[ ${#RemoteFiles[@]} -ne ${#RemoteFiles[@]} ]] || \
|
||||
[[ ${#RemoteFiles[@]} -ne ${#Basenames[@]} ]] || \
|
||||
[[ ${#RemoteFiles[@]} -ne ${#RemotePaths[@]} ]] || \
|
||||
[[ ${#RemoteFiles[@]} -ne ${#RemoteCategory[@]} ]] || \
|
||||
[[ ${#RemoteFiles[@]} -ne ${#FileSizes[@]} ]] ; then
|
||||
echo -e "${RED_BOLD}CRITICAL ERROR!${CLEAR}"
|
||||
echo -e "${RED}Parsed array lengths did not match. Aborting to prevent data loss${CLEAR}"
|
||||
exit 99
|
||||
fi
|
||||
|
||||
# Housekeeping...
|
||||
unset ValidLibrary
|
||||
unset RawLibrary
|
||||
unset hrefs
|
||||
}
|
||||
|
|
@ -259,14 +292,20 @@ flags() {
|
|||
|
||||
for index in "${!LocalZIMArray[@]}" ; do
|
||||
duplicated=0
|
||||
myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)")
|
||||
myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)")
|
||||
|
||||
# Sorting cannot reliably sort the month string so we need to check it to be sure
|
||||
MyDate="$(echo "${LocalZIMArray[$index]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")"
|
||||
MyYear="$(echo "$MyDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")"
|
||||
MyMonth="$(echo "$MyDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")"
|
||||
|
||||
for scanIndex in "${!LocalZIMArray[@]}"; do
|
||||
if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then
|
||||
if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi
|
||||
else
|
||||
if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi
|
||||
fi
|
||||
scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)")
|
||||
scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)")
|
||||
|
||||
if [[ "$myBasename" == "$scanBasename" ]]; then
|
||||
if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then
|
||||
|
|
@ -274,6 +313,22 @@ flags() {
|
|||
elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then
|
||||
echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log
|
||||
else
|
||||
ScanDate="$(echo "${LocalZIMArray[$scanIndex]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")"
|
||||
ScanYear="$(echo "$ScanDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")"
|
||||
ScanMonth="$(echo "$ScanDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")"
|
||||
|
||||
# this file is not shadowed if it is newer.
|
||||
# this is caused by `ls -1` and `sort` having weird lexicons that make the prefix sort unreliable that 02a can sort between 01 and 02 instead of after 02
|
||||
if [[ $MyYear > $ScanYear ]]; then
|
||||
echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log
|
||||
duplicated=2
|
||||
break
|
||||
elif [[ $MyYear == $ScanYear ]] && [[ $MyMonth > $ScanMonth ]]; then
|
||||
echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log
|
||||
duplicated=2
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log
|
||||
fi
|
||||
duplicated=1
|
||||
|
|
@ -281,6 +336,7 @@ flags() {
|
|||
fi
|
||||
done
|
||||
[[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]'
|
||||
[[ $duplicated -eq 2 ]] && unset -v 'LocalZIMArray[$scanIndex]'
|
||||
done
|
||||
LocalZIMArray=("${LocalZIMArray[@]}")
|
||||
|
||||
|
|
@ -305,7 +361,7 @@ flags() {
|
|||
|
||||
for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s).
|
||||
LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name.
|
||||
filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name.
|
||||
filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=$COMPLETE_ENDING_REGEX$)") # Extract file name.
|
||||
# IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_'
|
||||
|
||||
# Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL
|
||||
|
|
@ -509,13 +565,13 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do
|
|||
|
||||
[[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue
|
||||
|
||||
MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')"
|
||||
MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')"
|
||||
MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')"
|
||||
MatchedDate="$(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")"
|
||||
MatchedYear="$(echo "$MatchedDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")"
|
||||
MatchedMonth="$(echo "$MatchedDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")"
|
||||
|
||||
LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')"
|
||||
LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')"
|
||||
LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')"
|
||||
LocalDate="$(echo "$FileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")"
|
||||
LocalYear="$(echo "$LocalDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")"
|
||||
LocalMonth="$(echo "$LocalDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")"
|
||||
|
||||
FileTooSmall=0
|
||||
[[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1
|
||||
|
|
@ -563,16 +619,16 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do
|
|||
else
|
||||
if [ $FileTooSmall -eq 1 ]; then
|
||||
LocalRequiresDownloadArray+=(0)
|
||||
[[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}"
|
||||
[[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}"
|
||||
[[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}"
|
||||
[[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}"
|
||||
elif [ $FileTooLarge -eq 1 ]; then
|
||||
LocalRequiresDownloadArray+=(0)
|
||||
[[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}"
|
||||
[[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}"
|
||||
elif [ "$MatchedYear" -lt "$LocalYear" ]; then
|
||||
[[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}"
|
||||
[[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}"
|
||||
elif [[ $MatchedYear < $LocalYear ]]; then
|
||||
LocalRequiresDownloadArray+=(0)
|
||||
echo " ✗ No new update"
|
||||
elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then
|
||||
elif [[ "$MatchedYear" == "$LocalYear" ]] && { [[ $MatchedMonth < $LocalMonth ]] || [[ "$MatchedMonth" == "$LocalMonth" ]] ; }; then
|
||||
LocalRequiresDownloadArray+=(0)
|
||||
echo " ✗ No new update"
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue