From f58f2177f33ac1b89673e079b9eec5ed2dae9951 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Tue, 27 Feb 2024 01:22:46 -0700 Subject: [PATCH 1/9] Make sure index is non-empty --- kiwix-zim-updater.sh | 2 +- kiwix-zim.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 7a6c717..9d15ba6 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -64,7 +64,7 @@ master_scrape() { indexIsValid=0 else indexDate="$(head -1 "kiwix-index")" - if [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then + if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then indexIsValid=0 fi fi diff --git a/kiwix-zim.sh b/kiwix-zim.sh index 7a6c717..9d15ba6 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -64,7 +64,7 @@ master_scrape() { indexIsValid=0 else indexDate="$(head -1 "kiwix-index")" - if [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then + if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then indexIsValid=0 fi fi From e8ad68b1df391461bd1125f47ff49ec0fdf64804 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Sat, 23 Mar 2024 10:15:00 -0600 Subject: [PATCH 2/9] Do not take matched files in the past. This happened because the newest version of Gutenberg became un-served --- kiwix-zim-updater.sh | 16 +++++++++++++++- kiwix-zim.sh | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 7a6c717..a5addaa 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -445,6 +445,14 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} + MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" + MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" + MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" + + LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" + LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')" + LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')" + FileTooSmall=0 [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 FileTooLarge=0 @@ -497,10 +505,16 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do LocalRequiresDownloadArray+=(0) [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + elif [[ $MatchedYear -lt $LocalYear ]]; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" + elif [[ $MatchedYear -eq $LocalYear ]] && [[ $MatchedMonth -le $LocalMonth ]]; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" else LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Update found! --> $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" fi fi fi diff --git a/kiwix-zim.sh b/kiwix-zim.sh index 7a6c717..a5addaa 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -445,6 +445,14 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} + MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" + MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" + MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" + + LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" + LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')" + LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')" + FileTooSmall=0 [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 FileTooLarge=0 @@ -497,10 +505,16 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do LocalRequiresDownloadArray+=(0) [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + elif [[ $MatchedYear -lt $LocalYear ]]; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" + elif [[ $MatchedYear -eq $LocalYear ]] && [[ $MatchedMonth -le $LocalMonth ]]; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" else LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Update found! --> $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" fi fi fi From f66ea16a77b5d954f6b03d8895532ad5f723a7cb Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Sat, 23 Mar 2024 10:19:49 -0600 Subject: [PATCH 3/9] Update version string --- kiwix-zim-updater.sh | 2 +- kiwix-zim.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index a5addaa..633dcae 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,6 +1,6 @@ #!/bin/bash -VER="3.1" +VER="3.2" # This array will contain all of the local zims, with the file extension LocalZIMArray=() diff --git a/kiwix-zim.sh b/kiwix-zim.sh index a5addaa..633dcae 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -1,6 +1,6 @@ #!/bin/bash -VER="3.1" +VER="3.2" # This array will contain all of the local zims, with the file extension LocalZIMArray=() From db2e337b962b11ac1deba3d653f80aad186a7373 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Sat, 27 Jul 2024 18:37:02 -0600 Subject: [PATCH 4/9] Add simple torrenting --- README.md | 25 +++++--- kiwix-zim-updater.sh | 138 +++++++++++++++++++++++++++++++++---------- kiwix-zim.sh | 138 +++++++++++++++++++++++++++++++++---------- 3 files changed, 228 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index aa17ed8..9cbf0cb 100644 --- a/README.md +++ b/README.md @@ -88,20 +88,27 @@ NOTE: if you are not tracking the `main` branch, the update check will be skippe /full/path/ Full path to ZIM directory - Options: - -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. - -f, --verify-library Verifies that the entire library has the correct checksums as found online. - Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet - -d, --disable-dry-run Dry-Run Override. - *** Caution *** - + Universal Options: -h, --help Show this usage and exit. - -p, --skip-purge Skips purging any replaced ZIMs. -u, --skip-update Skips checking for script updates (very useful for development). + -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day -n , --min-size Minimum ZIM Size to be downloaded. Specify units include M Mi G Gi, etc. See `man numfmt` -x , --max-size Maximum ZIM Size to be downloaded. Specify units include M Mi G Gi, etc. See `man numfmt` + -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums. + + Action Method Options: + -w, --web Downloads zims over http(s). + -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD. + + -f, --verify-library Verifies that the entire library has the correct checksums as found online. + Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet. + Disable this using -S + Web Download Options: + -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. + -p, --skip-purge Skips purging any replaced ZIMs. -l , --location Country Code to prefer mirrors from - -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day + -d, --disable-dry-run Dry-Run Override. + *** Caution *** ``` diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 633dcae..d15b3ba 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,6 +1,6 @@ #!/bin/bash -VER="3.2" +VER="3.3" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -36,8 +36,10 @@ DEBUG=1 # This forces the script to default to "dry-run/simulation mode" MIN_SIZE=0 MAX_SIZE=0 CALCULATE_CHECKSUM=0 +CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 +DOWNLOAD_METHOD=1 # 1: web 2: torrent BaseURL="https://download.kiwix.org/zim/" ZIMPath="" @@ -157,25 +159,34 @@ self_update() { # usage_example - Show Usage and Exit usage_example() { echo 'Usage: ./kiwix-zim-updater.sh /full/path/' - echo + echo '' echo ' /full/path/ Full path to ZIM directory' - echo - echo 'Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - echo + echo '' + echo 'Universal Options:' echo ' -h, --help Show this usage and exit.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -u, --skip-update Skips checking for script updates (very useful for development).' + echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' echo ' -n , --min-size Minimum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' echo ' -x , --max-size Maximum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' + echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' + echo ' ' + echo 'Action Method Options:' + echo ' -w, --web Downloads zims over http(s).' + echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' + echo ' ' + echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' + echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' + echo ' Disable this using -S' + echo '' + echo 'Web Download Options:' + echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' + echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -l , --location Country Code to prefer mirrors from' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' + echo ' -d, --disable-dry-run Dry-Run Override.' + echo ' *** Caution ***' + echo exit 0 } @@ -235,6 +246,8 @@ flags() { if [[ "$myBasename" == "$scanBasename" ]]; then if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log + elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then + echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log else echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log fi @@ -303,6 +316,13 @@ mirror_search() { MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') + + # still run the last wget so that the logic is simple for verification. + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL + return + fi + RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. @@ -351,7 +371,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" + echo "Invlaid country code, falling back to default kiwix behavior" >> download.log fi shift # discard value ;; @@ -372,6 +392,18 @@ while [[ $# -gt 0 ]]; do FORCE_FETCH_INDEX=1 shift ;; + -w | --web) + DOWNLOAD_METHOD=1 + shift + ;; + -t | --torrent) + DOWNLOAD_METHOD=2 + shift + ;; + -S | --no-sha) + CHECKSUM_FILES=0 + shift + ;; *) # We can either parse the arg here, or just tuck it away for safekeeping POSITIONAL_ARGS+=("$1") # save positional arg @@ -384,6 +416,10 @@ set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped e clear # Clear screen +if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then + echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" +fi + # Display Header echo "==========================================" echo " kiwix-zim-updater" @@ -440,11 +476,14 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue + MatchingSize=${FileSizes[$RemoteIndex]} MatchingFileName=${RemoteFiles[$RemoteIndex]} MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue + MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" @@ -460,6 +499,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do FileSizeAcceptable=0 [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 + if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) @@ -553,13 +593,29 @@ if [ $AnyDownloads -eq 1 ]; then NewZIMPath=$ZIMPath$NewZIM fi + FilePath=$ZIMPath$NewZIM # Set destination path with file name LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name + TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name RequiresDownload=0 if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + if [[ -f $NewZIM ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" + echo + elif [[ -f $TorrentFilePath ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" + echo + else + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" + RequiresDownload=1 + fi + elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" echo @@ -583,6 +639,9 @@ if [ $AnyDownloads -eq 1 ]; then # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" RequiresDownload=1 + elif [[ -f $TorrentFilePath ]]; then + echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" + echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log else # actually verify the file echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" @@ -649,26 +708,41 @@ if [ $AnyDownloads -eq 1 ]; then [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log # Here is where we actually download the files and log to the download.log file. if [[ $RequiresDownload -eq 1 ]]; then - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + FilePath="$FilePath.torrent" + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi + + continue + else + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + echo >>download.log + echo "=======================================================================" >>download.log + echo "File : $NewZIM" >>download.log + [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log + [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log + echo >>download.log - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + # Before we actually download, let's just check to see that it isn't already in the folder. + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi fi fi diff --git a/kiwix-zim.sh b/kiwix-zim.sh index 633dcae..d15b3ba 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -1,6 +1,6 @@ #!/bin/bash -VER="3.2" +VER="3.3" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -36,8 +36,10 @@ DEBUG=1 # This forces the script to default to "dry-run/simulation mode" MIN_SIZE=0 MAX_SIZE=0 CALCULATE_CHECKSUM=0 +CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 +DOWNLOAD_METHOD=1 # 1: web 2: torrent BaseURL="https://download.kiwix.org/zim/" ZIMPath="" @@ -157,25 +159,34 @@ self_update() { # usage_example - Show Usage and Exit usage_example() { echo 'Usage: ./kiwix-zim-updater.sh /full/path/' - echo + echo '' echo ' /full/path/ Full path to ZIM directory' - echo - echo 'Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - echo + echo '' + echo 'Universal Options:' echo ' -h, --help Show this usage and exit.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -u, --skip-update Skips checking for script updates (very useful for development).' + echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' echo ' -n , --min-size Minimum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' echo ' -x , --max-size Maximum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' + echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' + echo ' ' + echo 'Action Method Options:' + echo ' -w, --web Downloads zims over http(s).' + echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' + echo ' ' + echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' + echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' + echo ' Disable this using -S' + echo '' + echo 'Web Download Options:' + echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' + echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -l , --location Country Code to prefer mirrors from' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' + echo ' -d, --disable-dry-run Dry-Run Override.' + echo ' *** Caution ***' + echo exit 0 } @@ -235,6 +246,8 @@ flags() { if [[ "$myBasename" == "$scanBasename" ]]; then if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log + elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then + echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log else echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log fi @@ -303,6 +316,13 @@ mirror_search() { MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') + + # still run the last wget so that the logic is simple for verification. + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL + return + fi + RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. @@ -351,7 +371,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" + echo "Invlaid country code, falling back to default kiwix behavior" >> download.log fi shift # discard value ;; @@ -372,6 +392,18 @@ while [[ $# -gt 0 ]]; do FORCE_FETCH_INDEX=1 shift ;; + -w | --web) + DOWNLOAD_METHOD=1 + shift + ;; + -t | --torrent) + DOWNLOAD_METHOD=2 + shift + ;; + -S | --no-sha) + CHECKSUM_FILES=0 + shift + ;; *) # We can either parse the arg here, or just tuck it away for safekeeping POSITIONAL_ARGS+=("$1") # save positional arg @@ -384,6 +416,10 @@ set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped e clear # Clear screen +if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then + echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" +fi + # Display Header echo "==========================================" echo " kiwix-zim-updater" @@ -440,11 +476,14 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue + MatchingSize=${FileSizes[$RemoteIndex]} MatchingFileName=${RemoteFiles[$RemoteIndex]} MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue + MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" @@ -460,6 +499,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do FileSizeAcceptable=0 [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 + if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) @@ -553,13 +593,29 @@ if [ $AnyDownloads -eq 1 ]; then NewZIMPath=$ZIMPath$NewZIM fi + FilePath=$ZIMPath$NewZIM # Set destination path with file name LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name + TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name RequiresDownload=0 if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + if [[ -f $NewZIM ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" + echo + elif [[ -f $TorrentFilePath ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" + echo + else + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" + RequiresDownload=1 + fi + elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" echo @@ -583,6 +639,9 @@ if [ $AnyDownloads -eq 1 ]; then # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" RequiresDownload=1 + elif [[ -f $TorrentFilePath ]]; then + echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" + echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log else # actually verify the file echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" @@ -649,26 +708,41 @@ if [ $AnyDownloads -eq 1 ]; then [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log # Here is where we actually download the files and log to the download.log file. if [[ $RequiresDownload -eq 1 ]]; then - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + FilePath="$FilePath.torrent" + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi + + continue + else + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + echo >>download.log + echo "=======================================================================" >>download.log + echo "File : $NewZIM" >>download.log + [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log + [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log + echo >>download.log - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + # Before we actually download, let's just check to see that it isn't already in the folder. + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi fi fi From 72aa3666fc3e2bff119f101fb62f4e5fddddf29d Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Mon, 5 Aug 2024 18:35:02 -0600 Subject: [PATCH 5/9] Remove False Alarm If the newest zim and torrent both existed, would give strange message --- kiwix-zim-updater.sh | 2 +- kiwix-zim.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index d15b3ba..32ed81b 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -482,7 +482,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && ! [[ -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) || continue MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" diff --git a/kiwix-zim.sh b/kiwix-zim.sh index d15b3ba..32ed81b 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -482,7 +482,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && ! [[ -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) || continue MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" From f5dd3bc50451099a440540a7126d72e9837b8cf7 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Mon, 5 Aug 2024 18:43:56 -0600 Subject: [PATCH 6/9] Oops Dont do that --- kiwix-zim-updater.sh | 2 +- kiwix-zim.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 32ed81b..eaac047 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -482,7 +482,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && ! [[ -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) || continue + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" diff --git a/kiwix-zim.sh b/kiwix-zim.sh index 32ed81b..eaac047 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -482,7 +482,7 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && ! [[ -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) || continue + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" From 7b640c8a0427c71ad811783def0ab4d39cb49f5c Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Wed, 7 Aug 2024 00:08:59 -0600 Subject: [PATCH 7/9] Fix octal bug and readme --- README.md | 14 +++++++------- kiwix-zim-updater.sh | 5 ++--- kiwix-zim.sh | 5 ++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9cbf0cb..051032d 100644 --- a/README.md +++ b/README.md @@ -90,25 +90,25 @@ NOTE: if you are not tracking the `main` branch, the update check will be skippe Universal Options: -h, --help Show this usage and exit. + -d, --disable-dry-run Dry-Run Override. + *** Caution *** -u, --skip-update Skips checking for script updates (very useful for development). -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day -n , --min-size Minimum ZIM Size to be downloaded. Specify units include M Mi G Gi, etc. See `man numfmt` -x , --max-size Maximum ZIM Size to be downloaded. - Specify units include M Mi G Gi, etc. See `man numfmt` - -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums. + Specify units include M Mi G Gi, etc. See `man numfmt` Action Method Options: -w, --web Downloads zims over http(s). -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD. - -f, --verify-library Verifies that the entire library has the correct checksums as found online. - Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet. - Disable this using -S Web Download Options: -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. -p, --skip-purge Skips purging any replaced ZIMs. -l , --location Country Code to prefer mirrors from - -d, --disable-dry-run Dry-Run Override. - *** Caution *** + -f, --verify-library Verifies that the entire library has the correct checksums as found online. + Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet. + Disable this using -S + -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums. ``` diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index eaac047..354ffd8 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -499,7 +499,6 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do FileSizeAcceptable=0 [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 - if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) @@ -545,10 +544,10 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do LocalRequiresDownloadArray+=(0) [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [[ $MatchedYear -lt $LocalYear ]]; then + elif [ "$MatchedYear" -lt "$LocalYear" ]; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" - elif [[ $MatchedYear -eq $LocalYear ]] && [[ $MatchedMonth -le $LocalMonth ]]; then + elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" else diff --git a/kiwix-zim.sh b/kiwix-zim.sh index eaac047..354ffd8 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim.sh @@ -499,7 +499,6 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do FileSizeAcceptable=0 [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 - if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) @@ -545,10 +544,10 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do LocalRequiresDownloadArray+=(0) [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [[ $MatchedYear -lt $LocalYear ]]; then + elif [ "$MatchedYear" -lt "$LocalYear" ]; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" - elif [[ $MatchedYear -eq $LocalYear ]] && [[ $MatchedMonth -le $LocalMonth ]]; then + elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" else From 1345a1598bd1329db645bb69d1dd0b70e1c0c856 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Mon, 25 May 2026 23:37:14 -0700 Subject: [PATCH 8/9] v3.4: Fix library urls (#58) * Remove deprecated kiwix-zim.sh * change shebang * update urls * extract catalog url * correct typos Co-authored-by: 4754442+jbnance@users.noreply.github.com * update readme * remove useless `|&` * add mac disclaimer * add copyright * bump version number * minor bugs * teeny improvements * miniscule changes --- README.md | 8 +- kiwix-zim-updater.sh | 89 +++-- kiwix-zim.sh | 874 ------------------------------------------- 3 files changed, 59 insertions(+), 912 deletions(-) delete mode 100755 kiwix-zim.sh diff --git a/README.md b/README.md index 051032d..e9a0c70 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,7 @@ A script to check `download.kiwix.org` for updates to your local ZIM library. Just pass this script your ZIM directory and away it goes. *(see Usage below)* -Tested on PopOS! 22.04, and should work out of the box on most debian systems, but I have not tested that. - -# DEPRECATION WARNING -`kiwix-zim.sh` has been deprecated in favor of the more descriptive `kiwix-zim-updater.sh`. A hard link from `kiwix-zim.sh` to `kiwix-zim-updater.sh` will be maintained on this repo until at least `2025-01-01T00:00:00Z` for compatability with CRON users. - -***CALLING `kiwix-zim.sh` WILL RESULT IN AN EXIT CODE OF 2 ON SUCCESSFUL EXECUTION. THIS BEHAVIOR IS EXPECTED. CALL `kiwix-zim-updater.sh` INSTEAD TO GET A 0 EXIT CODE*** +Tested on Debian 13, and should work out of the box on most debian systems, but I have not tested that. ## What It Does @@ -58,6 +53,7 @@ Creates `downloads.log` for the following reasons: ## Limitations +- This script relies heavily on `grep`'s `-P` flag for lookarounds and the matcher reset, `\K`. This may cause compatibility problems on macOS - This script is only for ZIM(s) hosted by `download.kiwix.org` due to the file naming standard they use. If you have self-made ZIM(s) or ZIM(s) downloaded from somewhere else, they most likely do not use the same naming standards and will not be processed by this script. - If you have ZIM(s) from `download.kiwix.org`, but you have changed their file names, this script will treat them like the previous limitation explains. - This script does not attempt to update any `library.xml` that may or may not exist/be needed for your install/setup of Kiwix. If needed, you'll need to handle this part on your own. diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 5c6ef15..571b4f7 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,6 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -VER="3.3" +# Kiwix-Zim-Updater. A script to update your kiwix library +# Copyright (C) 2022 DocDrydenn +# Copyright (C) 2023 jojo2357 +# SPDX-License-Identifier: GPL-2.0-only + +VER="3.4" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -8,7 +13,7 @@ LocalZIMArray=() LocalZIMNameArray=() # This array will map the local zim to the index in the remote arrays that contains the same base file name LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop +# This array is a boolean array which remembers if a given local zim should be processed in the download loop LocalRequiresDownloadArray=() # After updating, this array will be used to store hanging locks and to deal with them HangingFileLocks=(); @@ -40,7 +45,9 @@ CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 DOWNLOAD_METHOD=1 # 1: web 2: torrent -BaseURL="https://download.kiwix.org/zim/" +# sort of a default, will be overwritten when the library is parsed. Set here to what it probably will be +BaseURL="https://lb.download.kiwix.org/zim/" +CatalogURL="https://opds.library.kiwix.org/catalog/v2/entries?count=-1" ZIMPath="" RED_REGULAR="\033[0;31m" @@ -72,9 +79,10 @@ master_scrape() { fi if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary - RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" - + # both write the file timestamp to the index file and save all of the links to RawLibrary. + # Lets defer touching the index until we have actually fetched it + RawLibrary="$(wget --show-progress -q -O - "$CatalogURL")" + RawLibrary="$(echo "$RawLibrary" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" echo "$RawLibrary" >> kiwix-index else RawLibrary=$(grep -i '> download.log @@ -371,7 +396,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" >> download.log + echo "Invalid country code, falling back to default kiwix behavior" >> download.log fi shift # discard value ;; @@ -379,7 +404,7 @@ while [[ $# -gt 0 ]]; do CALCULATE_CHECKSUM=1 shift ;; - -f | --verfiy-library) + -f | --verfiy-library | --verify-library) VERIFY_LIBRARY=1 CALCULATE_CHECKSUM=1 shift @@ -707,45 +732,45 @@ if [ $AnyDownloads -eq 1 ]; then [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log # Here is where we actually download the files and log to the download.log file. if [[ $RequiresDownload -eq 1 ]]; then + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + + echo >>download.log + echo "=======================================================================" >>download.log + echo "File : $NewZIM" >>download.log + [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log + [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log + echo >>download.log + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then FilePath="$FilePath.torrent" if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log fi continue else - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log - - # Before we actually download, let's just check to see that it isn't already in the folder. if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log else # New ZIM not found, so we'll go ahead and download it. [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log fi fi fi - echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" + [[ $DEBUG -eq 0 ]] && echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then @@ -778,12 +803,12 @@ if [ $AnyDownloads -eq 1 ]; then echo >> download.log [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later + # in all of these cases, we will not re-purge and will leave the lockfile so we know to resume later if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" - [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" continue fi @@ -799,12 +824,12 @@ if [ $AnyDownloads -eq 1 ]; then if [[ -f "$NewZIMPath" ]]; then # New ZIM found if [[ $DEBUG -eq 0 ]]; then if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully." >>download.log # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM else - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully. Old ZIM purged.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully. Old ZIM purged." >>download.log [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM fi else diff --git a/kiwix-zim.sh b/kiwix-zim.sh deleted file mode 100755 index 5c6ef15..0000000 --- a/kiwix-zim.sh +++ /dev/null @@ -1,874 +0,0 @@ -#!/bin/bash - -VER="3.3" - -# This array will contain all of the local zims, with the file extension -LocalZIMArray=() -# This array will contain all of the local zims, without the file extension -LocalZIMNameArray=() -# This array will map the local zim to the index in the remote arrays that contains the same base file name -LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop -LocalRequiresDownloadArray=() -# After updating, this array will be used to store hanging locks and to deal with them -HangingFileLocks=(); - -# This array stores the file names that kiwix has to offer, with .zim extensions -RemoteFiles=() -# Ditto, without the YYYY-MM (note a trailing _) -Basenames=() -# Contains the absolute path to this file, from /zims/ -RemotePaths=() -# Contains the folder this file is in relative to /zims/ -RemoteCategory=() - -# Set Script Strings -SCRIPT="$(readlink -f "$0")" -SCRIPTFILE="$(basename "$SCRIPT")" -SCRIPTPATH="$(dirname "$SCRIPT")" -CALLEDSCRIPTNAME="$0" -CALLEDSCRIPTFILE="$(basename "$CALLEDSCRIPTNAME")" -CALLEDSCRIPTPATH="$(dirname "$CALLEDSCRIPTNAME")" -ARGS=("$@") -BRANCH="main" -SKIP_UPDATE=0 -DEBUG=1 # This forces the script to default to "dry-run/simulation mode" -MIN_SIZE=0 -MAX_SIZE=0 -CALCULATE_CHECKSUM=0 -CHECKSUM_FILES=1 -VERIFY_LIBRARY=0 -FORCE_FETCH_INDEX=0 -DOWNLOAD_METHOD=1 # 1: web 2: torrent -BaseURL="https://download.kiwix.org/zim/" -ZIMPath="" - -RED_REGULAR="\033[0;31m" -RED_BOLD="\033[1;31m" -YELLOW_REGULAR="\033[0;33m" -YELLOW_BOLD="\033[1;33m" -GREEN_REGULAR="\033[0;32m" -GREEN_BOLD="\033[1;32m" -BLUE_REGULAR="\033[0;34m" -BLUE_BOLD="\033[1;34m" -CLEAR="\033[0m" - -# This will ask the api what files it has to offer and store them in arrays -master_scrape() { - unset RemoteFiles - unset Basenames - unset RemotePaths - unset RemoteCategory - - indexIsValid=1 - - if [[ ! -f kiwix-index ]]; then - indexIsValid=0 - else - indexDate="$(head -1 "kiwix-index")" - if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then - indexIsValid=0 - fi - fi - - if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary - RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" - - echo "$RawLibrary" >> kiwix-index - else - RawLibrary=$(grep -i '> download.log - exit 0 - else - echo -e "${GREEN_BOLD} ✓ Found ${#RemoteFiles[@]} files online" - echo "✓ Found ${#RemoteFiles[@]} files online" >> download.log - fi - - # Housekeeping... - unset RawLibrary - unset hrefs -} - -# self_update - Script Self-Update Function -self_update() { - echo -e "${YELLOW_BOLD}1. Checking for Script Updates...${CLEAR}" - echo "1. Checking for Script Updates..." >> download.log - echo - # Check if script path is a git clone. - # If true, then check for update. - # If false, skip self-update check/funciton. - if [ $SKIP_UPDATE -eq 1 ]; then - echo -e "${YELLOW_REGULAR} Check Skipped${CLEAR}" - echo "Check Skipped" >> download.log - elif [[ -d "$SCRIPTPATH/.git" ]]; then - echo -e "${GREEN_BOLD} ✓ Git Clone Detected: Checking Script Version...${CLEAR}" - echo "✓ Git Clone Detected: Checking Script Version..." >> download.log - cd "$SCRIPTPATH" || exit 1 - [[ $(timeout 1s git rev-parse --abbrev-ref HEAD) != "$BRANCH" ]] && echo -e "${YELLOW_BOLD} You appear to be on a different branch so I will assume you are developing and do not want an update${CLEAR}" && echo && return - timeout 1s git fetch --quiet - timeout 1s git diff --quiet --exit-code "origin/$BRANCH" "$SCRIPTFILE" - [ $? -eq 1 ] && { - echo -e "${RED_REGULAR} ✗ Version: Mismatched${CLEAR}" - echo "✗ Version: Mismatched" >> download.log - echo - echo -e "${YELLOW_BOLD}1a. Fetching Update...${CLEAR}" - echo "1a. Fetching Update..." >> download.log - echo - if [ -n "$(git status --porcelain)" ]; then - git stash push -m 'local changes stashed before self update' --quiet - fi - git pull --force --quiet - git checkout $BRANCH --quiet - git pull --force --quiet - echo -e "${GREEN_BOLD} ✓ Update Complete. Running New Version. Standby...${CLEAR}" - echo "✓ Update Complete. Running New Version. Standby..." >> download.log - sleep 3 - cd - >/dev/null || exit 1 - - exec "$CALLEDSCRIPTNAME" "${ARGS[@]}" - - # Exit this old instance of the script - exit 1 - } - echo -e "${GREEN_BOLD} ✓ Version: Current${CLEAR}" - else - echo -e "${RED_REGULAR} ✗ Git Clone Not Detected: Skipping Update Check${CLEAR}" - fi - echo -} - -# usage_example - Show Usage and Exit -usage_example() { - echo 'Usage: ./kiwix-zim-updater.sh /full/path/' - echo '' - echo ' /full/path/ Full path to ZIM directory' - echo '' - echo 'Universal Options:' - echo ' -h, --help Show this usage and exit.' - echo ' -u, --skip-update Skips checking for script updates (very useful for development).' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' - echo ' -n , --min-size Minimum ZIM Size to be downloaded.' - echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' - echo ' -x , --max-size Maximum ZIM Size to be downloaded.' - echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' - echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' - echo ' ' - echo 'Action Method Options:' - echo ' -w, --web Downloads zims over http(s).' - echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' - echo ' ' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' - echo ' Disable this using -S' - echo '' - echo 'Web Download Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' - echo ' -l , --location Country Code to prefer mirrors from' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - - echo - exit 0 -} - -# flags - Flag and ZIM Processing Functions -flags() { - echo -e "${YELLOW_BOLD}2. Preprocessing...${CLEAR}" - echo "2. Preprocessing..." >> download.log - echo - echo -e "${BLUE_BOLD} -Validating ZIM directory...${CLEAR}" - echo "Validating ZIM directory..." >> download.log - - # Let's identify which argument is the ZIM directory path and if it's an actual directory. - if [[ -d ${1} ]]; then - if [[ -w ${1} ]]; then - ZIMPath=$1 - else - ZIMPath=$1 - echo -e "${RED_REGULAR} ✗ Cannot write to '${1}', continuing in dry-run${CLEAR}" - echo "✗ Cannot write to '${1}', continuing in dry-run" >> download.log - echo - DEBUG=1 - fi - else # Um... no ZIM directory path provided? Okay, let's show the usage and exit. - if [[ -z ${1} ]]; then - echo -e "${RED_REGULAR} ✗ Kiwix ZIM Directory not provided${CLEAR}" - echo "✗ Kiwix ZIM Directory not provided" >> download.log - else - echo -e "${RED_REGULAR} ✗ '$1' is not a directory${CLEAR}" - echo "✗ '$1' is not a directory" >> download.log - fi - echo - usage_example - fi - - # Check for and add if missing, trailing slash. - [[ "${ZIMPath}" != */ ]] && ZIMPath="${ZIMPath}/" - - # Now we need to check for ZIM files. - shopt -s nullglob # This is in case there are no matching files - - # Load all found ZIM(s) w/path into LocalZIMArray - IFS=$'\n' read -r -d '' -a LocalZIMArray < <(ls -1 "$ZIMPath" | grep -iP "\.zim$") - unset IFS - - for index in "${!LocalZIMArray[@]}" ; do - duplicated=0 - myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") - for scanIndex in "${!LocalZIMArray[@]}"; do - if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then - if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi - else - if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi - fi - scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") - - if [[ "$myBasename" == "$scanBasename" ]]; then - if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then - echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log - elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then - echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log - else - echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log - fi - duplicated=1 - break - fi - done - [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' - done - LocalZIMArray=("${LocalZIMArray[@]}") - - # Check that ZIM(s) were actually found/loaded. - if [ ${#LocalZIMArray[@]} -eq 0 ]; then # No ZIM(s) were found in the directory... I guess there's nothing else for us to do, so we'll Exit. - echo -e "${RED_REGULAR} ✗ No ZIMs found. Exiting...${CLEAR}" - exit 0 - else - echo -e "${GREEN_BOLD} ✓ Valid ZIM Directory ${CLEAR}" - fi - echo - - echo -e "${BLUE_BOLD} -Building online ZIM list...${CLEAR}" - - # Build online ZIM list. - master_scrape - - echo - - # Populate ZIM arrays from found ZIM(s) - echo -e "${BLUE_BOLD} -Parsing ZIM(s)...${CLEAR}" - - for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). - LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. - filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. - # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' - - # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL - for ((z = 0; z < ${#Basenames[@]}; z++)); do - if [[ ${Basenames[$z]} == "$filename" ]]; then # Match Found (ignore the filename datepart). - LocalZIMRemoteIndexArray[$i]="$z" - break - else # No Match Found. - LocalZIMRemoteIndexArray[$i]="-1" - fi - done - - if [[ LocalZIMRemoteIndexArray[$i] -eq -1 ]]; then - echo -e "${RED_REGULAR} ✗ ${LocalZIMNameArray[$i]} No online match found.${CLEAR}" - else - echo -e "${GREEN_BOLD} ✓ ${LocalZIMNameArray[$i]} [${RemoteCategory[${LocalZIMRemoteIndexArray[$i]}]}]${CLEAR}" - fi - done - - echo - echo -e "${GREEN_REGULAR} ${#LocalZIMNameArray[*]} ZIM(s) found.${CLEAR}" -} - -# mirror_search - Find ZIM URL Priority #1 mirror from meta4 Function -mirror_search() { - IsMirror=0 - DownloadURL="" - RemotePath="${RemotePaths[${LocalZIMRemoteIndexArray[$z]}]}" - ExpectedSize="${FileSizes[${LocalZIMRemoteIndexArray[$z]}]}" - - # If we need the checksum, we need a link and the hash, which we can get both by using .meta4, otherwise we only need - # Silently fetch (via wget) the associated meta4 xml and extract the mirror URL marked priority="1" - MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") - ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') - ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') - - # still run the last wget so that the logic is simple for verification. - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL - return - fi - - RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') - - # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. - if [[ $RawMirror == *"http"* ]]; then # Mirror URL found - DownloadURL="$RawMirror" # Set the mirror URL as our download URL - IsMirror=1 - else # Mirror URL not found - DownloadURL="$BaseURL$RemotePath" # Set the direct download URL as our download URL - fi -} - -######################### -# Begin Script Execute -######################### - -while [[ $# -gt 0 ]]; do - case $1 in - -h | --help) - usage_example - ;; - -d | --disable-dry-run) - DEBUG=0 - shift # discard argument - ;; - -v | --version) - echo "$VER" - exit 0 - ;; - -p | --skip-purge) - SKIP_PURGE=1 - shift # discard argument - ;; - -n | --min-size) - shift # discard -n argument - MIN_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes - shift # discard value - ;; - -x | --max-size) - shift # discard -x argument - MAX_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes - shift # discard value - ;; - -l | --location) - shift # discard -l argument - if [[ "$1" =~ ^[A-Z]{2}$ ]]; then - COUNTRY_CODE=$1 # convert passed arg to bytes - else - COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" >> download.log - fi - shift # discard value - ;; - -c | --calculate-checksum) - CALCULATE_CHECKSUM=1 - shift - ;; - -f | --verfiy-library) - VERIFY_LIBRARY=1 - CALCULATE_CHECKSUM=1 - shift - ;; - -u | --skip-update) - SKIP_UPDATE=1 - shift - ;; - -g | --get-index) - FORCE_FETCH_INDEX=1 - shift - ;; - -w | --web) - DOWNLOAD_METHOD=1 - shift - ;; - -t | --torrent) - DOWNLOAD_METHOD=2 - shift - ;; - -S | --no-sha) - CHECKSUM_FILES=0 - shift - ;; - *) - # We can either parse the arg here, or just tuck it away for safekeeping - POSITIONAL_ARGS+=("$1") # save positional arg - shift # past argument - ;; - esac -done - -set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped earlier - -clear # Clear screen - -if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then - echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" -fi - -# Display Header -echo "==========================================" -echo " kiwix-zim-updater" -if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then - echo " WARNING: kiwix-zim.sh has been deprecated " - echo " in favor of kiwix-zim-updater " - echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01" >> download.log -fi -echo " download.kiwix.org ZIM Updater" -echo -echo " v$VER by DocDrydenn and jojo2357" -echo "==========================================" -echo -echo " DRY-RUN/SIMULATION" -[[ $DEBUG -eq 1 ]] && echo " - ENABLED -" -[[ $DEBUG -eq 1 ]] && echo -[[ $DEBUG -eq 1 ]] && echo " Use '-d' to disable." -[[ $DEBUG -eq 0 ]] && echo " - DISABLED -" -[[ $DEBUG -eq 0 ]] && echo -[[ $DEBUG -eq 0 ]] && echo " !!! Caution !!!" -echo -echo "==========================================" -echo - -# First, Self-Update Check. -# Shouldnt this be first? it is not dependent on anything else and resets everything, so may as well reset it before getting all invested? -self_update - -# Second, Flag Check. -flags "$@" - -echo -echo -e "${YELLOW_BOLD}3. Processing ZIM(s)...${CLEAR}" -echo "3. Processing ZIM(s)..." >> download.log -echo - -AnyDownloads=0 - -for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do - RemoteIndex=${LocalZIMRemoteIndexArray[$i]} - if [[ $RemoteIndex -eq -1 ]]; then - if [[ $VERIFY_LIBRARY -eq 1 ]] && [[ -f "$FileName.sha256" ]]; then - LocalRequiresDownloadArray+=(3) - AnyDownloads=1 - echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" - echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - fi - continue - fi - - FileName=${LocalZIMNameArray[$i]} - echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" - [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue - - - MatchingSize=${FileSizes[$RemoteIndex]} - MatchingFileName=${RemoteFiles[$RemoteIndex]} - MatchingFullPath=${RemotePaths[$RemoteIndex]} - MatchingCategory=${RemoteCategory[$RemoteIndex]} - - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue - - MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" - MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" - - LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')" - LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')" - - FileTooSmall=0 - [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 - FileTooLarge=0 - [[ $MAX_SIZE -gt 0 ]] && [[ $MatchingSize -gt $MAX_SIZE ]] && FileTooLarge=1 - FileSizeAcceptable=0 - [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 - - if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then - if [ $FileTooSmall -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - elif [ $FileTooLarge -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - fi - elif [[ "$MatchingFileName" == "$FileName" ]]; then - if [ $VERIFY_LIBRARY -eq 1 ]; then - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - fi - else - if [ $VERIFY_LIBRARY -eq 1 ]; then - if [[ -f "$ZIMPath$FileName.sha256" ]]; then - LocalRequiresDownloadArray+=(2) - AnyDownloads=1 - echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" - else - echo " Checking for online checksum..." - if wget -S --spider -q -O - "$BaseURL$MatchingCategory/$FileName.sha256" >/dev/null 2>&1; then - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - echo " ✗ Online Version Not Found" - fi - fi - else - if [ $FileTooSmall -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [ $FileTooLarge -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [ "$MatchedYear" -lt "$LocalYear" ]; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - else - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" - fi - fi - fi - echo -done - -echo -e "${YELLOW_BOLD}4. Downloading New ZIM(s)...${CLEAR}" -echo -e "4. Downloading New ZIM(s)..." >> download.log -echo - -# Let's clear out any possible duplicates - -# Let's Start the download process, but only if we have actual downloads to do. -if [ $AnyDownloads -eq 1 ]; then - for ((z = 0; z < ${#LocalZIMNameArray[@]}; z++)); do # Iterate through the download queue. - [[ ${LocalRequiresDownloadArray[$z]} -eq 0 ]] && continue - - OldZIM=${LocalZIMNameArray[$z]} - OldZIMPath=$ZIMPath$OldZIM - - echo -e "${BLUE_BOLD} Processing $OldZIM${CLEAR}" - - if [[ ${LocalRequiresDownloadArray[$z]} -eq 3 ]]; then - ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") - - NewZIM="$OldZIM" - NewZIMPath="$OldZIMPath" - else - mirror_search # Let's look for a mirror URL first. - - if [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]]; then - ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") - fi - - NewZIM=${RemoteFiles[${LocalZIMRemoteIndexArray[$z]}]} - NewZIMPath=$ZIMPath$NewZIM - fi - - - FilePath=$ZIMPath$NewZIM # Set destination path with file name - LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name - TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name - - RequiresDownload=0 - - if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - if [[ -f $NewZIM ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" - echo - elif [[ -f $TorrentFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" - echo - else - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" - RequiresDownload=1 - fi - elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" - echo - else # New ZIM not found, so we'll go ahead and download it. - RequiresDownload=1 - if [[ -f $LockFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM download was interrupted. Continuing...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM download was interrupted. Continuing...${CLEAR}" - else - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM doesn't exist on disk. Downloading...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM doesn't exist on disk. Downloading...${CLEAR}" - fi - echo - fi - else - # lockfile implies an incomplete download - if [[ -f $LockFilePath ]]; then - echo "Incomplete download detected, resuming" >>download.log - echo -e "${YELLOW_REGULAR} Status: Incomplete download detected, resuming${CLEAR}" - - # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" - # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" - RequiresDownload=1 - elif [[ -f $TorrentFilePath ]]; then - echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" - echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log - else - # actually verify the file - echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" - echo "Calculating checksum for : $OldZIM" >>download.log - [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && echo "$ExpectedHash $OldZIM" >"$OldZIMPath.sha256" - if [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && [[ $(du -b "$OldZIMPath" | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then - RequiresDownload=1 - if [[ $DEBUG -eq 0 ]]; then - if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" - echo "✗ Status : File size verification failed, removing corrupt file" >>download.log - rm "$OldZIMPath" - else - echo -e "${RED_BOLD} ✗ Status : File size verification failed but purge skipped${CLEAR}" - echo "✗ Status : File size verification failed but purge skipped" >>download.log - fi - else - [[ $SKIP_PURGE -eq 0 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed, removing corrupt file ($FilePath)${CLEAR}" - [[ $SKIP_PURGE -eq 1 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed but purge skipped ($FilePath)${CLEAR}" - fi - - echo - elif [[ ${#ExpectedHash} -ne 64 ]]; then - echo " This hash doesn't look quite right...skipping" - elif (cd "$ZIMPath" && ! sha256sum --status -c "$OldZIM.sha256"); then - # we checked a very old file, but we will choose to not replace it because we cannot, so we will leave it. A regular update will purge it naturally - [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && echo -e "${RED_BOLD} Checksum failed, online file not found, continuing${CLEAR}" && echo && continue - if [[ $DEBUG -eq 0 ]]; then - if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : Checksum failed, removing corrupt file${CLEAR}" - echo "✗ Status : Checksum failed, removing corrupt file" >>download.log - rm "$OldZIMPath" - rm "$OldZIMPath.sha256" 2>/dev/null - else - echo -e "${RED_BOLD} ✗ Status : Checksum failed but purge was skipped${CLEAR}" - echo "✗ Status : Checksum failed but purge was skipped" >>download.log - echo - continue - fi - else - echo -e "${RED_BOLD} ✗ Status : *** Simulated *** Checksum failed, removing corrupt file ($FilePath)${CLEAR}" - fi - - RequiresDownload=1 - echo - else - echo -e "${GREEN_BOLD} ✓ Status : Checksum passed${CLEAR}" - echo "✓ Status : Checksum passed" >>download.log - - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - - # rm "$OldZIMPath.sha256" - - echo - continue - fi - [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && continue - # rm "$OldZIMPath.sha256" - fi - fi - echo >>download.log - [[ $DEBUG -eq 0 ]] && echo "Start : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log - # Here is where we actually download the files and log to the download.log file. - if [[ $RequiresDownload -eq 1 ]]; then - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - FilePath="$FilePath.torrent" - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi - - continue - else - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log - - - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi - fi - fi - - echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" - if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then - echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" - if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then - if [[ $DEBUG -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" - echo "✗ Status : File size verification failed, removing corrupt file" >>download.log - rm "$NewZIMPath" - else - echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" - fi - elif [[ ${#ExpectedHash} -ne 64 ]]; then - echo -e "${YELLOW_BOLD} This hash doesn't look quite right...skipping${CLEAR}" - elif [[ $DEBUG -eq 0 ]] && (cd "$ZIMPath" && ! sha256sum --status -c "$NewZIM.sha256"); then - echo -e "${RED_BOLD} ✗ Checksum failed, removing corrupt file${CLEAR}" - rm "$NewZIMPath" - touch "$NewZIMPath" - DownloadFailed=1 - else - if [[ $DEBUG -eq 0 ]]; then - echo -e "${GREEN_BOLD} ✓ Checksum passed${CLEAR}" - else - echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" - fi - fi - # rm "$NewZIMPath.sha256" - # rm "$LockFilePath" - echo - fi - - echo >> download.log - [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - - # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later - if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" - [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" - continue - fi - - [[ $RequiresDownload -eq 1 ]] && [[ $DEBUG -eq 0 ]] && rm "$LockFilePath" - - ######################################## - - echo -e "${BLUE_REGULAR} Old : $OldZIM${CLEAR}" - echo "Old : $OldZIM" >>download.log - echo -e "${BLUE_BOLD} New : $NewZIM${CLEAR}" - echo "New : $NewZIM" >>download.log - # Check for the new ZIM on disk. - if [[ -f "$NewZIMPath" ]]; then # New ZIM found - if [[ $DEBUG -eq 0 ]]; then - if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully." >>download.log - # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM - else - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log - [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM - fi - else - echo -e "${GREEN_BOLD} ✓ Status : *** Simulated ***${CLEAR}" - echo "✓ Status : *** Simulated ***" >>download.log - fi - else # New ZIM not found. Something went wrong, so we will skip this purge. - if [[ $DEBUG -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : New ZIM failed verification. Old ZIM not purged.${CLEAR}" - echo "✗ Status : New ZIM failed verification. Old ZIM not purged." >>download.log - else - if [[ $RequiresDownload -eq 1 ]]; then - echo -e "${GREEN_BOLD} ✓ Status : *** Simulated *** New zim exists, old zim purged${CLEAR}" - echo "✓ Status : *** Simulated *** New zim exists, old zim purged" >>download.log - else - echo -e "${YELLOW_BOLD} ✗ Status : *** Simulated *** Zim was skipped, and will not be purged${CLEAR}" - echo "✗ Status : *** Simulated *** Zim not purged" >>download.log - fi - fi - fi - echo - echo >>download.log - - ######################################### - - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - done - - if [[ $DEBUG -eq 0 ]]; then - IFS=$'\n' HangingFileLocks=("$ZIMPath".~lock.*.zim) - unset IFS - - if [[ ${#HangingFileLocks[@]} -gt 0 ]]; then - echo -e "${YELLOW_BOLD}5. Cleaning up...${CLEAR}" - echo "5. Cleaning up..." >> download.log - echo - - for ((i = 0; i < ${#HangingFileLocks[@]}; i++)); do - baseFileName=$(echo "${HangingFileLocks[$i]}" | grep -ioP "(?<=\.~lock\.).*$") - echo -e " ${RED_BOLD}Found broken lock: ${HangingFileLocks[$i]}${CLEAR}" - echo "Found broken lock: ${HangingFileLocks[$i]}" >> download.log - if [[ -f "$ZIMPath$baseFileName" ]]; then - echo -e " ${BLUE_BOLD}Found abandoned download: $ZIMPath$baseFileName${CLEAR}" - echo -e "Found abandoned download: $ZIMPath$baseFileName" >> download.log - rm "$ZIMPath$baseFileName" - fi - if [[ -f "$ZIMPath$baseFileName.sha256" ]]; then - echo -e " ${BLUE_BOLD}Found abandoned checksum: $ZIMPath$baseFileName.sha256${CLEAR}" - echo -e "Found abandoned checksum: $ZIMPath$baseFileName.sha256" >> download.log - rm "$ZIMPath$baseFileName.sha256" - fi - rm "${HangingFileLocks[$i]}" - echo - done - fi - fi -else - echo -e "${GREEN_REGULAR} ✓ Download: Nothing to download.${CLEAR}" - echo "✓ Download: Nothing to download." >> download.log - echo -fi - -if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then - echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01.\n This exit code is nonzero, but the script has not errored out. This is simply so you may notice and switch to using kiwix-zim-updater " >> download.log - exit 2 -fi \ No newline at end of file From b7512275375f0391a6d7d36b9c6dc0f77578d455 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:21:43 -0700 Subject: [PATCH 9/9] Support new* kiwix file date format (#60) * update sorting and regexes * improve regex validation and quote escaping * comment * more safety Co-authored-by: Jailford --------- Co-authored-by: Jailford --- kiwix-zim-updater.sh | 106 +++++++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 25 deletions(-) diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 571b4f7..7b5b692 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # Kiwix-Zim-Updater. A script to update your kiwix library -# Copyright (C) 2022 DocDrydenn -# Copyright (C) 2023 jojo2357 +# Copyright (C) 2022 DocDrydenn +# Copyright (C) 2023-2026 jojo2357 # SPDX-License-Identifier: GPL-2.0-only -VER="3.4" +VER="3.5" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -60,6 +60,12 @@ BLUE_REGULAR="\033[0;34m" BLUE_BOLD="\033[1;34m" CLEAR="\033[0m" +MONTH_REGEX="\d{2}[a-z]?" +YEAR_REGEX="\d{4}" +END_ANCHOR_REGEX="\.zim" + +COMPLETE_ENDING_REGEX="$YEAR_REGEX-$MONTH_REGEX$END_ANCHOR_REGEX" + # This will ask the api what files it has to offer and store them in arrays master_scrape() { unset RemoteFiles @@ -88,10 +94,26 @@ master_scrape() { RawLibrary=$(grep -i '> download.log fi + if [[ ${#RemoteFiles[@]} -ne ${#RemoteFiles[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#Basenames[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#RemotePaths[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#RemoteCategory[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#FileSizes[@]} ]] ; then + echo -e "${RED_BOLD}CRITICAL ERROR!${CLEAR}" + echo -e "${RED}Parsed array lengths did not match. Aborting to prevent data loss${CLEAR}" + exit 99 + fi + # Housekeeping... + unset ValidLibrary unset RawLibrary unset hrefs } @@ -259,14 +292,20 @@ flags() { for index in "${!LocalZIMArray[@]}" ; do duplicated=0 - myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") + + # Sorting cannot reliably sort the month string so we need to check it to be sure + MyDate="$(echo "${LocalZIMArray[$index]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + MyYear="$(echo "$MyDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + MyMonth="$(echo "$MyDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + for scanIndex in "${!LocalZIMArray[@]}"; do if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi else if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi fi - scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") if [[ "$myBasename" == "$scanBasename" ]]; then if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then @@ -274,6 +313,22 @@ flags() { elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log else + ScanDate="$(echo "${LocalZIMArray[$scanIndex]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + ScanYear="$(echo "$ScanDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + ScanMonth="$(echo "$ScanDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + + # this file is not shadowed if it is newer. + # this is caused by `ls -1` and `sort` having weird lexicons that make the prefix sort unreliable that 02a can sort between 01 and 02 instead of after 02 + if [[ $MyYear > $ScanYear ]]; then + echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log + duplicated=2 + break + elif [[ $MyYear == $ScanYear ]] && [[ $MyMonth > $ScanMonth ]]; then + echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log + duplicated=2 + break + fi + echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log fi duplicated=1 @@ -281,6 +336,7 @@ flags() { fi done [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' + [[ $duplicated -eq 2 ]] && unset -v 'LocalZIMArray[$scanIndex]' done LocalZIMArray=("${LocalZIMArray[@]}") @@ -305,7 +361,7 @@ flags() { for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. - filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. + filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=$COMPLETE_ENDING_REGEX$)") # Extract file name. # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL @@ -509,13 +565,13 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue - MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" - MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" + MatchedDate="$(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + MatchedYear="$(echo "$MatchedDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + MatchedMonth="$(echo "$MatchedDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" - LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')" - LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')" + LocalDate="$(echo "$FileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + LocalYear="$(echo "$LocalDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + LocalMonth="$(echo "$LocalDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" FileTooSmall=0 [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 @@ -563,16 +619,16 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do else if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" elif [ $FileTooLarge -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [ "$MatchedYear" -lt "$LocalYear" ]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + elif [[ $MatchedYear < $LocalYear ]]; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" - elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then + elif [[ "$MatchedYear" == "$LocalYear" ]] && { [[ $MatchedMonth < $LocalMonth ]] || [[ "$MatchedMonth" == "$LocalMonth" ]] ; }; then LocalRequiresDownloadArray+=(0) echo " ✗ No new update" else