diff --git a/README.md b/README.md index e9a0c70..aa17ed8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,12 @@ A script to check `download.kiwix.org` for updates to your local ZIM library. Just pass this script your ZIM directory and away it goes. *(see Usage below)* -Tested on Debian 13, and should work out of the box on most debian systems, but I have not tested that. +Tested on PopOS! 22.04, and should work out of the box on most debian systems, but I have not tested that. + +# DEPRECATION WARNING +`kiwix-zim.sh` has been deprecated in favor of the more descriptive `kiwix-zim-updater.sh`. A hard link from `kiwix-zim.sh` to `kiwix-zim-updater.sh` will be maintained on this repo until at least `2025-01-01T00:00:00Z` for compatability with CRON users. + +***CALLING `kiwix-zim.sh` WILL RESULT IN AN EXIT CODE OF 2 ON SUCCESSFUL EXECUTION. THIS BEHAVIOR IS EXPECTED. CALL `kiwix-zim-updater.sh` INSTEAD TO GET A 0 EXIT CODE*** ## What It Does @@ -53,7 +58,6 @@ Creates `downloads.log` for the following reasons: ## Limitations -- This script relies heavily on `grep`'s `-P` flag for lookarounds and the matcher reset, `\K`. This may cause compatibility problems on macOS - This script is only for ZIM(s) hosted by `download.kiwix.org` due to the file naming standard they use. If you have self-made ZIM(s) or ZIM(s) downloaded from somewhere else, they most likely do not use the same naming standards and will not be processed by this script. - If you have ZIM(s) from `download.kiwix.org`, but you have changed their file names, this script will treat them like the previous limitation explains. - This script does not attempt to update any `library.xml` that may or may not exist/be needed for your install/setup of Kiwix. If needed, you'll need to handle this part on your own. @@ -84,27 +88,20 @@ NOTE: if you are not tracking the `main` branch, the update check will be skippe /full/path/ Full path to ZIM directory - Universal Options: - -h, --help Show this usage and exit. + Options: + -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. + -f, --verify-library Verifies that the entire library has the correct checksums as found online. + Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet -d, --disable-dry-run Dry-Run Override. - *** Caution *** + *** Caution *** + + -h, --help Show this usage and exit. + -p, --skip-purge Skips purging any replaced ZIMs. -u, --skip-update Skips checking for script updates (very useful for development). - -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day -n , --min-size Minimum ZIM Size to be downloaded. Specify units include M Mi G Gi, etc. See `man numfmt` -x , --max-size Maximum ZIM Size to be downloaded. - Specify units include M Mi G Gi, etc. See `man numfmt` - - Action Method Options: - -w, --web Downloads zims over http(s). - -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD. - - Web Download Options: - -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. - -p, --skip-purge Skips purging any replaced ZIMs. + Specify units include M Mi G Gi, etc. See `man numfmt` -l , --location Country Code to prefer mirrors from - -f, --verify-library Verifies that the entire library has the correct checksums as found online. - Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet. - Disable this using -S - -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums. + -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day ``` diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 7b5b692..7a6c717 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,11 +1,6 @@ -#!/usr/bin/env bash +#!/bin/bash -# Kiwix-Zim-Updater. A script to update your kiwix library -# Copyright (C) 2022 DocDrydenn -# Copyright (C) 2023-2026 jojo2357 -# SPDX-License-Identifier: GPL-2.0-only - -VER="3.5" +VER="3.1" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -13,7 +8,7 @@ LocalZIMArray=() LocalZIMNameArray=() # This array will map the local zim to the index in the remote arrays that contains the same base file name LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim should be processed in the download loop +# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop LocalRequiresDownloadArray=() # After updating, this array will be used to store hanging locks and to deal with them HangingFileLocks=(); @@ -41,13 +36,9 @@ DEBUG=1 # This forces the script to default to "dry-run/simulation mode" MIN_SIZE=0 MAX_SIZE=0 CALCULATE_CHECKSUM=0 -CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 -DOWNLOAD_METHOD=1 # 1: web 2: torrent -# sort of a default, will be overwritten when the library is parsed. Set here to what it probably will be -BaseURL="https://lb.download.kiwix.org/zim/" -CatalogURL="https://opds.library.kiwix.org/catalog/v2/entries?count=-1" +BaseURL="https://download.kiwix.org/zim/" ZIMPath="" RED_REGULAR="\033[0;31m" @@ -60,12 +51,6 @@ BLUE_REGULAR="\033[0;34m" BLUE_BOLD="\033[1;34m" CLEAR="\033[0m" -MONTH_REGEX="\d{2}[a-z]?" -YEAR_REGEX="\d{4}" -END_ANCHOR_REGEX="\.zim" - -COMPLETE_ENDING_REGEX="$YEAR_REGEX-$MONTH_REGEX$END_ANCHOR_REGEX" - # This will ask the api what files it has to offer and store them in arrays master_scrape() { unset RemoteFiles @@ -79,66 +64,32 @@ master_scrape() { indexIsValid=0 else indexDate="$(head -1 "kiwix-index")" - if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then + if [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then indexIsValid=0 fi fi if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary. - # Lets defer touching the index until we have actually fetched it - RawLibrary="$(wget --show-progress -q -O - "$CatalogURL")" - RawLibrary="$(echo "$RawLibrary" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" + # both write the file timestamp to the index file and save all of the links to RawLibrary + RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" + echo "$RawLibrary" >> kiwix-index else RawLibrary=$(grep -i '> download.log fi - if [[ ${#RemoteFiles[@]} -ne ${#RemoteFiles[@]} ]] || \ - [[ ${#RemoteFiles[@]} -ne ${#Basenames[@]} ]] || \ - [[ ${#RemoteFiles[@]} -ne ${#RemotePaths[@]} ]] || \ - [[ ${#RemoteFiles[@]} -ne ${#RemoteCategory[@]} ]] || \ - [[ ${#RemoteFiles[@]} -ne ${#FileSizes[@]} ]] ; then - echo -e "${RED_BOLD}CRITICAL ERROR!${CLEAR}" - echo -e "${RED}Parsed array lengths did not match. Aborting to prevent data loss${CLEAR}" - exit 99 - fi - # Housekeeping... - unset ValidLibrary unset RawLibrary unset hrefs } @@ -173,7 +113,7 @@ self_update() { echo # Check if script path is a git clone. # If true, then check for update. - # If false, skip self-update check/function. + # If false, skip self-update check/funciton. if [ $SKIP_UPDATE -eq 1 ]; then echo -e "${YELLOW_REGULAR} Check Skipped${CLEAR}" echo "Check Skipped" >> download.log @@ -217,34 +157,25 @@ self_update() { # usage_example - Show Usage and Exit usage_example() { echo 'Usage: ./kiwix-zim-updater.sh /full/path/' - echo '' + echo echo ' /full/path/ Full path to ZIM directory' - echo '' - echo 'Universal Options:' + echo + echo 'Options:' + echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' + echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' + echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet' + echo ' -d, --disable-dry-run Dry-Run Override.' + echo ' *** Caution ***' + echo echo ' -h, --help Show this usage and exit.' + echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -u, --skip-update Skips checking for script updates (very useful for development).' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' echo ' -n , --min-size Minimum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' echo ' -x , --max-size Maximum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' - echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' - echo ' ' - echo 'Action Method Options:' - echo ' -w, --web Downloads zims over http(s).' - echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' - echo ' ' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' - echo ' Disable this using -S' - echo '' - echo 'Web Download Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -l , --location Country Code to prefer mirrors from' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - + echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' echo exit 0 } @@ -292,43 +223,19 @@ flags() { for index in "${!LocalZIMArray[@]}" ; do duplicated=0 - myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") - - # Sorting cannot reliably sort the month string so we need to check it to be sure - MyDate="$(echo "${LocalZIMArray[$index]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" - MyYear="$(echo "$MyDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" - MyMonth="$(echo "$MyDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" - + myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") for scanIndex in "${!LocalZIMArray[@]}"; do if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi else if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi fi - scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") + scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") if [[ "$myBasename" == "$scanBasename" ]]; then if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log - elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then - echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log else - ScanDate="$(echo "${LocalZIMArray[$scanIndex]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" - ScanYear="$(echo "$ScanDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" - ScanMonth="$(echo "$ScanDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" - - # this file is not shadowed if it is newer. - # this is caused by `ls -1` and `sort` having weird lexicons that make the prefix sort unreliable that 02a can sort between 01 and 02 instead of after 02 - if [[ $MyYear > $ScanYear ]]; then - echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log - duplicated=2 - break - elif [[ $MyYear == $ScanYear ]] && [[ $MyMonth > $ScanMonth ]]; then - echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log - duplicated=2 - break - fi - echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log fi duplicated=1 @@ -336,7 +243,6 @@ flags() { fi done [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' - [[ $duplicated -eq 2 ]] && unset -v 'LocalZIMArray[$scanIndex]' done LocalZIMArray=("${LocalZIMArray[@]}") @@ -361,7 +267,7 @@ flags() { for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. - filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=$COMPLETE_ENDING_REGEX$)") # Extract file name. + filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL @@ -397,13 +303,6 @@ mirror_search() { MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') - - # still run the last wget so that the logic is simple for verification. - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL - return - fi - RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. @@ -452,7 +351,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invalid country code, falling back to default kiwix behavior" >> download.log + echo "Invlaid country code, falling back to default kiwix behavior" fi shift # discard value ;; @@ -460,7 +359,7 @@ while [[ $# -gt 0 ]]; do CALCULATE_CHECKSUM=1 shift ;; - -f | --verfiy-library | --verify-library) + -f | --verfiy-library) VERIFY_LIBRARY=1 CALCULATE_CHECKSUM=1 shift @@ -473,18 +372,6 @@ while [[ $# -gt 0 ]]; do FORCE_FETCH_INDEX=1 shift ;; - -w | --web) - DOWNLOAD_METHOD=1 - shift - ;; - -t | --torrent) - DOWNLOAD_METHOD=2 - shift - ;; - -S | --no-sha) - CHECKSUM_FILES=0 - shift - ;; *) # We can either parse the arg here, or just tuck it away for safekeeping POSITIONAL_ARGS+=("$1") # save positional arg @@ -497,10 +384,6 @@ set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped e clear # Clear screen -if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then - echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" -fi - # Display Header echo "==========================================" echo " kiwix-zim-updater" @@ -557,22 +440,11 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue - MatchingSize=${FileSizes[$RemoteIndex]} MatchingFileName=${RemoteFiles[$RemoteIndex]} MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue - - MatchedDate="$(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" - MatchedYear="$(echo "$MatchedDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" - MatchedMonth="$(echo "$MatchedDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" - - LocalDate="$(echo "$FileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" - LocalYear="$(echo "$LocalDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" - LocalMonth="$(echo "$LocalDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" - FileTooSmall=0 [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 FileTooLarge=0 @@ -619,22 +491,16 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do else if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" elif [ $FileTooLarge -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" - elif [[ $MatchedYear < $LocalYear ]]; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - elif [[ "$MatchedYear" == "$LocalYear" ]] && { [[ $MatchedMonth < $LocalMonth ]] || [[ "$MatchedMonth" == "$LocalMonth" ]] ; }; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" else LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" + echo -e "${GREEN_BOLD} ✓ Update found! --> $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" fi fi fi @@ -673,29 +539,13 @@ if [ $AnyDownloads -eq 1 ]; then NewZIMPath=$ZIMPath$NewZIM fi - FilePath=$ZIMPath$NewZIM # Set destination path with file name LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name - TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name RequiresDownload=0 if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - if [[ -f $NewZIM ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" - echo - elif [[ -f $TorrentFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" - echo - else - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" - RequiresDownload=1 - fi - elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. + if [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" echo @@ -719,9 +569,6 @@ if [ $AnyDownloads -eq 1 ]; then # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" RequiresDownload=1 - elif [[ -f $TorrentFilePath ]]; then - echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" - echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log else # actually verify the file echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" @@ -790,7 +637,6 @@ if [ $AnyDownloads -eq 1 ]; then if [[ $RequiresDownload -eq 1 ]]; then [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log echo "=======================================================================" >>download.log echo "File : $NewZIM" >>download.log @@ -798,35 +644,21 @@ if [ $AnyDownloads -eq 1 ]; then [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log echo >>download.log - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - FilePath="$FilePath.torrent" - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi - continue - else - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi + # Before we actually download, let's just check to see that it isn't already in the folder. + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log fi fi - [[ $DEBUG -eq 0 ]] && echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" + echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then @@ -859,12 +691,12 @@ if [ $AnyDownloads -eq 1 ]; then echo >> download.log [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - # in all of these cases, we will not re-purge and will leave the lockfile so we know to resume later + # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $DEBUG -eq 0 ]] && [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" - [[ $DEBUG -eq 0 ]] && [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" + [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" continue fi @@ -880,12 +712,12 @@ if [ $AnyDownloads -eq 1 ]; then if [[ -f "$NewZIMPath" ]]; then # New ZIM found if [[ $DEBUG -eq 0 ]]; then if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully.${CLEAR}" - echo "✓ Status : New ZIM downloaded successfully." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" + echo "✓ Status : New ZIM downloaded succesfully." >>download.log # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM else - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully. Old ZIM purged.${CLEAR}" - echo "✓ Status : New ZIM downloaded successfully. Old ZIM purged." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" + echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM fi else diff --git a/kiwix-zim.sh b/kiwix-zim.sh new file mode 100755 index 0000000..7a6c717 --- /dev/null +++ b/kiwix-zim.sh @@ -0,0 +1,787 @@ +#!/bin/bash + +VER="3.1" + +# This array will contain all of the local zims, with the file extension +LocalZIMArray=() +# This array will contain all of the local zims, without the file extension +LocalZIMNameArray=() +# This array will map the local zim to the index in the remote arrays that contains the same base file name +LocalZIMRemoteIndexArray=() +# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop +LocalRequiresDownloadArray=() +# After updating, this array will be used to store hanging locks and to deal with them +HangingFileLocks=(); + +# This array stores the file names that kiwix has to offer, with .zim extensions +RemoteFiles=() +# Ditto, without the YYYY-MM (note a trailing _) +Basenames=() +# Contains the absolute path to this file, from /zims/ +RemotePaths=() +# Contains the folder this file is in relative to /zims/ +RemoteCategory=() + +# Set Script Strings +SCRIPT="$(readlink -f "$0")" +SCRIPTFILE="$(basename "$SCRIPT")" +SCRIPTPATH="$(dirname "$SCRIPT")" +CALLEDSCRIPTNAME="$0" +CALLEDSCRIPTFILE="$(basename "$CALLEDSCRIPTNAME")" +CALLEDSCRIPTPATH="$(dirname "$CALLEDSCRIPTNAME")" +ARGS=("$@") +BRANCH="main" +SKIP_UPDATE=0 +DEBUG=1 # This forces the script to default to "dry-run/simulation mode" +MIN_SIZE=0 +MAX_SIZE=0 +CALCULATE_CHECKSUM=0 +VERIFY_LIBRARY=0 +FORCE_FETCH_INDEX=0 +BaseURL="https://download.kiwix.org/zim/" +ZIMPath="" + +RED_REGULAR="\033[0;31m" +RED_BOLD="\033[1;31m" +YELLOW_REGULAR="\033[0;33m" +YELLOW_BOLD="\033[1;33m" +GREEN_REGULAR="\033[0;32m" +GREEN_BOLD="\033[1;32m" +BLUE_REGULAR="\033[0;34m" +BLUE_BOLD="\033[1;34m" +CLEAR="\033[0m" + +# This will ask the api what files it has to offer and store them in arrays +master_scrape() { + unset RemoteFiles + unset Basenames + unset RemotePaths + unset RemoteCategory + + indexIsValid=1 + + if [[ ! -f kiwix-index ]]; then + indexIsValid=0 + else + indexDate="$(head -1 "kiwix-index")" + if [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then + indexIsValid=0 + fi + fi + + if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then + # both write the file timestamp to the index file and save all of the links to RawLibrary + RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" + + echo "$RawLibrary" >> kiwix-index + else + RawLibrary=$(grep -i '> download.log + exit 0 + else + echo -e "${GREEN_BOLD} ✓ Found ${#RemoteFiles[@]} files online" + echo "✓ Found ${#RemoteFiles[@]} files online" >> download.log + fi + + # Housekeeping... + unset RawLibrary + unset hrefs +} + +# self_update - Script Self-Update Function +self_update() { + echo -e "${YELLOW_BOLD}1. Checking for Script Updates...${CLEAR}" + echo "1. Checking for Script Updates..." >> download.log + echo + # Check if script path is a git clone. + # If true, then check for update. + # If false, skip self-update check/funciton. + if [ $SKIP_UPDATE -eq 1 ]; then + echo -e "${YELLOW_REGULAR} Check Skipped${CLEAR}" + echo "Check Skipped" >> download.log + elif [[ -d "$SCRIPTPATH/.git" ]]; then + echo -e "${GREEN_BOLD} ✓ Git Clone Detected: Checking Script Version...${CLEAR}" + echo "✓ Git Clone Detected: Checking Script Version..." >> download.log + cd "$SCRIPTPATH" || exit 1 + [[ $(timeout 1s git rev-parse --abbrev-ref HEAD) != "$BRANCH" ]] && echo -e "${YELLOW_BOLD} You appear to be on a different branch so I will assume you are developing and do not want an update${CLEAR}" && echo && return + timeout 1s git fetch --quiet + timeout 1s git diff --quiet --exit-code "origin/$BRANCH" "$SCRIPTFILE" + [ $? -eq 1 ] && { + echo -e "${RED_REGULAR} ✗ Version: Mismatched${CLEAR}" + echo "✗ Version: Mismatched" >> download.log + echo + echo -e "${YELLOW_BOLD}1a. Fetching Update...${CLEAR}" + echo "1a. Fetching Update..." >> download.log + echo + if [ -n "$(git status --porcelain)" ]; then + git stash push -m 'local changes stashed before self update' --quiet + fi + git pull --force --quiet + git checkout $BRANCH --quiet + git pull --force --quiet + echo -e "${GREEN_BOLD} ✓ Update Complete. Running New Version. Standby...${CLEAR}" + echo "✓ Update Complete. Running New Version. Standby..." >> download.log + sleep 3 + cd - >/dev/null || exit 1 + + exec "$CALLEDSCRIPTNAME" "${ARGS[@]}" + + # Exit this old instance of the script + exit 1 + } + echo -e "${GREEN_BOLD} ✓ Version: Current${CLEAR}" + else + echo -e "${RED_REGULAR} ✗ Git Clone Not Detected: Skipping Update Check${CLEAR}" + fi + echo +} + +# usage_example - Show Usage and Exit +usage_example() { + echo 'Usage: ./kiwix-zim-updater.sh /full/path/' + echo + echo ' /full/path/ Full path to ZIM directory' + echo + echo 'Options:' + echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' + echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' + echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet' + echo ' -d, --disable-dry-run Dry-Run Override.' + echo ' *** Caution ***' + echo + echo ' -h, --help Show this usage and exit.' + echo ' -p, --skip-purge Skips purging any replaced ZIMs.' + echo ' -u, --skip-update Skips checking for script updates (very useful for development).' + echo ' -n , --min-size Minimum ZIM Size to be downloaded.' + echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' + echo ' -x , --max-size Maximum ZIM Size to be downloaded.' + echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' + echo ' -l , --location Country Code to prefer mirrors from' + echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' + echo + exit 0 +} + +# flags - Flag and ZIM Processing Functions +flags() { + echo -e "${YELLOW_BOLD}2. Preprocessing...${CLEAR}" + echo "2. Preprocessing..." >> download.log + echo + echo -e "${BLUE_BOLD} -Validating ZIM directory...${CLEAR}" + echo "Validating ZIM directory..." >> download.log + + # Let's identify which argument is the ZIM directory path and if it's an actual directory. + if [[ -d ${1} ]]; then + if [[ -w ${1} ]]; then + ZIMPath=$1 + else + ZIMPath=$1 + echo -e "${RED_REGULAR} ✗ Cannot write to '${1}', continuing in dry-run${CLEAR}" + echo "✗ Cannot write to '${1}', continuing in dry-run" >> download.log + echo + DEBUG=1 + fi + else # Um... no ZIM directory path provided? Okay, let's show the usage and exit. + if [[ -z ${1} ]]; then + echo -e "${RED_REGULAR} ✗ Kiwix ZIM Directory not provided${CLEAR}" + echo "✗ Kiwix ZIM Directory not provided" >> download.log + else + echo -e "${RED_REGULAR} ✗ '$1' is not a directory${CLEAR}" + echo "✗ '$1' is not a directory" >> download.log + fi + echo + usage_example + fi + + # Check for and add if missing, trailing slash. + [[ "${ZIMPath}" != */ ]] && ZIMPath="${ZIMPath}/" + + # Now we need to check for ZIM files. + shopt -s nullglob # This is in case there are no matching files + + # Load all found ZIM(s) w/path into LocalZIMArray + IFS=$'\n' read -r -d '' -a LocalZIMArray < <(ls -1 "$ZIMPath" | grep -iP "\.zim$") + unset IFS + + for index in "${!LocalZIMArray[@]}" ; do + duplicated=0 + myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + for scanIndex in "${!LocalZIMArray[@]}"; do + if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then + if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi + else + if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi + fi + scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + + if [[ "$myBasename" == "$scanBasename" ]]; then + if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then + echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log + else + echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log + fi + duplicated=1 + break + fi + done + [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' + done + LocalZIMArray=("${LocalZIMArray[@]}") + + # Check that ZIM(s) were actually found/loaded. + if [ ${#LocalZIMArray[@]} -eq 0 ]; then # No ZIM(s) were found in the directory... I guess there's nothing else for us to do, so we'll Exit. + echo -e "${RED_REGULAR} ✗ No ZIMs found. Exiting...${CLEAR}" + exit 0 + else + echo -e "${GREEN_BOLD} ✓ Valid ZIM Directory ${CLEAR}" + fi + echo + + echo -e "${BLUE_BOLD} -Building online ZIM list...${CLEAR}" + + # Build online ZIM list. + master_scrape + + echo + + # Populate ZIM arrays from found ZIM(s) + echo -e "${BLUE_BOLD} -Parsing ZIM(s)...${CLEAR}" + + for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). + LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. + filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. + # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' + + # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL + for ((z = 0; z < ${#Basenames[@]}; z++)); do + if [[ ${Basenames[$z]} == "$filename" ]]; then # Match Found (ignore the filename datepart). + LocalZIMRemoteIndexArray[$i]="$z" + break + else # No Match Found. + LocalZIMRemoteIndexArray[$i]="-1" + fi + done + + if [[ LocalZIMRemoteIndexArray[$i] -eq -1 ]]; then + echo -e "${RED_REGULAR} ✗ ${LocalZIMNameArray[$i]} No online match found.${CLEAR}" + else + echo -e "${GREEN_BOLD} ✓ ${LocalZIMNameArray[$i]} [${RemoteCategory[${LocalZIMRemoteIndexArray[$i]}]}]${CLEAR}" + fi + done + + echo + echo -e "${GREEN_REGULAR} ${#LocalZIMNameArray[*]} ZIM(s) found.${CLEAR}" +} + +# mirror_search - Find ZIM URL Priority #1 mirror from meta4 Function +mirror_search() { + IsMirror=0 + DownloadURL="" + RemotePath="${RemotePaths[${LocalZIMRemoteIndexArray[$z]}]}" + ExpectedSize="${FileSizes[${LocalZIMRemoteIndexArray[$z]}]}" + + # If we need the checksum, we need a link and the hash, which we can get both by using .meta4, otherwise we only need + # Silently fetch (via wget) the associated meta4 xml and extract the mirror URL marked priority="1" + MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") + ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') + ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') + RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') + + # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. + if [[ $RawMirror == *"http"* ]]; then # Mirror URL found + DownloadURL="$RawMirror" # Set the mirror URL as our download URL + IsMirror=1 + else # Mirror URL not found + DownloadURL="$BaseURL$RemotePath" # Set the direct download URL as our download URL + fi +} + +######################### +# Begin Script Execute +######################### + +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) + usage_example + ;; + -d | --disable-dry-run) + DEBUG=0 + shift # discard argument + ;; + -v | --version) + echo "$VER" + exit 0 + ;; + -p | --skip-purge) + SKIP_PURGE=1 + shift # discard argument + ;; + -n | --min-size) + shift # discard -n argument + MIN_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes + shift # discard value + ;; + -x | --max-size) + shift # discard -x argument + MAX_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes + shift # discard value + ;; + -l | --location) + shift # discard -l argument + if [[ "$1" =~ ^[A-Z]{2}$ ]]; then + COUNTRY_CODE=$1 # convert passed arg to bytes + else + COUNTRY_CODE="" + echo "Invlaid country code, falling back to default kiwix behavior" + fi + shift # discard value + ;; + -c | --calculate-checksum) + CALCULATE_CHECKSUM=1 + shift + ;; + -f | --verfiy-library) + VERIFY_LIBRARY=1 + CALCULATE_CHECKSUM=1 + shift + ;; + -u | --skip-update) + SKIP_UPDATE=1 + shift + ;; + -g | --get-index) + FORCE_FETCH_INDEX=1 + shift + ;; + *) + # We can either parse the arg here, or just tuck it away for safekeeping + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped earlier + +clear # Clear screen + +# Display Header +echo "==========================================" +echo " kiwix-zim-updater" +if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then + echo " WARNING: kiwix-zim.sh has been deprecated " + echo " in favor of kiwix-zim-updater " + echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01" >> download.log +fi +echo " download.kiwix.org ZIM Updater" +echo +echo " v$VER by DocDrydenn and jojo2357" +echo "==========================================" +echo +echo " DRY-RUN/SIMULATION" +[[ $DEBUG -eq 1 ]] && echo " - ENABLED -" +[[ $DEBUG -eq 1 ]] && echo +[[ $DEBUG -eq 1 ]] && echo " Use '-d' to disable." +[[ $DEBUG -eq 0 ]] && echo " - DISABLED -" +[[ $DEBUG -eq 0 ]] && echo +[[ $DEBUG -eq 0 ]] && echo " !!! Caution !!!" +echo +echo "==========================================" +echo + +# First, Self-Update Check. +# Shouldnt this be first? it is not dependent on anything else and resets everything, so may as well reset it before getting all invested? +self_update + +# Second, Flag Check. +flags "$@" + +echo +echo -e "${YELLOW_BOLD}3. Processing ZIM(s)...${CLEAR}" +echo "3. Processing ZIM(s)..." >> download.log +echo + +AnyDownloads=0 + +for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do + RemoteIndex=${LocalZIMRemoteIndexArray[$i]} + if [[ $RemoteIndex -eq -1 ]]; then + if [[ $VERIFY_LIBRARY -eq 1 ]] && [[ -f "$FileName.sha256" ]]; then + LocalRequiresDownloadArray+=(3) + AnyDownloads=1 + echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" + echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" + else + LocalRequiresDownloadArray+=(0) + fi + continue + fi + + FileName=${LocalZIMNameArray[$i]} + echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" + [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue + + MatchingSize=${FileSizes[$RemoteIndex]} + MatchingFileName=${RemoteFiles[$RemoteIndex]} + MatchingFullPath=${RemotePaths[$RemoteIndex]} + MatchingCategory=${RemoteCategory[$RemoteIndex]} + + FileTooSmall=0 + [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 + FileTooLarge=0 + [[ $MAX_SIZE -gt 0 ]] && [[ $MatchingSize -gt $MAX_SIZE ]] && FileTooLarge=1 + FileSizeAcceptable=0 + [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 + + if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then + if [ $FileTooSmall -eq 1 ]; then + LocalRequiresDownloadArray+=(0) + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + elif [ $FileTooLarge -eq 1 ]; then + LocalRequiresDownloadArray+=(0) + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + fi + elif [[ "$MatchingFileName" == "$FileName" ]]; then + if [ $VERIFY_LIBRARY -eq 1 ]; then + LocalRequiresDownloadArray+=(1) + AnyDownloads=1 + echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" + else + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" + fi + else + if [ $VERIFY_LIBRARY -eq 1 ]; then + if [[ -f "$ZIMPath$FileName.sha256" ]]; then + LocalRequiresDownloadArray+=(2) + AnyDownloads=1 + echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" + else + echo " Checking for online checksum..." + if wget -S --spider -q -O - "$BaseURL$MatchingCategory/$FileName.sha256" >/dev/null 2>&1; then + LocalRequiresDownloadArray+=(1) + AnyDownloads=1 + echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" + else + LocalRequiresDownloadArray+=(0) + echo " ✗ Online Version Not Found" + fi + fi + else + if [ $FileTooSmall -eq 1 ]; then + LocalRequiresDownloadArray+=(0) + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + elif [ $FileTooLarge -eq 1 ]; then + LocalRequiresDownloadArray+=(0) + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + else + LocalRequiresDownloadArray+=(1) + AnyDownloads=1 + echo -e "${GREEN_BOLD} ✓ Update found! --> $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" + fi + fi + fi + echo +done + +echo -e "${YELLOW_BOLD}4. Downloading New ZIM(s)...${CLEAR}" +echo -e "4. Downloading New ZIM(s)..." >> download.log +echo + +# Let's clear out any possible duplicates + +# Let's Start the download process, but only if we have actual downloads to do. +if [ $AnyDownloads -eq 1 ]; then + for ((z = 0; z < ${#LocalZIMNameArray[@]}; z++)); do # Iterate through the download queue. + [[ ${LocalRequiresDownloadArray[$z]} -eq 0 ]] && continue + + OldZIM=${LocalZIMNameArray[$z]} + OldZIMPath=$ZIMPath$OldZIM + + echo -e "${BLUE_BOLD} Processing $OldZIM${CLEAR}" + + if [[ ${LocalRequiresDownloadArray[$z]} -eq 3 ]]; then + ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") + + NewZIM="$OldZIM" + NewZIMPath="$OldZIMPath" + else + mirror_search # Let's look for a mirror URL first. + + if [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]]; then + ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") + fi + + NewZIM=${RemoteFiles[${LocalZIMRemoteIndexArray[$z]}]} + NewZIMPath=$ZIMPath$NewZIM + fi + + FilePath=$ZIMPath$NewZIM # Set destination path with file name + LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name + + RequiresDownload=0 + + if [ $VERIFY_LIBRARY -eq 0 ]; then + if [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" + echo + else # New ZIM not found, so we'll go ahead and download it. + RequiresDownload=1 + if [[ -f $LockFilePath ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM download was interrupted. Continuing...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM download was interrupted. Continuing...${CLEAR}" + else + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM doesn't exist on disk. Downloading...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM doesn't exist on disk. Downloading...${CLEAR}" + fi + echo + fi + else + # lockfile implies an incomplete download + if [[ -f $LockFilePath ]]; then + echo "Incomplete download detected, resuming" >>download.log + echo -e "${YELLOW_REGULAR} Status: Incomplete download detected, resuming${CLEAR}" + + # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" + # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" + RequiresDownload=1 + else + # actually verify the file + echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" + echo "Calculating checksum for : $OldZIM" >>download.log + [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && echo "$ExpectedHash $OldZIM" >"$OldZIMPath.sha256" + if [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && [[ $(du -b "$OldZIMPath" | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then + RequiresDownload=1 + if [[ $DEBUG -eq 0 ]]; then + if [[ $SKIP_PURGE -eq 0 ]]; then + echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" + echo "✗ Status : File size verification failed, removing corrupt file" >>download.log + rm "$OldZIMPath" + else + echo -e "${RED_BOLD} ✗ Status : File size verification failed but purge skipped${CLEAR}" + echo "✗ Status : File size verification failed but purge skipped" >>download.log + fi + else + [[ $SKIP_PURGE -eq 0 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed, removing corrupt file ($FilePath)${CLEAR}" + [[ $SKIP_PURGE -eq 1 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed but purge skipped ($FilePath)${CLEAR}" + fi + + echo + elif [[ ${#ExpectedHash} -ne 64 ]]; then + echo " This hash doesn't look quite right...skipping" + elif (cd "$ZIMPath" && ! sha256sum --status -c "$OldZIM.sha256"); then + # we checked a very old file, but we will choose to not replace it because we cannot, so we will leave it. A regular update will purge it naturally + [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && echo -e "${RED_BOLD} Checksum failed, online file not found, continuing${CLEAR}" && echo && continue + if [[ $DEBUG -eq 0 ]]; then + if [[ $SKIP_PURGE -eq 0 ]]; then + echo -e "${RED_BOLD} ✗ Status : Checksum failed, removing corrupt file${CLEAR}" + echo "✗ Status : Checksum failed, removing corrupt file" >>download.log + rm "$OldZIMPath" + rm "$OldZIMPath.sha256" 2>/dev/null + else + echo -e "${RED_BOLD} ✗ Status : Checksum failed but purge was skipped${CLEAR}" + echo "✗ Status : Checksum failed but purge was skipped" >>download.log + echo + continue + fi + else + echo -e "${RED_BOLD} ✗ Status : *** Simulated *** Checksum failed, removing corrupt file ($FilePath)${CLEAR}" + fi + + RequiresDownload=1 + echo + else + echo -e "${GREEN_BOLD} ✓ Status : Checksum passed${CLEAR}" + echo "✓ Status : Checksum passed" >>download.log + + [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log + [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log + + # rm "$OldZIMPath.sha256" + + echo + continue + fi + [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && continue + # rm "$OldZIMPath.sha256" + fi + fi + echo >>download.log + [[ $DEBUG -eq 0 ]] && echo "Start : $(date -u)" >>download.log + [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log + # Here is where we actually download the files and log to the download.log file. + if [[ $RequiresDownload -eq 1 ]]; then + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + echo >>download.log + echo "=======================================================================" >>download.log + echo "File : $NewZIM" >>download.log + [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log + [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log + echo >>download.log + + + # Before we actually download, let's just check to see that it isn't already in the folder. + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi + fi + + echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" + if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then + echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" + if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then + if [[ $DEBUG -eq 0 ]]; then + echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" + echo "✗ Status : File size verification failed, removing corrupt file" >>download.log + rm "$NewZIMPath" + else + echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" + fi + elif [[ ${#ExpectedHash} -ne 64 ]]; then + echo -e "${YELLOW_BOLD} This hash doesn't look quite right...skipping${CLEAR}" + elif [[ $DEBUG -eq 0 ]] && (cd "$ZIMPath" && ! sha256sum --status -c "$NewZIM.sha256"); then + echo -e "${RED_BOLD} ✗ Checksum failed, removing corrupt file${CLEAR}" + rm "$NewZIMPath" + touch "$NewZIMPath" + DownloadFailed=1 + else + if [[ $DEBUG -eq 0 ]]; then + echo -e "${GREEN_BOLD} ✓ Checksum passed${CLEAR}" + else + echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" + fi + fi + # rm "$NewZIMPath.sha256" + # rm "$LockFilePath" + echo + fi + + echo >> download.log + [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log + + # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later + if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then + [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log + [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log + [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" + continue + fi + + [[ $RequiresDownload -eq 1 ]] && [[ $DEBUG -eq 0 ]] && rm "$LockFilePath" + + ######################################## + + echo -e "${BLUE_REGULAR} Old : $OldZIM${CLEAR}" + echo "Old : $OldZIM" >>download.log + echo -e "${BLUE_BOLD} New : $NewZIM${CLEAR}" + echo "New : $NewZIM" >>download.log + # Check for the new ZIM on disk. + if [[ -f "$NewZIMPath" ]]; then # New ZIM found + if [[ $DEBUG -eq 0 ]]; then + if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" + echo "✓ Status : New ZIM downloaded succesfully." >>download.log + # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM + else + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" + echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log + [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM + fi + else + echo -e "${GREEN_BOLD} ✓ Status : *** Simulated ***${CLEAR}" + echo "✓ Status : *** Simulated ***" >>download.log + fi + else # New ZIM not found. Something went wrong, so we will skip this purge. + if [[ $DEBUG -eq 0 ]]; then + echo -e "${RED_BOLD} ✗ Status : New ZIM failed verification. Old ZIM not purged.${CLEAR}" + echo "✗ Status : New ZIM failed verification. Old ZIM not purged." >>download.log + else + if [[ $RequiresDownload -eq 1 ]]; then + echo -e "${GREEN_BOLD} ✓ Status : *** Simulated *** New zim exists, old zim purged${CLEAR}" + echo "✓ Status : *** Simulated *** New zim exists, old zim purged" >>download.log + else + echo -e "${YELLOW_BOLD} ✗ Status : *** Simulated *** Zim was skipped, and will not be purged${CLEAR}" + echo "✗ Status : *** Simulated *** Zim not purged" >>download.log + fi + fi + fi + echo + echo >>download.log + + ######################################### + + [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log + [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log + done + + if [[ $DEBUG -eq 0 ]]; then + IFS=$'\n' HangingFileLocks=("$ZIMPath".~lock.*.zim) + unset IFS + + if [[ ${#HangingFileLocks[@]} -gt 0 ]]; then + echo -e "${YELLOW_BOLD}5. Cleaning up...${CLEAR}" + echo "5. Cleaning up..." >> download.log + echo + + for ((i = 0; i < ${#HangingFileLocks[@]}; i++)); do + baseFileName=$(echo "${HangingFileLocks[$i]}" | grep -ioP "(?<=\.~lock\.).*$") + echo -e " ${RED_BOLD}Found broken lock: ${HangingFileLocks[$i]}${CLEAR}" + echo "Found broken lock: ${HangingFileLocks[$i]}" >> download.log + if [[ -f "$ZIMPath$baseFileName" ]]; then + echo -e " ${BLUE_BOLD}Found abandoned download: $ZIMPath$baseFileName${CLEAR}" + echo -e "Found abandoned download: $ZIMPath$baseFileName" >> download.log + rm "$ZIMPath$baseFileName" + fi + if [[ -f "$ZIMPath$baseFileName.sha256" ]]; then + echo -e " ${BLUE_BOLD}Found abandoned checksum: $ZIMPath$baseFileName.sha256${CLEAR}" + echo -e "Found abandoned checksum: $ZIMPath$baseFileName.sha256" >> download.log + rm "$ZIMPath$baseFileName.sha256" + fi + rm "${HangingFileLocks[$i]}" + echo + done + fi + fi +else + echo -e "${GREEN_REGULAR} ✓ Download: Nothing to download.${CLEAR}" + echo "✓ Download: Nothing to download." >> download.log + echo +fi + +if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then + echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01.\n This exit code is nonzero, but the script has not errored out. This is simply so you may notice and switch to using kiwix-zim-updater " >> download.log + exit 2 +fi \ No newline at end of file