From 1345a1598bd1329db645bb69d1dd0b70e1c0c856 Mon Sep 17 00:00:00 2001 From: jojo2357 <66704796+jojo2357@users.noreply.github.com> Date: Mon, 25 May 2026 23:37:14 -0700 Subject: [PATCH] v3.4: Fix library urls (#58) * Remove deprecated kiwix-zim.sh * change shebang * update urls * extract catalog url * correct typos Co-authored-by: 4754442+jbnance@users.noreply.github.com * update readme * remove useless `|&` * add mac disclaimer * add copyright * bump version number * minor bugs * teeny improvements * miniscule changes --- README.md | 8 +- kiwix-zim-updater.sh | 89 +++-- kiwix-zim.sh | 874 ------------------------------------------- 3 files changed, 59 insertions(+), 912 deletions(-) delete mode 100755 kiwix-zim.sh diff --git a/README.md b/README.md index 051032d..e9a0c70 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,7 @@ A script to check `download.kiwix.org` for updates to your local ZIM library. Just pass this script your ZIM directory and away it goes. *(see Usage below)* -Tested on PopOS! 22.04, and should work out of the box on most debian systems, but I have not tested that. - -# DEPRECATION WARNING -`kiwix-zim.sh` has been deprecated in favor of the more descriptive `kiwix-zim-updater.sh`. A hard link from `kiwix-zim.sh` to `kiwix-zim-updater.sh` will be maintained on this repo until at least `2025-01-01T00:00:00Z` for compatability with CRON users. - -***CALLING `kiwix-zim.sh` WILL RESULT IN AN EXIT CODE OF 2 ON SUCCESSFUL EXECUTION. THIS BEHAVIOR IS EXPECTED. CALL `kiwix-zim-updater.sh` INSTEAD TO GET A 0 EXIT CODE*** +Tested on Debian 13, and should work out of the box on most debian systems, but I have not tested that. ## What It Does @@ -58,6 +53,7 @@ Creates `downloads.log` for the following reasons: ## Limitations +- This script relies heavily on `grep`'s `-P` flag for lookarounds and the matcher reset, `\K`. This may cause compatibility problems on macOS - This script is only for ZIM(s) hosted by `download.kiwix.org` due to the file naming standard they use. If you have self-made ZIM(s) or ZIM(s) downloaded from somewhere else, they most likely do not use the same naming standards and will not be processed by this script. - If you have ZIM(s) from `download.kiwix.org`, but you have changed their file names, this script will treat them like the previous limitation explains. - This script does not attempt to update any `library.xml` that may or may not exist/be needed for your install/setup of Kiwix. If needed, you'll need to handle this part on your own. diff --git a/kiwix-zim-updater.sh b/kiwix-zim-updater.sh index 5c6ef15..571b4f7 100755 --- a/kiwix-zim-updater.sh +++ b/kiwix-zim-updater.sh @@ -1,6 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -VER="3.3" +# Kiwix-Zim-Updater. A script to update your kiwix library +# Copyright (C) 2022 DocDrydenn +# Copyright (C) 2023 jojo2357 +# SPDX-License-Identifier: GPL-2.0-only + +VER="3.4" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -8,7 +13,7 @@ LocalZIMArray=() LocalZIMNameArray=() # This array will map the local zim to the index in the remote arrays that contains the same base file name LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop +# This array is a boolean array which remembers if a given local zim should be processed in the download loop LocalRequiresDownloadArray=() # After updating, this array will be used to store hanging locks and to deal with them HangingFileLocks=(); @@ -40,7 +45,9 @@ CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 DOWNLOAD_METHOD=1 # 1: web 2: torrent -BaseURL="https://download.kiwix.org/zim/" +# sort of a default, will be overwritten when the library is parsed. Set here to what it probably will be +BaseURL="https://lb.download.kiwix.org/zim/" +CatalogURL="https://opds.library.kiwix.org/catalog/v2/entries?count=-1" ZIMPath="" RED_REGULAR="\033[0;31m" @@ -72,9 +79,10 @@ master_scrape() { fi if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary - RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" - + # both write the file timestamp to the index file and save all of the links to RawLibrary. + # Lets defer touching the index until we have actually fetched it + RawLibrary="$(wget --show-progress -q -O - "$CatalogURL")" + RawLibrary="$(echo "$RawLibrary" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" echo "$RawLibrary" >> kiwix-index else RawLibrary=$(grep -i '> download.log @@ -371,7 +396,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" >> download.log + echo "Invalid country code, falling back to default kiwix behavior" >> download.log fi shift # discard value ;; @@ -379,7 +404,7 @@ while [[ $# -gt 0 ]]; do CALCULATE_CHECKSUM=1 shift ;; - -f | --verfiy-library) + -f | --verfiy-library | --verify-library) VERIFY_LIBRARY=1 CALCULATE_CHECKSUM=1 shift @@ -707,45 +732,45 @@ if [ $AnyDownloads -eq 1 ]; then [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log # Here is where we actually download the files and log to the download.log file. if [[ $RequiresDownload -eq 1 ]]; then + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + + echo >>download.log + echo "=======================================================================" >>download.log + echo "File : $NewZIM" >>download.log + [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log + [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log + echo >>download.log + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then FilePath="$FilePath.torrent" if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log fi continue else - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log - - # Before we actually download, let's just check to see that it isn't already in the folder. if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log else # New ZIM not found, so we'll go ahead and download it. [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log fi fi fi - echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" + [[ $DEBUG -eq 0 ]] && echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then @@ -778,12 +803,12 @@ if [ $AnyDownloads -eq 1 ]; then echo >> download.log [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later + # in all of these cases, we will not re-purge and will leave the lockfile so we know to resume later if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" - [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" continue fi @@ -799,12 +824,12 @@ if [ $AnyDownloads -eq 1 ]; then if [[ -f "$NewZIMPath" ]]; then # New ZIM found if [[ $DEBUG -eq 0 ]]; then if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully." >>download.log # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM else - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully. Old ZIM purged.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully. Old ZIM purged." >>download.log [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM fi else diff --git a/kiwix-zim.sh b/kiwix-zim.sh deleted file mode 100755 index 5c6ef15..0000000 --- a/kiwix-zim.sh +++ /dev/null @@ -1,874 +0,0 @@ -#!/bin/bash - -VER="3.3" - -# This array will contain all of the local zims, with the file extension -LocalZIMArray=() -# This array will contain all of the local zims, without the file extension -LocalZIMNameArray=() -# This array will map the local zim to the index in the remote arrays that contains the same base file name -LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop -LocalRequiresDownloadArray=() -# After updating, this array will be used to store hanging locks and to deal with them -HangingFileLocks=(); - -# This array stores the file names that kiwix has to offer, with .zim extensions -RemoteFiles=() -# Ditto, without the YYYY-MM (note a trailing _) -Basenames=() -# Contains the absolute path to this file, from /zims/ -RemotePaths=() -# Contains the folder this file is in relative to /zims/ -RemoteCategory=() - -# Set Script Strings -SCRIPT="$(readlink -f "$0")" -SCRIPTFILE="$(basename "$SCRIPT")" -SCRIPTPATH="$(dirname "$SCRIPT")" -CALLEDSCRIPTNAME="$0" -CALLEDSCRIPTFILE="$(basename "$CALLEDSCRIPTNAME")" -CALLEDSCRIPTPATH="$(dirname "$CALLEDSCRIPTNAME")" -ARGS=("$@") -BRANCH="main" -SKIP_UPDATE=0 -DEBUG=1 # This forces the script to default to "dry-run/simulation mode" -MIN_SIZE=0 -MAX_SIZE=0 -CALCULATE_CHECKSUM=0 -CHECKSUM_FILES=1 -VERIFY_LIBRARY=0 -FORCE_FETCH_INDEX=0 -DOWNLOAD_METHOD=1 # 1: web 2: torrent -BaseURL="https://download.kiwix.org/zim/" -ZIMPath="" - -RED_REGULAR="\033[0;31m" -RED_BOLD="\033[1;31m" -YELLOW_REGULAR="\033[0;33m" -YELLOW_BOLD="\033[1;33m" -GREEN_REGULAR="\033[0;32m" -GREEN_BOLD="\033[1;32m" -BLUE_REGULAR="\033[0;34m" -BLUE_BOLD="\033[1;34m" -CLEAR="\033[0m" - -# This will ask the api what files it has to offer and store them in arrays -master_scrape() { - unset RemoteFiles - unset Basenames - unset RemotePaths - unset RemoteCategory - - indexIsValid=1 - - if [[ ! -f kiwix-index ]]; then - indexIsValid=0 - else - indexDate="$(head -1 "kiwix-index")" - if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then - indexIsValid=0 - fi - fi - - if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary - RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" - - echo "$RawLibrary" >> kiwix-index - else - RawLibrary=$(grep -i '> download.log - exit 0 - else - echo -e "${GREEN_BOLD} ✓ Found ${#RemoteFiles[@]} files online" - echo "✓ Found ${#RemoteFiles[@]} files online" >> download.log - fi - - # Housekeeping... - unset RawLibrary - unset hrefs -} - -# self_update - Script Self-Update Function -self_update() { - echo -e "${YELLOW_BOLD}1. Checking for Script Updates...${CLEAR}" - echo "1. Checking for Script Updates..." >> download.log - echo - # Check if script path is a git clone. - # If true, then check for update. - # If false, skip self-update check/funciton. - if [ $SKIP_UPDATE -eq 1 ]; then - echo -e "${YELLOW_REGULAR} Check Skipped${CLEAR}" - echo "Check Skipped" >> download.log - elif [[ -d "$SCRIPTPATH/.git" ]]; then - echo -e "${GREEN_BOLD} ✓ Git Clone Detected: Checking Script Version...${CLEAR}" - echo "✓ Git Clone Detected: Checking Script Version..." >> download.log - cd "$SCRIPTPATH" || exit 1 - [[ $(timeout 1s git rev-parse --abbrev-ref HEAD) != "$BRANCH" ]] && echo -e "${YELLOW_BOLD} You appear to be on a different branch so I will assume you are developing and do not want an update${CLEAR}" && echo && return - timeout 1s git fetch --quiet - timeout 1s git diff --quiet --exit-code "origin/$BRANCH" "$SCRIPTFILE" - [ $? -eq 1 ] && { - echo -e "${RED_REGULAR} ✗ Version: Mismatched${CLEAR}" - echo "✗ Version: Mismatched" >> download.log - echo - echo -e "${YELLOW_BOLD}1a. Fetching Update...${CLEAR}" - echo "1a. Fetching Update..." >> download.log - echo - if [ -n "$(git status --porcelain)" ]; then - git stash push -m 'local changes stashed before self update' --quiet - fi - git pull --force --quiet - git checkout $BRANCH --quiet - git pull --force --quiet - echo -e "${GREEN_BOLD} ✓ Update Complete. Running New Version. Standby...${CLEAR}" - echo "✓ Update Complete. Running New Version. Standby..." >> download.log - sleep 3 - cd - >/dev/null || exit 1 - - exec "$CALLEDSCRIPTNAME" "${ARGS[@]}" - - # Exit this old instance of the script - exit 1 - } - echo -e "${GREEN_BOLD} ✓ Version: Current${CLEAR}" - else - echo -e "${RED_REGULAR} ✗ Git Clone Not Detected: Skipping Update Check${CLEAR}" - fi - echo -} - -# usage_example - Show Usage and Exit -usage_example() { - echo 'Usage: ./kiwix-zim-updater.sh /full/path/' - echo '' - echo ' /full/path/ Full path to ZIM directory' - echo '' - echo 'Universal Options:' - echo ' -h, --help Show this usage and exit.' - echo ' -u, --skip-update Skips checking for script updates (very useful for development).' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' - echo ' -n , --min-size Minimum ZIM Size to be downloaded.' - echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' - echo ' -x , --max-size Maximum ZIM Size to be downloaded.' - echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' - echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' - echo ' ' - echo 'Action Method Options:' - echo ' -w, --web Downloads zims over http(s).' - echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' - echo ' ' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' - echo ' Disable this using -S' - echo '' - echo 'Web Download Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' - echo ' -l , --location Country Code to prefer mirrors from' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - - echo - exit 0 -} - -# flags - Flag and ZIM Processing Functions -flags() { - echo -e "${YELLOW_BOLD}2. Preprocessing...${CLEAR}" - echo "2. Preprocessing..." >> download.log - echo - echo -e "${BLUE_BOLD} -Validating ZIM directory...${CLEAR}" - echo "Validating ZIM directory..." >> download.log - - # Let's identify which argument is the ZIM directory path and if it's an actual directory. - if [[ -d ${1} ]]; then - if [[ -w ${1} ]]; then - ZIMPath=$1 - else - ZIMPath=$1 - echo -e "${RED_REGULAR} ✗ Cannot write to '${1}', continuing in dry-run${CLEAR}" - echo "✗ Cannot write to '${1}', continuing in dry-run" >> download.log - echo - DEBUG=1 - fi - else # Um... no ZIM directory path provided? Okay, let's show the usage and exit. - if [[ -z ${1} ]]; then - echo -e "${RED_REGULAR} ✗ Kiwix ZIM Directory not provided${CLEAR}" - echo "✗ Kiwix ZIM Directory not provided" >> download.log - else - echo -e "${RED_REGULAR} ✗ '$1' is not a directory${CLEAR}" - echo "✗ '$1' is not a directory" >> download.log - fi - echo - usage_example - fi - - # Check for and add if missing, trailing slash. - [[ "${ZIMPath}" != */ ]] && ZIMPath="${ZIMPath}/" - - # Now we need to check for ZIM files. - shopt -s nullglob # This is in case there are no matching files - - # Load all found ZIM(s) w/path into LocalZIMArray - IFS=$'\n' read -r -d '' -a LocalZIMArray < <(ls -1 "$ZIMPath" | grep -iP "\.zim$") - unset IFS - - for index in "${!LocalZIMArray[@]}" ; do - duplicated=0 - myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") - for scanIndex in "${!LocalZIMArray[@]}"; do - if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then - if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi - else - if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi - fi - scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") - - if [[ "$myBasename" == "$scanBasename" ]]; then - if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then - echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log - elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then - echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log - else - echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log - fi - duplicated=1 - break - fi - done - [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' - done - LocalZIMArray=("${LocalZIMArray[@]}") - - # Check that ZIM(s) were actually found/loaded. - if [ ${#LocalZIMArray[@]} -eq 0 ]; then # No ZIM(s) were found in the directory... I guess there's nothing else for us to do, so we'll Exit. - echo -e "${RED_REGULAR} ✗ No ZIMs found. Exiting...${CLEAR}" - exit 0 - else - echo -e "${GREEN_BOLD} ✓ Valid ZIM Directory ${CLEAR}" - fi - echo - - echo -e "${BLUE_BOLD} -Building online ZIM list...${CLEAR}" - - # Build online ZIM list. - master_scrape - - echo - - # Populate ZIM arrays from found ZIM(s) - echo -e "${BLUE_BOLD} -Parsing ZIM(s)...${CLEAR}" - - for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). - LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. - filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. - # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' - - # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL - for ((z = 0; z < ${#Basenames[@]}; z++)); do - if [[ ${Basenames[$z]} == "$filename" ]]; then # Match Found (ignore the filename datepart). - LocalZIMRemoteIndexArray[$i]="$z" - break - else # No Match Found. - LocalZIMRemoteIndexArray[$i]="-1" - fi - done - - if [[ LocalZIMRemoteIndexArray[$i] -eq -1 ]]; then - echo -e "${RED_REGULAR} ✗ ${LocalZIMNameArray[$i]} No online match found.${CLEAR}" - else - echo -e "${GREEN_BOLD} ✓ ${LocalZIMNameArray[$i]} [${RemoteCategory[${LocalZIMRemoteIndexArray[$i]}]}]${CLEAR}" - fi - done - - echo - echo -e "${GREEN_REGULAR} ${#LocalZIMNameArray[*]} ZIM(s) found.${CLEAR}" -} - -# mirror_search - Find ZIM URL Priority #1 mirror from meta4 Function -mirror_search() { - IsMirror=0 - DownloadURL="" - RemotePath="${RemotePaths[${LocalZIMRemoteIndexArray[$z]}]}" - ExpectedSize="${FileSizes[${LocalZIMRemoteIndexArray[$z]}]}" - - # If we need the checksum, we need a link and the hash, which we can get both by using .meta4, otherwise we only need - # Silently fetch (via wget) the associated meta4 xml and extract the mirror URL marked priority="1" - MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") - ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') - ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') - - # still run the last wget so that the logic is simple for verification. - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL - return - fi - - RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') - - # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. - if [[ $RawMirror == *"http"* ]]; then # Mirror URL found - DownloadURL="$RawMirror" # Set the mirror URL as our download URL - IsMirror=1 - else # Mirror URL not found - DownloadURL="$BaseURL$RemotePath" # Set the direct download URL as our download URL - fi -} - -######################### -# Begin Script Execute -######################### - -while [[ $# -gt 0 ]]; do - case $1 in - -h | --help) - usage_example - ;; - -d | --disable-dry-run) - DEBUG=0 - shift # discard argument - ;; - -v | --version) - echo "$VER" - exit 0 - ;; - -p | --skip-purge) - SKIP_PURGE=1 - shift # discard argument - ;; - -n | --min-size) - shift # discard -n argument - MIN_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes - shift # discard value - ;; - -x | --max-size) - shift # discard -x argument - MAX_SIZE=$(numfmt --from=auto "$1") # convert passed arg to bytes - shift # discard value - ;; - -l | --location) - shift # discard -l argument - if [[ "$1" =~ ^[A-Z]{2}$ ]]; then - COUNTRY_CODE=$1 # convert passed arg to bytes - else - COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" >> download.log - fi - shift # discard value - ;; - -c | --calculate-checksum) - CALCULATE_CHECKSUM=1 - shift - ;; - -f | --verfiy-library) - VERIFY_LIBRARY=1 - CALCULATE_CHECKSUM=1 - shift - ;; - -u | --skip-update) - SKIP_UPDATE=1 - shift - ;; - -g | --get-index) - FORCE_FETCH_INDEX=1 - shift - ;; - -w | --web) - DOWNLOAD_METHOD=1 - shift - ;; - -t | --torrent) - DOWNLOAD_METHOD=2 - shift - ;; - -S | --no-sha) - CHECKSUM_FILES=0 - shift - ;; - *) - # We can either parse the arg here, or just tuck it away for safekeeping - POSITIONAL_ARGS+=("$1") # save positional arg - shift # past argument - ;; - esac -done - -set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped earlier - -clear # Clear screen - -if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then - echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" -fi - -# Display Header -echo "==========================================" -echo " kiwix-zim-updater" -if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then - echo " WARNING: kiwix-zim.sh has been deprecated " - echo " in favor of kiwix-zim-updater " - echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01" >> download.log -fi -echo " download.kiwix.org ZIM Updater" -echo -echo " v$VER by DocDrydenn and jojo2357" -echo "==========================================" -echo -echo " DRY-RUN/SIMULATION" -[[ $DEBUG -eq 1 ]] && echo " - ENABLED -" -[[ $DEBUG -eq 1 ]] && echo -[[ $DEBUG -eq 1 ]] && echo " Use '-d' to disable." -[[ $DEBUG -eq 0 ]] && echo " - DISABLED -" -[[ $DEBUG -eq 0 ]] && echo -[[ $DEBUG -eq 0 ]] && echo " !!! Caution !!!" -echo -echo "==========================================" -echo - -# First, Self-Update Check. -# Shouldnt this be first? it is not dependent on anything else and resets everything, so may as well reset it before getting all invested? -self_update - -# Second, Flag Check. -flags "$@" - -echo -echo -e "${YELLOW_BOLD}3. Processing ZIM(s)...${CLEAR}" -echo "3. Processing ZIM(s)..." >> download.log -echo - -AnyDownloads=0 - -for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do - RemoteIndex=${LocalZIMRemoteIndexArray[$i]} - if [[ $RemoteIndex -eq -1 ]]; then - if [[ $VERIFY_LIBRARY -eq 1 ]] && [[ -f "$FileName.sha256" ]]; then - LocalRequiresDownloadArray+=(3) - AnyDownloads=1 - echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" - echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - fi - continue - fi - - FileName=${LocalZIMNameArray[$i]} - echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" - [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue - - - MatchingSize=${FileSizes[$RemoteIndex]} - MatchingFileName=${RemoteFiles[$RemoteIndex]} - MatchingFullPath=${RemotePaths[$RemoteIndex]} - MatchingCategory=${RemoteCategory[$RemoteIndex]} - - [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue - - MatchedDate="$(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - MatchedYear="$(echo "$MatchedDate" | grep -oP '\d{4}(?=-\d{2})')" - MatchedMonth="$(echo "$MatchedDate" | grep -oP '(?<=\d{4}-)\d{2}')" - - LocalDate="$(echo "$FileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')" - LocalYear="$(echo "$LocalDate" | grep -oP '\d{4}(?=-\d{2})')" - LocalMonth="$(echo "$LocalDate" | grep -oP '(?<=\d{4}-)\d{2}')" - - FileTooSmall=0 - [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 - FileTooLarge=0 - [[ $MAX_SIZE -gt 0 ]] && [[ $MatchingSize -gt $MAX_SIZE ]] && FileTooLarge=1 - FileSizeAcceptable=0 - [ $FileTooSmall -eq 0 ] && [ $FileTooLarge -eq 0 ] && FileSizeAcceptable=1 - - if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then - if [ $FileTooSmall -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - elif [ $FileTooLarge -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" - fi - elif [[ "$MatchingFileName" == "$FileName" ]]; then - if [ $VERIFY_LIBRARY -eq 1 ]; then - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - fi - else - if [ $VERIFY_LIBRARY -eq 1 ]; then - if [[ -f "$ZIMPath$FileName.sha256" ]]; then - LocalRequiresDownloadArray+=(2) - AnyDownloads=1 - echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" - else - echo " Checking for online checksum..." - if wget -S --spider -q -O - "$BaseURL$MatchingCategory/$FileName.sha256" >/dev/null 2>&1; then - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" - else - LocalRequiresDownloadArray+=(0) - echo " ✗ Online Version Not Found" - fi - fi - else - if [ $FileTooSmall -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [ $FileTooLarge -eq 1 ]; then - LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')${CLEAR}" - elif [ "$MatchedYear" -lt "$LocalYear" ]; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - elif [ "$MatchedYear" -eq "$LocalYear" ] && [ "$MatchedMonth" -le "$LocalMonth" ]; then - LocalRequiresDownloadArray+=(0) - echo " ✗ No new update" - else - LocalRequiresDownloadArray+=(1) - AnyDownloads=1 - echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" - fi - fi - fi - echo -done - -echo -e "${YELLOW_BOLD}4. Downloading New ZIM(s)...${CLEAR}" -echo -e "4. Downloading New ZIM(s)..." >> download.log -echo - -# Let's clear out any possible duplicates - -# Let's Start the download process, but only if we have actual downloads to do. -if [ $AnyDownloads -eq 1 ]; then - for ((z = 0; z < ${#LocalZIMNameArray[@]}; z++)); do # Iterate through the download queue. - [[ ${LocalRequiresDownloadArray[$z]} -eq 0 ]] && continue - - OldZIM=${LocalZIMNameArray[$z]} - OldZIMPath=$ZIMPath$OldZIM - - echo -e "${BLUE_BOLD} Processing $OldZIM${CLEAR}" - - if [[ ${LocalRequiresDownloadArray[$z]} -eq 3 ]]; then - ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") - - NewZIM="$OldZIM" - NewZIMPath="$OldZIMPath" - else - mirror_search # Let's look for a mirror URL first. - - if [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]]; then - ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") - fi - - NewZIM=${RemoteFiles[${LocalZIMRemoteIndexArray[$z]}]} - NewZIMPath=$ZIMPath$NewZIM - fi - - - FilePath=$ZIMPath$NewZIM # Set destination path with file name - LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name - TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name - - RequiresDownload=0 - - if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - if [[ -f $NewZIM ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" - echo - elif [[ -f $TorrentFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" - echo - else - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" - RequiresDownload=1 - fi - elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" - echo - else # New ZIM not found, so we'll go ahead and download it. - RequiresDownload=1 - if [[ -f $LockFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM download was interrupted. Continuing...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM download was interrupted. Continuing...${CLEAR}" - else - [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM doesn't exist on disk. Downloading...${CLEAR}" - [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM doesn't exist on disk. Downloading...${CLEAR}" - fi - echo - fi - else - # lockfile implies an incomplete download - if [[ -f $LockFilePath ]]; then - echo "Incomplete download detected, resuming" >>download.log - echo -e "${YELLOW_REGULAR} Status: Incomplete download detected, resuming${CLEAR}" - - # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" - # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" - RequiresDownload=1 - elif [[ -f $TorrentFilePath ]]; then - echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" - echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log - else - # actually verify the file - echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" - echo "Calculating checksum for : $OldZIM" >>download.log - [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && echo "$ExpectedHash $OldZIM" >"$OldZIMPath.sha256" - if [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && [[ $(du -b "$OldZIMPath" | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then - RequiresDownload=1 - if [[ $DEBUG -eq 0 ]]; then - if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" - echo "✗ Status : File size verification failed, removing corrupt file" >>download.log - rm "$OldZIMPath" - else - echo -e "${RED_BOLD} ✗ Status : File size verification failed but purge skipped${CLEAR}" - echo "✗ Status : File size verification failed but purge skipped" >>download.log - fi - else - [[ $SKIP_PURGE -eq 0 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed, removing corrupt file ($FilePath)${CLEAR}" - [[ $SKIP_PURGE -eq 1 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed but purge skipped ($FilePath)${CLEAR}" - fi - - echo - elif [[ ${#ExpectedHash} -ne 64 ]]; then - echo " This hash doesn't look quite right...skipping" - elif (cd "$ZIMPath" && ! sha256sum --status -c "$OldZIM.sha256"); then - # we checked a very old file, but we will choose to not replace it because we cannot, so we will leave it. A regular update will purge it naturally - [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && echo -e "${RED_BOLD} Checksum failed, online file not found, continuing${CLEAR}" && echo && continue - if [[ $DEBUG -eq 0 ]]; then - if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : Checksum failed, removing corrupt file${CLEAR}" - echo "✗ Status : Checksum failed, removing corrupt file" >>download.log - rm "$OldZIMPath" - rm "$OldZIMPath.sha256" 2>/dev/null - else - echo -e "${RED_BOLD} ✗ Status : Checksum failed but purge was skipped${CLEAR}" - echo "✗ Status : Checksum failed but purge was skipped" >>download.log - echo - continue - fi - else - echo -e "${RED_BOLD} ✗ Status : *** Simulated *** Checksum failed, removing corrupt file ($FilePath)${CLEAR}" - fi - - RequiresDownload=1 - echo - else - echo -e "${GREEN_BOLD} ✓ Status : Checksum passed${CLEAR}" - echo "✓ Status : Checksum passed" >>download.log - - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - - # rm "$OldZIMPath.sha256" - - echo - continue - fi - [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && continue - # rm "$OldZIMPath.sha256" - fi - fi - echo >>download.log - [[ $DEBUG -eq 0 ]] && echo "Start : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log - # Here is where we actually download the files and log to the download.log file. - if [[ $RequiresDownload -eq 1 ]]; then - if [[ $DOWNLOAD_METHOD -eq 2 ]]; then - FilePath="$FilePath.torrent" - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi - - continue - else - [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" - [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" - echo >>download.log - echo "=======================================================================" >>download.log - echo "File : $NewZIM" >>download.log - [[ $IsMirror -eq 0 ]] && echo "URL (direct) : $DownloadURL" >>download.log - [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log - echo >>download.log - - - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log - fi - fi - fi - - echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" - if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then - echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" - if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then - if [[ $DEBUG -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" - echo "✗ Status : File size verification failed, removing corrupt file" >>download.log - rm "$NewZIMPath" - else - echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" - fi - elif [[ ${#ExpectedHash} -ne 64 ]]; then - echo -e "${YELLOW_BOLD} This hash doesn't look quite right...skipping${CLEAR}" - elif [[ $DEBUG -eq 0 ]] && (cd "$ZIMPath" && ! sha256sum --status -c "$NewZIM.sha256"); then - echo -e "${RED_BOLD} ✗ Checksum failed, removing corrupt file${CLEAR}" - rm "$NewZIMPath" - touch "$NewZIMPath" - DownloadFailed=1 - else - if [[ $DEBUG -eq 0 ]]; then - echo -e "${GREEN_BOLD} ✓ Checksum passed${CLEAR}" - else - echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" - fi - fi - # rm "$NewZIMPath.sha256" - # rm "$LockFilePath" - echo - fi - - echo >> download.log - [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - - # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later - if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" - [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" - continue - fi - - [[ $RequiresDownload -eq 1 ]] && [[ $DEBUG -eq 0 ]] && rm "$LockFilePath" - - ######################################## - - echo -e "${BLUE_REGULAR} Old : $OldZIM${CLEAR}" - echo "Old : $OldZIM" >>download.log - echo -e "${BLUE_BOLD} New : $NewZIM${CLEAR}" - echo "New : $NewZIM" >>download.log - # Check for the new ZIM on disk. - if [[ -f "$NewZIMPath" ]]; then # New ZIM found - if [[ $DEBUG -eq 0 ]]; then - if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully." >>download.log - # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM - else - echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.${CLEAR}" - echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log - [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM - fi - else - echo -e "${GREEN_BOLD} ✓ Status : *** Simulated ***${CLEAR}" - echo "✓ Status : *** Simulated ***" >>download.log - fi - else # New ZIM not found. Something went wrong, so we will skip this purge. - if [[ $DEBUG -eq 0 ]]; then - echo -e "${RED_BOLD} ✗ Status : New ZIM failed verification. Old ZIM not purged.${CLEAR}" - echo "✗ Status : New ZIM failed verification. Old ZIM not purged." >>download.log - else - if [[ $RequiresDownload -eq 1 ]]; then - echo -e "${GREEN_BOLD} ✓ Status : *** Simulated *** New zim exists, old zim purged${CLEAR}" - echo "✓ Status : *** Simulated *** New zim exists, old zim purged" >>download.log - else - echo -e "${YELLOW_BOLD} ✗ Status : *** Simulated *** Zim was skipped, and will not be purged${CLEAR}" - echo "✗ Status : *** Simulated *** Zim not purged" >>download.log - fi - fi - fi - echo - echo >>download.log - - ######################################### - - [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log - [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - done - - if [[ $DEBUG -eq 0 ]]; then - IFS=$'\n' HangingFileLocks=("$ZIMPath".~lock.*.zim) - unset IFS - - if [[ ${#HangingFileLocks[@]} -gt 0 ]]; then - echo -e "${YELLOW_BOLD}5. Cleaning up...${CLEAR}" - echo "5. Cleaning up..." >> download.log - echo - - for ((i = 0; i < ${#HangingFileLocks[@]}; i++)); do - baseFileName=$(echo "${HangingFileLocks[$i]}" | grep -ioP "(?<=\.~lock\.).*$") - echo -e " ${RED_BOLD}Found broken lock: ${HangingFileLocks[$i]}${CLEAR}" - echo "Found broken lock: ${HangingFileLocks[$i]}" >> download.log - if [[ -f "$ZIMPath$baseFileName" ]]; then - echo -e " ${BLUE_BOLD}Found abandoned download: $ZIMPath$baseFileName${CLEAR}" - echo -e "Found abandoned download: $ZIMPath$baseFileName" >> download.log - rm "$ZIMPath$baseFileName" - fi - if [[ -f "$ZIMPath$baseFileName.sha256" ]]; then - echo -e " ${BLUE_BOLD}Found abandoned checksum: $ZIMPath$baseFileName.sha256${CLEAR}" - echo -e "Found abandoned checksum: $ZIMPath$baseFileName.sha256" >> download.log - rm "$ZIMPath$baseFileName.sha256" - fi - rm "${HangingFileLocks[$i]}" - echo - done - fi - fi -else - echo -e "${GREEN_REGULAR} ✓ Download: Nothing to download.${CLEAR}" - echo "✓ Download: Nothing to download." >> download.log - echo -fi - -if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then - echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01.\n This exit code is nonzero, but the script has not errored out. This is simply so you may notice and switch to using kiwix-zim-updater " >> download.log - exit 2 -fi \ No newline at end of file