diff --git a/scripts/index-sync-rotate.sh b/scripts/index-sync-rotate.sh index bc08c669..034f137f 100755 --- a/scripts/index-sync-rotate.sh +++ b/scripts/index-sync-rotate.sh @@ -39,7 +39,8 @@ SPHINX_FILE_EXTENSIONS=('spa' 'spd' 'spe' 'sph' 'spi' 'spk' 'spm' 'spp' 'sps') SPHINX_INDEX_READY=('spd' 'spe' 'sph' 'spi' 'spp' 'sps') SPHINX_INDEXES=$(grep -E "^[^#]+ path" "${SPHINXCONFIG}" | awk -F"=" '{print $2}' | sed -n -e 's|^.*/||p') -LOCKFILE="/var/lock/$(basename "$0")" +# global locking, no parallel syncing from efs into docker volumes +LOCKFILE="${SPHINX_EFS}$(basename "$0")" LOCKFD=99 # PRIVATE @@ -114,61 +115,40 @@ for sphinx_index in ${SPHINX_INDEXES[@]}; do check_if_index_is_ready "${sphinx_index}" || { echo "${LOG_PREFIX}-> $(date +"%F %T") skipping partially updated index: ${sphinx_index} ..."; continue; } # sync EFS to VOLUME - # do not delete anything in local volume, in case the efs has been cleaned / removed indexes will still exist in local storage - echo "${LOG_PREFIX}-> $(date +"%F %T") start sync: ${sphinx_index} ..." - rsync --update -av --include-from "${RSYNC_INCLUDE}" --exclude '*' ${SPHINX_EFS} ${SPHINX_VOLUME} - - # rename new files from *.sp* to .*.new.sp* - echo "${LOG_PREFIX}-> $(date +"%F %T") rename already rotated, new index files: ${new_files[*]}..." - # strip file extension from filename and expand it to the full list of existing files in SPHINX Volume - pushd "${SPHINX_VOLUME}" + echo "${LOG_PREFIX}-> $(date +"%F %T") start sync and rename files in target folder: ${sphinx_index} ..." tmp_array=() - for new_file in "${new_files[@]}";do - # skip empty elements - [[ -z ${new_file} ]] && continue - # skip names with *.new.* in it - [[ "${new_file}" == *.new.* ]] && continue - base=${new_file%.*} - tmp_array+=("${base}"*) - done + + while IFS= read -r -d '' new_file; do + new_file=$(basename "${new_file}") + # shellcheck disable=2001 + new_file_renamed=$(sed 's/\.sp\(\w\)$/.new.sp\1/' <<< "${new_file}") + cp -fa "${SPHINX_EFS}${new_file}" "${SPHINX_VOLUME}${new_file_renamed}" + tmp_array+=("${new_file_renamed}") + done < <(find "${SPHINX_EFS}" -name "${sphinx_index}*" -print0) + if ((${#tmp_array[@]})); then - mapfile -t local_new_files < <(printf "%s\\n" "${tmp_array[@]}" | sort -u | tr '\n' ' ') - for rotated in ${local_new_files[@]}; do # shellcheck disable=SC2068 - # skip empty elements - [[ -z ${rotated} ]] && continue - # skip names with *.new.* in it - [[ "${rotated}" == *.new.* ]] && continue - base=${rotated%.*} - extension=${rotated##*.} - new_file="${base}.new.${extension}" - mv -f "${rotated}" "${new_file}" - new_files_merged+=("${new_file}") - done + # remove blank strings from array + IFS=" " read -r -a new_files_merged <<< ${tmp_array[@]} + if ((${#new_files_merged[@]})); then + # start index rotation + echo "${LOG_PREFIX}-> $(date +"%F %T") restart searchd for index rotation..." + pkill -1 searchd + + # wait until all files new_files and locally renamed files have been renamed / rotated in SPHINX_VOLUME + echo "${LOG_PREFIX}-> $(date +"%F %T") wait for index rotation..." + all_files_are_gone=false + while ! ${all_files_are_gone}; do + all_files_are_gone=true + for new_file in ${new_files_merged[@]}; do + # skip empty elements + [[ -z ${new_file} ]] && continue + [ -f "${SPHINX_VOLUME}${new_file}" ] && all_files_are_gone=false + done + sleep 5 + done + fi fi - popd done -# remove duplicates from array with new index files -mapfile -t new_files_merged < <(printf "%s\\n" "${new_files_merged[@]}" | sort -u | tr '\n' ' ') -# remove blank strings from array -IFS=" " read -r -a new_files_merged <<< ${new_files_merged[@]} -if ((${#new_files_merged[@]})); then - # start index rotation if new files have been synced - echo "${LOG_PREFIX}-> $(date +"%F %T") restart searchd for index rotation..." - pkill -1 searchd - - # wait until all files new_files and locally renamed files have been renamed / rotated in SPHINX_VOLUME - echo "${LOG_PREFIX}-> $(date +"%F %T") wait for index rotation..." - all_files_are_gone=false - while ! ${all_files_are_gone}; do - all_files_are_gone=true - for new_file in ${new_files_merged[@]}; do - # skip empty elements - [[ -z ${new_file} ]] && continue - [ -f "${SPHINX_VOLUME}${new_file}" ] && all_files_are_gone=false - done - sleep 5 - done -fi echo "${LOG_PREFIX}-> $(date +"%F %T") finished" \ No newline at end of file