-
Notifications
You must be signed in to change notification settings - Fork 0
/
aws-s3-sync.sh
executable file
·116 lines (89 loc) · 2.38 KB
/
aws-s3-sync.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/bin/sh
set -e
if [ "$AWS_DEBUG" ]; then
set -x
fi
DIR="$(readlink -f "$(dirname "$0")")"
REGION=$1
BUCKET=$2
REMOTE=${3%%/}/
LOCALS=${4%%/}/
if [ $# -ge 4 ]; then
shift 4
else
shift $#
fi
path="$REMOTE"
orig="$LOCALS"
sync="$LOCALS.sync"
rm -rf "$sync"
mkdir -p "$orig" "$sync"
for file in `ls -1 "$orig"`; do
touch "$sync/$file"
done
more='true'
while [ "$more" ] && "$more"; do
if [ "${next}" ]; then
next="continuation-token ${next}"
fi
qstr="`$DIR/aws-s3-data.sh list-type 2 prefix "${path}" delimiter / ${next}`"
list="`$DIR/aws-s3.sh "" "${REGION}" "${BUCKET}" "" "${qstr}"`"
list="${list:?}"
echo "$list" | tr -d '\n' | hxselect -s'\n' 'ListBucketResult>Contents' | while read elem; do
name=`echo "$elem" | hxselect -c 'Key'`
file=${name##${path}}
etag=`echo "$elem" | hxselect -c 'ETag'`
if [ -z "$file" ] || [ "$name" = "$file" ]; then
continue
fi
if echo "$file" | grep -q '/'; then
continue
fi
r_dt=`echo "$elem" | hxselect -c 'LastModified'`
r_dt=`echo -n "$r_dt" | date -uf - +%s`
l_dt=`date -ur "$orig/$file" -Is 2>/dev/null || :`
l_dt=`echo -n "$l_dt" | date -uf - +%s`
r_sz=`echo "$elem" | hxselect -c 'Size'`
l_sz=`wc -c "$orig/$file" 2>/dev/null | cut -f1 -d' '`
echo
echo "Filename: $file"
if [ -z "$l_dt" ] || [ -z "$r_dt" ] || [ "${l_sz:-0}" -gt "$r_sz" ]; then
echo "Remote size: $r_sz"
echo " Local size: $l_sz"
echo "Downloading..."
echo
$DIR/aws-s3.sh "" "${REGION}" "${BUCKET}" "$name" "" "$@" -o "$sync/$file"
mv -f "$sync/$file" "$orig/$file"
elif [ "$l_dt" -ne "$r_dt" ]; then
echo "Remote date: `date -d @$r_dt -Is`"
echo " Local date: `date -d @$l_dt -Is`"
echo "Downloading..."
echo
$DIR/aws-s3.sh "" "${REGION}" "${BUCKET}" "$name" "" "$@" -o "$sync/$file"
mv -f "$sync/$file" "$orig/$file"
elif [ "$l_sz" -lt "$r_sz" ]; then
echo "Remote size: $r_sz"
echo " Local size: $l_sz"
echo "Continueing..."
echo
$DIR/aws-s3.sh "" "${REGION}" "${BUCKET}" "$name" "" "$@" -o "$orig/$file" -C -
rm -f "$sync/$file"
else
echo "Skipping..."
echo
rm -f "$sync/$file"
fi
touch -d @$r_dt "$orig/$file"
done
next=`echo "$list" | hxselect -c 'NextContinuationToken'`
more=`echo "$list" | hxselect -c 'IsTruncated'`
done
for file in `ls -1 "$sync"`; do
echo
echo "Filename: $file"
echo "NotFound..."
echo
rm -f "$sync/$file"
rm -f "$orig/$file"
done
rmdir "$sync"