Like 0
Fork 0

Last active 1740064516

s3 s3cmd scp shell ssh

anthony's Avatar anthony revised this gist 1740064515. Go to revision

No changes

anthony's Avatar anthony revised this gist 1740035334. Go to revision

No changes

anthony's Avatar anthony revised this gist 1740033559. Go to revision

No changes

anthony's Avatar anthony revised this gist 1740033554. Go to revision

1 file changed, 341 insertions

s3-backup.sh(file created)

@@ -0,0 +1,341 @@
1 + #!/bin/bash
2 + #####################################################################
3 + # #
4 + # Stupidly simple backup script for own projects #
5 + # #
6 + # Author: Anthony Axenov (Антон Аксенов) #
7 + # Version: 1.2 #
8 + # License: WTFPLv2 More info (RU): https://axenov.dev/?p=1272 #
9 + # #
10 + #####################################################################
11 +
12 + # use remote storages ===============================================
13 +
14 + USE_SSH=1
15 + USE_S3=1
16 +
17 + # database credentials ==============================================
18 +
19 + DBUSER=
20 + DBPASS=
21 + DBNAME=
22 + DBCHARSET="utf8"
23 +
24 + # dates for file structure ==========================================
25 +
26 + TODAY_DIR="$(date +%Y.%m.%d)"
27 + TODAY_FILE="$(date +%H.%M)"
28 +
29 + # local storage =====================================================
30 +
31 + LOCAL_BAK_DIR="/backup"
32 + LOCAL_BAK_PATH="$LOCAL_BAK_DIR/$TODAY_DIR"
33 +
34 + # database backup file
35 + LOCAL_SQL_FILE="$TODAY_FILE-db.sql.gz"
36 + LOCAL_SQL_PATH="$LOCAL_BAK_PATH/$LOCAL_SQL_FILE"
37 +
38 + # project path and backup file
39 + LOCAL_SRC_DIR="/var/www/html"
40 + LOCAL_SRC_FILE="$TODAY_FILE-src.tar.gz"
41 + LOCAL_SRC_PATH="$LOCAL_BAK_PATH/$LOCAL_SRC_FILE"
42 +
43 + # log file
44 + LOG_FILE="$TODAY_FILE.log"
45 + LOG_PATH="$LOCAL_BAK_PATH/$LOG_FILE"
46 +
47 + # remote storages ===================================================
48 +
49 + SSH_HOST="user@example.com"
50 + SSH_BAK_DIR="/backup"
51 + SSH_BAK_PATH="$SSH_BAK_DIR/$TODAY_DIR"
52 + SSH_SQL_FILE="$SSH_BAK_PATH/$LOCAL_SQL_FILE"
53 + SSH_SRC_FILE="$SSH_BAK_PATH/$LOCAL_SRC_FILE"
54 + SSH_LOG_FILE="$SSH_BAK_PATH/$LOG_FILE"
55 +
56 + S3_BUCKET="s3://my.bucket"
57 + S3_DIR="$S3_BUCKET/$TODAY_DIR"
58 + S3_SQL_FILE="$S3_DIR/$LOCAL_SQL_FILE"
59 + S3_SRC_FILE="$S3_DIR/$LOCAL_SRC_FILE"
60 + S3_LOG_FILE="$S3_DIR/$LOG_FILE"
61 +
62 + # autoremove ========================================================
63 +
64 + # time to live on different storages
65 + TTL_LOCAL=3
66 + TTL_SSH=7
67 + TTL_S3=60
68 +
69 + # autoremove flags
70 + CLEAR_SSH=1
71 + CLEAR_S3=1
72 +
73 + # notifications =====================================================
74 +
75 + USE_NTFY=1
76 + NTFY_TITLE="Backup script"
77 + NTFY_CHANNEL=
78 +
79 + #====================================================================
80 + #
81 + # Functions used for the whole backup flow
82 + #
83 + #====================================================================
84 +
85 + # prints arguments to stdout and into log file
86 + log() {
87 + echo -e "[$(date +%H:%M:%S)] $*" | tee -a "$LOG_PATH"
88 + }
89 +
90 + # sends notification with information
91 + ntfy_info() {
92 + [ $USE_NTFY == 1 ] && ntfy send \
93 + --title "$NTFY_TITLE" \
94 + --message "$1" \
95 + --priority 1 \
96 + "$NTFY_CHANNEL"
97 + }
98 +
99 + # sends notification with warning
100 + ntfy_warn() {
101 + [ $USE_NTFY == 1 ] && ntfy send \
102 + --title "$NTFY_TITLE" \
103 + --tags "warning" \
104 + --message "$1" \
105 + --priority 5 \
106 + "$NTFY_CHANNEL"
107 + }
108 +
109 + # prints initialized parameters
110 + show_params() {
111 + log "Initialized parameters:"
112 +
113 + log "├ [ Remotes ]"
114 + log "│\t├ USE_SSH = $USE_SSH"
115 + [ $USE_SSH == 1 ] && log "│\t├ SSH_HOST = $SSH_HOST"
116 + log "│\t├ USE_S3 = $USE_S3"
117 + [ $USE_S3 == 1 ] && log "│\t├ S3_BUCKET = $S3_BUCKET"
118 +
119 + log "├ [ Database ]"
120 + log "│\t├ DBUSER = $DBUSER"
121 + log "│\t├ DBNAME = $DBNAME"
122 + log "│\t├ DBCHARSET = $DBCHARSET"
123 + log "│\t├ LOCAL_SQL_PATH = $LOCAL_SQL_PATH"
124 + [ $USE_SSH == 1 ] && log "│\t├ SSH_SQL_FILE = $SSH_SQL_FILE"
125 + [ $USE_S3 == 1 ] && log "│\t├ S3_SQL_FILE = $S3_SQL_FILE"
126 +
127 + log "├ [ Sources ]"
128 + log "│\t├ LOCAL_SRC_DIR = $LOCAL_SRC_DIR"
129 + log "│\t├ LOCAL_SRC_PATH = $LOCAL_SRC_PATH"
130 + [ $USE_SSH == 1 ] && log "│\t├ SSH_SRC_FILE = $SSH_SRC_FILE"
131 + [ $USE_S3 == 1 ] && log "│\t├ S3_SRC_FILE = $S3_SRC_FILE"
132 +
133 + log "├ [ Log ]"
134 + log "│\t├ LOG_PATH = $LOG_PATH"
135 + [ $USE_SSH == 1 ] && log "│\t├ SSH_LOG_FILE = $SSH_LOG_FILE"
136 + [ $USE_S3 == 1 ] && log "│\t├ S3_LOG_FILE = $S3_LOG_FILE"
137 +
138 + log "├ [ Autoclear ]"
139 + log "│\t├ TTL_LOCAL = $TTL_LOCAL"
140 + [ $USE_SSH == 1 ] && {
141 + log "│\t├ CLEAR_SSH = $CLEAR_SSH"
142 + log "│\t├ TTL_SSH = $TTL_SSH"
143 + }
144 + [ $USE_S3 == 1 ] && {
145 + log "│\t├ CLEAR_S3 = $CLEAR_S3"
146 + log "│\t├ TTL_S3 = $TTL_S3"
147 + }
148 +
149 + log "└ [ ntfy ]"
150 + log "\t├ USE_NTFY = $USE_NTFY"
151 + [ $USE_NTFY == 1 ] && log "\t├ NTFY_TITLE = $NTFY_TITLE"
152 + [ $USE_NTFY == 1 ] && log "\t└ NTFY_CHANNEL = $NTFY_CHANNEL"
153 + }
154 +
155 + # initializes directories for backup
156 + init_dirs() {
157 + if [ ! -d "$LOCAL_BAK_PATH" ]; then
158 + mkdir -p $LOCAL_BAK_PATH
159 + fi
160 + [ $USE_SSH == 1 ] && ssh $SSH_HOST "mkdir -p $SSH_BAK_PATH"
161 + }
162 +
163 + # clears old local backups
164 + clear_local_backups() {
165 + log "\tLocal:"
166 + log $(find "$LOCAL_BAK_DIR" -type d -mtime +"$TTL_LOCAL" | sort)
167 + find "$LOCAL_BAK_DIR" -type d -mtime +"$TTL_LOCAL" | xargs rm -rf
168 + }
169 +
170 + # clears old backups on remote ssh storage
171 + clear_ssh_backups() {
172 + if [ $USE_SSH == 1 ] && [ $CLEAR_SSH == 1 ]; then
173 + log "\tSSH:"
174 + log $(ssh "$SSH_HOST" "find $SSH_BAK_DIR -type d -mtime +$TTL_SSH" | sort)
175 + ssh "$SSH_HOST" "find $SSH_BAK_DIR -type d -mtime +$TTL_SSH | xargs rm -rf"
176 + else
177 + log "\tSSH: disabled (\$USE_SSH, \$CLEAR_SSH)"
178 + fi
179 + }
180 +
181 + # clears backups on remote s3 storage
182 + clear_s3_backups() {
183 + # https://gist.github.com/JProffitt71/9044744?permalink_comment_id=3539681#gistcomment-3539681
184 + if [ $USE_S3 == 1 ] && [ $CLEAR_S3 == 1 ]; then
185 + log "\tS3:"
186 + OLDER_THAN=$(date -d "$TTL_S3 days ago" "+%s")
187 + s3cmd ls -r $S3_DIR | while read -r line; do
188 + FILETIME=$(echo "$line" | awk {'print $1" "$2'})
189 + FILETIME=$(date -d "$FILETIME" "+%s")
190 + if [[ $FILETIME -le $OLDER_THAN ]]; then
191 + FILEPATH=$(echo "$line" | awk {'print $4'})
192 + if [ $FILEPATH != "" ]; then
193 + log "$line"
194 + s3cmd del $FILEPATH
195 + fi
196 + fi
197 + done
198 + else
199 + log "\tS3: disabled (\$USE_S3 + \$CLEAR_S3)"
200 + fi
201 + }
202 +
203 + # clears old backups
204 + clear_backups() {
205 + echo
206 + log "1/7 Removing old backups..."
207 + clear_local_backups
208 + clear_ssh_backups
209 + clear_s3_backups
210 + }
211 +
212 + # makes archive with database dump
213 + backup_db() {
214 + echo
215 + log "2/7 Dumping DB: $DBNAME..."
216 + mysqldump \
217 + --user=$DBUSER \
218 + --password=$DBPASS \
219 + --opt \
220 + --default-character-set=$DBCHARSET \
221 + --quick \
222 + $DBNAME | gzip > $LOCAL_SQL_PATH
223 + if [ $? == 0 ]; then
224 + log "\t- OK"
225 + send_db_ssh
226 + send_db_s3
227 + else
228 + log "\t- ERROR: failed to create dump. Exit-code: $?"
229 + ntfy_warn "ERROR: failed to create dump"
230 + log "3/7 Sending database backup to $SSH_HOST... skipped"
231 + log "4/7 Sending database backup to $S3_DIR... skipped"
232 + fi
233 + }
234 +
235 + # sends database archive into ssh remote storage
236 + send_db_ssh() {
237 + echo
238 + log "3/7 Sending database backup to $SSH_HOST..."
239 + if [ $USE_SSH == 1 ]; then
240 + rsync --progress "$LOCAL_SQL_PATH" "$SSH_HOST:$SSH_SQL_FILE"
241 + if [ $? == 0 ]; then
242 + log "\t- OK"
243 + else
244 + log "\t- ERROR: failed to send DB backup to $SSH_HOST. Exit-code: $?"
245 + ntfy_warn "ERROR: failed to send DB backup to $SSH_HOST"
246 + fi
247 + else
248 + log "\t- disabled (\$USE_SSH)"
249 + fi
250 + }
251 +
252 + # sends database archive into s3 remote storage
253 + send_db_s3() {
254 + echo
255 + log "4/7 Sending database backup to $S3_DIR..."
256 + if [ $USE_S3 == 1 ]; then
257 + s3cmd put "$LOCAL_SQL_PATH" "$S3_SQL_FILE"
258 + if [ $? == 0 ]; then
259 + log "\t- OK"
260 + else
261 + log "\t- ERROR: failed to send DB backup to $S3_DIR. Exit-code: $?"
262 + ntfy_warn "ERROR: failed to send DB backup to $S3_DIR"
263 + fi
264 + else
265 + log "\t- disabled (\$USE_SSH)"
266 + fi
267 + }
268 +
269 + # makes archive with project sources
270 + backup_src() {
271 + echo
272 + log "5/7 Compressing project dir: $LOCAL_SRC_DIR..."
273 + tar -zcf "$LOCAL_SRC_PATH" "$LOCAL_SRC_DIR"
274 + if [ $? == 0 ]; then
275 + log "\t- OK"
276 + send_src_ssh
277 + send_src_s3
278 + else
279 + log "\t- ERROR: failed to compress project. Exit-code: $?"
280 + ntfy_warn "ERROR: failed to compress project"
281 + log "6/7 Sending project backup to $SSH_HOST... skipped"
282 + log "7/7 Sending project backup to $S3_DIR... skipped"
283 + fi
284 + }
285 +
286 + # sends sources archive into ssh remote storage
287 + send_src_ssh() {
288 + echo
289 + log "6/7 Sending project backup to $SSH_HOST..."
290 + if [ $USE_SSH == 1 ]; then
291 + rsync --progress "$LOCAL_SRC_PATH" "$SSH_HOST:$SSH_SRC_FILE"
292 + if [ $? == 0 ]; then
293 + log "\t- OK"
294 + else
295 + log "\t- ERROR: failed to send project backup to $SSH_HOST. Exit-code: $?"
296 + ntfy_warn "ERROR: failed to send project backup to $SSH_HOST"
297 + fi
298 + else
299 + log "\t- disabled"
300 + fi
301 + }
302 +
303 + # sends sources archive into s3 remote storage
304 + send_src_s3() {
305 + echo
306 + log "7/7 Sending project backup to $S3_DIR..."
307 + s3cmd put "$LOCAL_SRC_PATH" "$S3_SRC_FILE"
308 + if [ $? == 0 ]; then
309 + log "\t- OK"
310 + else
311 + log "\t- ERROR: failed to send database backup to $S3_DIR. Exit-code: $?"
312 + ntfy_warn "ERROR: failed to send project backup to $S3_DIR"
313 + fi
314 + }
315 +
316 + # prints used/free space on local storage
317 + show_finish() {
318 + echo
319 + log "Finish!"
320 + log "Used space: $(du -h "$LOCAL_BAK_PATH" | tail -n1)" # вывод размера папки с бэкапами за текущий день
321 + log "Free space: $(df -h "$LOCAL_BAK_PATH" | tail -n1 | awk '{print $4}')" # вывод свободного места на локальном диске
322 + echo
323 + }
324 +
325 + # sends log file into both remote storage
326 + send_log() {
327 + [ $USE_SSH == 1 ] && rsync --progress "$LOG_PATH" "$SSH_HOST:$SSH_LOG_FILE"
328 + [ $USE_S3 == 1 ] && s3cmd put "$LOG_PATH" "$S3_LOG_FILE"
329 + }
330 +
331 + # main flow =========================================================
332 +
333 + log "Start ----------------------------------------------------------"
334 + show_params
335 + init_dirs
336 + clear_backups
337 + backup_db
338 + backup_src
339 + show_finish
340 + send_log
341 + ntfy_info "Finish!"
Newer Older

Powered by Opengist ⋅ Load: 144ms⋅

English